From 388027c55bc16296c30c0be5983b6ca52bfa86a3 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Wed, 19 Aug 2026 10:57:19 -0700 Subject: [PATCH] feat(redis): add stale-on-error recovery --- README.md | 106 +- package.json | 1 + scripts/benchmark-request-local.mjs | 25 +- scripts/benchmark-stale-on-error.mjs | 378 +++++++ scripts/test-package.mjs | 165 ++- src/config.ts | 15 + src/datadog.ts | 11 + src/dialcache.ts | 130 ++- src/index.ts | 2 + src/internal/cache-result.ts | 11 +- src/internal/redis-cache.ts | 121 ++- src/internal/redis-payload.ts | 104 +- src/internal/redis-script-reply.ts | 20 + src/internal/redis-scripts.ts | 14 + src/internal/runtime-config.ts | 70 +- src/metrics.ts | 26 +- src/node-redis.ts | 110 +- src/prometheus.ts | 19 + src/redis-client.ts | 70 +- src/redis-protocol.ts | 11 +- src/valkey-glide.ts | 121 ++- test/datadog.test.ts | 42 + test/dialcache-config-ramp.test.ts | 248 ++++- test/dialcache-liveness.test.ts | 3 + test/dialcache-logger.test.ts | 1 + test/dialcache-metrics.test.ts | 136 ++- .../dialcache-observability-internals.test.ts | 57 +- test/dialcache-redis-read-deadline.test.ts | 15 + test/dialcache-redis.test.ts | 5 +- test/dialcache-shadow-confirmation.test.ts | 89 +- test/dialcache-shadow-validation.test.ts | 18 +- test/dialcache-stale-on-error.test.ts | 984 ++++++++++++++++++ test/fake-redis.ts | 28 +- test/node-redis.test.ts | 272 ++++- test/prometheus.test.ts | 51 + test/redis-cluster.integration.test.ts | 25 +- test/redis-payload.test.ts | 75 +- test/redis-real.integration.test.ts | 264 ++++- test/valkey-glide.test.ts | 355 +++++-- 39 files changed, 3794 insertions(+), 404 deletions(-) create mode 100644 scripts/benchmark-stale-on-error.mjs create mode 100644 test/dialcache-stale-on-error.test.ts diff --git a/README.md b/README.md index 405f0e6..9f58031 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![Codecov](https://codecov.io/gh/lan17/DialCache/branch/main/graph/badge.svg)](https://codecov.io/gh/lan17/DialCache) [![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/lan17/DialCache/badge)](https://scorecard.dev/viewer/?uri=github.com/lan17/DialCache) -Fine-grained TypeScript caching with explicit enabled contexts, request-local memoization, process-local and Redis TTL caching, stable key construction, runtime rollout controls, request coalescing, adapter-based observability, and Redis watermark-based targeted invalidation. +Fine-grained TypeScript caching with explicit enabled contexts, request-local memoization, process-local and Redis TTL caching, opt-in stale-on-error recovery, stable key construction, runtime rollout controls, request coalescing, adapter-based observability, and Redis watermark-based targeted invalidation. ## Contents @@ -17,7 +17,7 @@ Fine-grained TypeScript caching with explicit enabled contexts, request-local me - [Keys, ids, and extra dimensions](#keys-ids-and-extra-dimensions) - [Runtime config and ramp controls](#runtime-config-and-ramp-controls) - [Cache layers](#cache-layers) - - [Request-local cache](#request-local-cache) · [Process-local cache](#process-local-cache) · [Redis-backed TTL cache](#redis-backed-ttl-cache) · [Remote read deadlines](#remote-read-deadlines-and-async-liveness) · [Serialization](#serialization) · [Compression](#compression) · [Shadow validation](#shadow-validation) + - [Request-local cache](#request-local-cache) · [Process-local cache](#process-local-cache) · [Redis-backed TTL cache](#redis-backed-ttl-cache) · [Stale on source error](#stale-on-source-error) · [Remote read deadlines](#remote-read-deadlines-and-async-liveness) · [Serialization](#serialization) · [Compression](#compression) · [Shadow validation](#shadow-validation) - [Cached-value ownership](#cached-value-ownership) - [Targeted invalidation and watermarks](#targeted-invalidation-and-watermarks) - [Request coalescing](#request-coalescing) @@ -82,9 +82,9 @@ request-local cache -> process-local cache -> Redis cache -> fallback function - Results from the lower chain are memoized request-locally when that layer is enabled. - Process-local hits return immediately. - Process-local misses try Redis and populate the process-local cache on a Redis hit. -- Redis misses call the fallback and attempt to populate Redis and, when active, the process-local cache. Tracked invalidation may suppress both publications. +- Redis misses call the fallback and attempt to populate Redis and, when active, the process-local cache. Tracked invalidation may suppress both publications. When stale-on-error is opted in, a fallback rejection instead triggers one bounded Redis recovery read that may return a retained value within the configured maximum age. - Selected Redis keys can execute non-serving [shadow work](#shadow-validation) that validates hits and fills clean misses, even before Redis is allowed to serve callers. -- Redis read failures and timeouts are logged, counted in metrics, and fail open without attempting a second Redis operation. Redis write failures also fail open. `invalidateRemote` requires a configured Redis client; missing configuration and Redis failures are logged, counted, and rethrown so callers do not assume invalidation succeeded. +- Initial Redis read failures and timeouts are logged, counted in metrics, and fail open without attempting stale recovery. Redis write failures also fail open. `invalidateRemote` requires a configured Redis client; missing configuration and Redis failures are logged, counted, and rethrown so callers do not assume invalidation succeeded. - Cache-key construction and config-provider failures also fail open and run the fallback uncached. - A missing effective process-local/Redis TTL disables that layer by policy; a configured TTL with no ramp defaults to 100%. Disabled layers record a disabled reason and fall through to the next layer/fallback. @@ -209,21 +209,21 @@ Instance-wide behavior is set through the `DialCache` constructor: | `metrics` | disabled | A `DialCacheMetricsAdapter` (see [Metrics](#metrics)). | | `logger` | `console` | Receives operational cache failures and opted-in confirmed shadow mismatch warnings (`debug`, `warn`, `error`). Synchronous throws and rejections from returned promises or thenables are isolated without being awaited. | -Per-invocation cache policy is a `DialCacheKeyConfig`: per-layer `ttlSec` and `ramp` maps keyed by `CacheLayer.LOCAL` (process-local) and `CacheLayer.REMOTE` (Redis), a `requestLocal` boolean, a `coalesce` boolean (see [Request coalescing](#request-coalescing)), an optional `remoteReadTimeoutMs`, and an optional `shadow` group. `ShadowConfig` contains the independent shadow `ramp` percentage plus the default-off `logMismatches` control. +Per-invocation cache policy is a `DialCacheKeyConfig`: per-layer `ttlSec` and `ramp` maps keyed by `CacheLayer.LOCAL` (process-local) and `CacheLayer.REMOTE` (Redis), a `requestLocal` boolean, a `coalesce` boolean (see [Request coalescing](#request-coalescing)), an optional `staleOnErrorMaxAgeSec`, an optional `remoteReadTimeoutMs`, and an optional `shadow` group. `ShadowConfig` contains the independent shadow `ramp` percentage plus the default-off `logMismatches` control. Every cached definition or `getOrLoad()` invocation can provide an optional per-use-case `defaultConfig`. It is the baseline policy, and the `cacheConfigProvider` result is a sparse field-level overlay on that baseline. For cache enablement fields, precedence is runtime config, then `defaultConfig`, then DialCache's disabled baseline. For the remote-read deadline, precedence is runtime `remoteReadTimeoutMs`, `defaultConfig.remoteReadTimeoutMs`, `redis.readTimeoutMs`, then the 50 ms library default. -The disabled baseline sets `requestLocal` to false, leaves the process-local and Redis TTLs unset, and sets `shadow.ramp` to 0% with mismatch logging false. A shared layer with no effective TTL is disabled by policy. When a shared layer has an effective TTL but no effective ramp, its ramp defaults to 100%. Shadow work remains disabled unless `shadow.ramp` is explicitly greater than zero. +The disabled baseline sets `requestLocal` to false, leaves the process-local and Redis TTLs and stale-on-error maximum unset, and sets `shadow.ramp` to 0% with mismatch logging false. A shared layer with no effective TTL is disabled by policy. When a shared layer has an effective TTL but no effective ramp, its ramp defaults to 100%. Shadow work remains disabled unless `shadow.ramp` is explicitly greater than zero. -`DialCacheKeyConfig` preserves an omitted `requestLocal` as `undefined` so the overlay can distinguish omission from an explicit `false`; the effective value still defaults to false after resolution. An omitted `coalesce` is preserved the same way, and its effective value defaults to true, so request coalescing stays on unless a use case explicitly opts out. +`DialCacheKeyConfig` preserves an omitted `requestLocal` as `undefined` so the overlay can distinguish omission from an explicit `false`; the effective value still defaults to false after resolution. An omitted `coalesce` is preserved the same way, and its effective value defaults to true, so request coalescing stays on unless a use case explicitly opts out. An omitted `staleOnErrorMaxAgeSec` inherits an earlier value in a sparse runtime overlay and otherwise leaves recovery off; `0` explicitly disables an inherited stale policy. -A provider result of `null` (or defensive `undefined`) applies no overrides. An empty `DialCacheKeyConfig` and omitted runtime fields also inherit the baseline. Top-level fields, cache-layer leaves, and leaves inside `shadow` merge independently; an explicit `false` logging flag overrides an inherited `true`. Use explicit values to override inherited policy: `requestLocal: false` disables request-local caching and a layer ramp of `0` disables that shared layer. `DialCacheKeyConfig.disabled()` is the complete new-cache-invocation kill switch in one call: request-local and shadow work off, shadow logging off, and both shared layers ramped to 0. It leaves `coalesce` unset: with every layer off there is no in-flight sharing to disable, and a use case ramped back up at runtime coalesces again unless it explicitly opts out. It does not cancel already-admitted work, and explicit maintenance operations such as `invalidateRemote()` remain available. To stop new cache-invocation Redis reads and fills while preserving other runtime settings, explicitly set both `ramp.remote` and `shadow.ramp` to `0`; the remote ramp alone stops serving but does not override an inherited nonzero shadow ramp. +A provider result of `null` (or defensive `undefined`) applies no overrides. An empty `DialCacheKeyConfig` and omitted runtime fields also inherit the baseline. Top-level fields, cache-layer leaves, and leaves inside `shadow` merge independently; an explicit `false` logging flag overrides an inherited `true`. Use explicit values to override inherited policy: `requestLocal: false` disables request-local caching, `staleOnErrorMaxAgeSec: 0` disables stale recovery, and a layer ramp of `0` disables that shared layer. `DialCacheKeyConfig.disabled()` is the complete new-cache-invocation kill switch in one call: request-local, stale recovery, and shadow work off, shadow logging off, and both shared layers ramped to 0. It leaves `coalesce` unset: with every layer off there is no in-flight sharing to disable, and a use case ramped back up at runtime coalesces again unless it explicitly opts out. It does not cancel already-admitted work, and explicit maintenance operations such as `invalidateRemote()` remain available. To stop new cache-invocation Redis reads and fills while preserving other runtime settings, explicitly set both `ramp.remote` and `shadow.ramp` to `0`; the remote ramp alone stops serving but does not override an inherited nonzero shadow ramp. -DialCache validates `defaultConfig` when `cached()` registers a definition and whenever `getOrLoad()` is invoked: TTLs must be positive safe integers no greater than 31,536,000 seconds (a fixed 365-day duration), remote-read deadlines must be positive safe integers within their documented limit, layer and shadow ramps must be finite percentages from 0 to 100, layer maps and `shadow` must be objects, and `requestLocal`, `coalesce`, and `shadow.logMismatches` must be booleans when present. Invalid defaults are rejected immediately. +DialCache validates `defaultConfig` when `cached()` registers a definition and whenever `getOrLoad()` is invoked: TTLs must be positive safe integers no greater than 31,536,000 seconds (a fixed 365-day duration), a positive stale-on-error maximum must be a safe integer in the same range and strictly greater than the remote TTL, remote-read deadlines must be positive safe integers within their documented limit, layer and shadow ramps must be finite percentages from 0 to 100, layer maps and `shadow` must be objects, and `requestLocal`, `coalesce`, and `shadow.logMismatches` must be booleans when present. `0` is the one valid non-positive stale maximum and explicitly disables recovery. Invalid defaults are rejected immediately. Each registration or one-shot invocation captures an immutable internal snapshot of `defaultConfig`; mutating the supplied config or its maps later does not change that operation's baseline. Runtime policy changes belong in the provider's returned overlay. -Runtime TTL and ramp leaves are used as supplied instead of falling back to valid default leaves. A TTL outside the same 1-to-31,536,000-second range disables that layer with `invalid_ttl`; a nonnumeric, non-finite, or out-of-range ramp disables it with `invalid_ramp`. Valid ramps include both `0` and `100`. Other layers can still run, and invalid leaves also record a `config_resolution` error so provider garbage is alertable separately from intentional ramp-downs. A malformed runtime config object, layer-map shape, `requestLocal` value, `coalesce` value, explicit `remoteReadTimeoutMs`, or removed top-level `shadowRamp` fails config resolution for the invocation, records `config_error`, and executes the fallback uncached without attempting Redis. The public `DialCacheKeyConfig` constructor and static defaults likewise reject `shadowRamp` immediately; migrate it to `shadow.ramp`. +Runtime TTL and ramp leaves are used as supplied instead of falling back to valid default leaves. A TTL outside the same 1-to-31,536,000-second range disables that layer with `invalid_ttl`; a nonnumeric, non-finite, or out-of-range ramp disables it with `invalid_ramp`. Valid ramps include both `0` and `100`. Other layers can still run, and invalid leaves also record a `config_resolution` error so provider garbage is alertable separately from intentional ramp-downs. A malformed runtime config object, layer-map shape, `requestLocal` value, `coalesce` value, explicit `remoteReadTimeoutMs`, or removed top-level `shadowRamp` fails config resolution for the invocation, records `config_error`, and executes the fallback uncached without attempting Redis. An invalid runtime `staleOnErrorMaxAgeSec` is narrower: DialCache records `config_resolution`, disables recovery for that invocation, and preserves an otherwise valid fresh Redis policy. The public `DialCacheKeyConfig` constructor and static defaults likewise reject `shadowRamp` immediately; migrate it to `shadow.ramp`. An invalid runtime `shadow.ramp` does not affect the cache result or disable an otherwise valid Redis policy. If normal traversal reaches an otherwise shadow-eligible Redis path, DialCache skips shadow work and records a `config_resolution` error. An invalid runtime `shadow.logMismatches` likewise preserves the cache result, Redis policy, shadow result, and shadow metric while suppressing the warning. DialCache validates this diagnostic leaf only after the metrics hook, exact-key cohort, and capacity gates admit shadow work, then records one remote `config_resolution` error for that admitted resolution. @@ -246,6 +246,8 @@ const dialcache = new DialCache({ }, // Can be changed by the provider at runtime for this use case. remoteReadTimeoutMs: 35, + // Sparse override of the maximum retained age; use 0 to turn recovery off. + staleOnErrorMaxAgeSec: 1_800, }); } return null; // apply no overrides; use the cached function's baseline @@ -259,6 +261,8 @@ const getUser = dialcache.cached((userId: string) => db.fetchUser(userId), { defaultConfig: new DialCacheKeyConfig({ // Omitted ramps default to 100% because these layers have TTLs. ttlSec: { [CacheLayer.LOCAL]: 30, [CacheLayer.REMOTE]: 300 }, + // Keep Redis data for up to one hour and serve it only after a source error. + staleOnErrorMaxAgeSec: 3_600, }), }); ``` @@ -358,7 +362,7 @@ async function shutdown(): Promise { } ``` -`redis.client` is required when Redis is configured and accepts the semantic `DialCacheRedisClient` interface. `redis.readTimeoutMs` is optional and sets the instance default for remote reads; omit it to use 50 ms. Create and connect the underlying client before constructing `DialCache`. Node-redis users should register the supplied mutation scripts and wrap their client with `createNodeRedisDialCacheClient` as shown above; the adapter performs reads with native commands. The registered `dialcache*` methods are DialCache's wiring, not a write API: they return raw script replies — the stamp's `2` means the placeholder was lost, not success — so code invoking them directly must map stamp replies through `resolveTrackedRedisWriteReply` from `dialcache/redis-protocol`. The helper requires node-redis's promise API and does not support `legacyMode`, whose callback surface and `.v4` view do not expose the complete native-command-plus-custom-script contract together. +`redis.client` is required when Redis is configured and accepts the semantic `DialCacheRedisClient` interface. `redis.readTimeoutMs` is optional and sets the instance default for remote reads; omit it to use 50 ms. Create and connect the underlying client before constructing `DialCache`. Node-redis users should register the supplied mutation scripts and wrap their client with `createNodeRedisDialCacheClient` as shown above; the adapter performs reads with native commands. The registered `dialcache*` methods are DialCache's wiring, not a write API: both stamp methods return raw script replies — `2` means the placeholder was lost, not success — so code invoking them directly must map replies through `resolveUntrackedRedisWriteReply` or `resolveTrackedRedisWriteReply` from `dialcache/redis-protocol`. The helper requires node-redis's promise API and does not support `legacyMode`, whose callback surface and `.v4` view do not expose the complete native-command-plus-custom-script contract together. Valkey GLIDE users pass an already-created standalone or cluster client and its module namespace to the GLIDE adapter: @@ -402,17 +406,52 @@ Awaiting those public promises does not drain detached shadow work. Shadow sched Neither adapter owns additional resources: both dispatch their mutation scripts by source SHA1 and hold no native handles, so the application simply closes the underlying client after draining work. Applications that construct their own GLIDE `Script` objects should know that on GLIDE 2.0.0, releasing a handle has been observed to break other live handles for the same script source despite GLIDE's documented reference counting. -Reads use native `GET` for untracked entries and one atomic `MGET` for each tracked value-and-watermark pair. The adapters validate and decode the returned frame in the Node process. Tracked reads are deliberately routed to primaries so a lagging replica cannot hide an invalidation watermark. +Every semantic read sends native `GET` for an untracked entry or one atomic `MGET` for a tracked value-and-watermark pair, followed by Redis `TIME`. The two commands are enqueued together in an ordered pipeline or batch and routed to the same primary; the adapters validate and decode the frame in the Node process, then accept it only when `serverNowMs - createdAtMs < maxAgeMs`. Normal reads always pass the effective remote TTL as `maxAgeMs`, even when stale-on-error is off, while a recovery read passes the larger configured maximum. The payload never crosses the Redis-to-Lua boundary. Tracked `MGET` remains an authoritative atomic snapshot so a lagging replica cannot hide an invalidation watermark; its following `TIME` supplies the same server's age clock. -Writes are native too, so the payload never crosses the Redis-to-Lua boundary. An untracked write is one `SET` of a client-encoded frame. A tracked write pipelines two ordered commands on one connection: a `SET` of a version-0 placeholder frame carrying a fresh per-write nonce, then the small payload-free `WRITE_TRACKED_STAMP_SCRIPT`, which fences against the watermark, promotes exactly the placeholder carrying its nonce to a served frame with Redis server time, and maintains the watermark TTL. A placeholder is unreadable on both read paths until promoted, so an interleaved or lost stamp degrades to a miss that expires with the value TTL rather than partial state — including briefly blanking a previously readable key the write replaces. The nonce means the stamp can never revive a frame it does not own: if its paired `SET` was rejected, overwritten, or expired, the stamp reports the placeholder gone and the write fails with `DialCacheRedisPlaceholderLostError` as a `cache_write` error instead of publishing another write's leftovers. Losing a same-key write race is one such outcome, so `cache_write` carries a benign, self-healing floor that concentrates on hot tracked keys at TTL expiry — size write-error alerts for it. The `cache_write` metric itself stays one bounded counter; the error's class and name distinguish the lost-placeholder case in logs, and in the `catch` blocks of code that calls an adapter's `write()` directly — DialCache's own request paths absorb it fail-open rather than rethrowing to callers. Each occurrence also emits one warn through the configured logger (the default is `console`), so fleets expecting hot-key write contention should supply a logger that rate-limits or filters that class. A `SET` failure is the write's outcome even when the stamp settled. The pair is deliberately not a `MULTI`/`EXEC` transaction, which would consume caller-owned `WATCH` state. +Writes use the same placeholder pattern for both key types, so serving-critical creation time always comes from Redis without sending the payload through Lua. Each write pipelines two ordered commands on one connection: a native `SET` of a version-0 placeholder frame carrying the payload and a fresh nonce, then a tiny payload-free stamp script. `WRITE_UNTRACKED_STAMP_SCRIPT` verifies the nonce and promotes the untracked placeholder with Redis server time. `WRITE_TRACKED_STAMP_SCRIPT` additionally fences against the watermark and maintains its TTL. A placeholder is unreadable on both read paths until promoted, so an interleaved or lost stamp degrades to a miss that expires with the value TTL rather than partial state — including briefly blanking a previously readable key the write replaces. The nonce means neither stamp can revive a frame it does not own: if its paired `SET` was rejected, overwritten, or expired, the stamp reports the placeholder gone and the write fails with `DialCacheRedisPlaceholderLostError` as a `cache_write` error instead of publishing another write's leftovers. Losing a same-key write race is one such outcome, so `cache_write` carries a benign, self-healing floor that concentrates on hot keys at TTL expiry — size write-error alerts for it. The `cache_write` metric itself stays one bounded counter; the error's class and name distinguish the lost-placeholder case in logs, and in the `catch` blocks of code that calls an adapter's `write()` directly — DialCache's own request paths absorb it fail-open rather than rethrowing to callers. Each occurrence also emits one warn through the configured logger (the default is `console`), so fleets expecting hot-key write contention should supply a logger that rate-limits or filters that class. A `SET` failure is the write's outcome even when the stamp settled. Each pair is deliberately not a `MULTI`/`EXEC` transaction, which would consume caller-owned `WATCH` state. + +The steady-state network shape is therefore one pipeline/batch round trip containing two top-level commands per semantic read (`GET` or `MGET`, then `TIME`) and two per write (`SET`, then the matching stamp). Redis CPU remains linear and payload-size-sensitive only in the native value command: read payloads are decoded in Node and write payloads are never copied through Lua. The small stamp scripts inspect only the fixed-size header, with tracked stamping doing bounded watermark/TTL maintenance. A source rejection with stale recovery enabled adds one second read pair; fresh hits and successful source refreshes do not. Use the maintainer benchmarks below to measure the actual CPU, network, and latency deltas on the target Redis/Valkey version and payload distribution. Native commands retain Redis's wrong-type behavior. An untracked `GET` surfaces `WRONGTYPE`; tracked `MGET` represents a wrong-type member as a missing value. A wrong-type tracked value is therefore a clean miss and may be replaced with a valid DialCache frame after the fallback succeeds, while a wrong-type watermark prevents the tracked write from succeeding: its paired `SET` still lands, leaving only an unreadable placeholder until expiry or a later successful write. -Node-redis forces tracked cluster commands to the slot primary. GLIDE uses an explicit primary route in cluster mode; in standalone mode it sends `MGET` through a one-command non-atomic batch because direct read commands follow the client's replica-read preference. Standalone batches use the primary, and `MGET` itself provides the atomic snapshot without consuming caller-owned `WATCH` state. The GLIDE helper distinguishes those modes from the direct client's runtime identity and rejects ambiguous clients instead of silently choosing a route. +Node-redis forces every cluster read command and its `TIME` to the value key's slot primary. GLIDE uses an explicit primary route in cluster mode and sends the ordered read-plus-`TIME` pair through a primary batch in standalone mode because direct reads can follow the client's replica preference. `MGET` itself provides the tracked atomic snapshot without consuming caller-owned `WATCH` state. The GLIDE helper distinguishes those modes from the direct client's runtime identity and rejects ambiguous clients instead of silently choosing a route. + +For both stamp scripts and invalidation, node-redis computes each source's SHA, uses `EVALSHA`, and retries with `EVAL` after `NOSCRIPT`; its cluster client routes commands by the first key and performs that fallback on the selected shard. That retry extends the unreadable-placeholder gap by one round trip on a cold script cache. The GLIDE adapter batches each write's `SET` with the matching stamp `EVALSHA` — routing cluster write batches to the slot primary — and recovers from a flushed script cache by re-sending that source as `EVAL`, so the first write of either kind against a cold script cache pays one extra round trip. A late stamp stays paired to its own placeholder through the nonce; if the placeholder is gone by then, the write fails rather than publishing. Invalidation dispatches as `EVALSHA` by the script's source SHA1 on both adapters, and both retry a rejected dispatch once by re-sending the source as `EVAL`: the invalidation script is idempotent — its watermark only advances and its TTL only widens — so a duplicate execution after an ambiguous failure is harmless, and the retry heals a flushed script cache and an `EVALSHA`-rejecting proxy without depending on error wording. Reply-domain violations are deterministic and are not retried. When the retry also fails, the surfaced error is the retry's. On GLIDE the original rejection is attached as the retry error's `cause` unless it already carries one, and a failing invalidation is bounded by roughly two `requestTimeout` windows. On node-redis the retry rejection surfaces unmodified and the original is discarded — the library rejects every command flushed by a single disconnect with one shared error instance, so the adapter never mutates it — and no per-command deadline exists: `disableOfflineQueue`, `commandsQueueMaxLength`, and `reconnectStrategy` bound queueing and dispatch (the setup snippet above disables the offline queue, which makes a disconnected retry fail fast instead of waiting for reconnect), but a command already written to a hung connection has no reply deadline. Its own `NOSCRIPT` recovery may also add one round trip before the adapter's retry. A retry that heals is indistinguishable from a first-attempt success in DialCache's metrics and logs. The genuinely silent regime is invalidation-dispatch healing: watch server-side `INFO commandstats` for `cmdstat_eval` calls rising in step with invalidation volume while `cmdstat_evalsha` stays flat (a proxy rejecting `EVALSHA` before it reaches Redis) or accrues `rejected_calls` (an ACL denial). A sustained stamp fault is loud by contrast — the ACL paragraph below describes its amplitude. + +A tracked write rejected by an active future watermark uses `UNLINK` to remove the value key — the placeholder it just stored, along with any logically stale frame — without synchronously freeing it on Redis's command path. The mutation protocol therefore requires a server that implements `UNLINK` (Redis 4.0 or later, or a compatible Valkey release). Command-restricted Redis ACLs must allow native `GET`, `MGET`, `TIME`, and `SET`, plus `EVALSHA` for steady-state mutation dispatch and `EVAL` for source fallback after a flushed script cache (the adapters never require `SCRIPT LOAD`). Server versions differ on whether script-invoked commands are also checked against the invoking user, so grant what both stamp scripts invoke as well: `TIME`, `GETRANGE`, and `SETRANGE`; tracked paths additionally require `GET`, `SET`, `PTTL`, `PEXPIRE`, and `UNLINK`. Verify those grants before upgrading. A sustained stamp failure (denied command or a proxy rejecting script dispatch) still lands every paired `SET`, so each write replaces the last served value with an unreadable placeholder. Tracked paths also suppress process-local publication; untracked paths retain their existing fail-open local fill. Within one TTL horizon Redis traffic for that key therefore misses and reaches the source. DialCache's integration matrix covers Redis 6.2 and Valkey 8. + +#### Stale on source error + +Stale-on-error is an opt-in Redis policy with two ages: + +- `F = ttlSec.remote` is the logical fresh lifetime. Every ordinary Redis read enforces `F`, regardless of whether recovery is enabled. +- `M = staleOnErrorMaxAgeSec` is the absolute retained-value maximum. When configured, Redis stores the frame with physical TTL `M`, but ordinary reads still treat it as a miss at age `F`. + +After a definitive ordinary miss, DialCache calls the source of truth. If that call rejects — including with `FallbackTimeoutError` — DialCache performs one recovery read of the same key with maximum age `M`. A frame is returned only when its Redis-server age is strictly less than `M`; equality is expired. Any source rejection qualifies, rather than only selected error classes. If recovery serves a value, the original rejection is suppressed for that caller. If recovery misses, times out, fails, or cannot deserialize/decompress the frame, DialCache rethrows the original source rejection unchanged. + +```ts +import { CacheLayer, DialCacheKeyConfig } from "dialcache"; + +const getUser = dialcache.cached((userId: string) => db.fetchUser(userId), { + keyType: "user_id", + useCase: "GetUserWithStaleRecovery", + cacheKey: (userId) => userId, + defaultConfig: new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 60 }, // F: normal reads serve for 60 seconds + staleOnErrorMaxAgeSec: 300, // M: recovery may serve through 5 minutes + }), +}); +``` + +Omission keeps recovery off and inherits a configured default when used in a sparse runtime overlay. An explicit `0` disables it. A positive `M` requires an enabled remote TTL and must satisfy `0 < F < M <= 31_536_000`; invalid static defaults throw, while an invalid runtime overlay records `config_resolution`, disables only stale recovery for that invocation, and leaves valid ordinary Redis reads active. The effective config snapshot is fixed for the invocation: both reads use its `F`, `M`, and remote-read deadline even if the provider changes while the source call is in flight. + +The recovery read receives a new, independent instance of the same effective `remoteReadTimeoutMs` budget; time spent in the initial read and source call does not reduce it. An initial Redis error or timeout never triggers recovery because DialCache has not established a definitive logical miss. A frame rejected during the initial deserialize/decompression path is also not reread as stale. Recovery does not write Redis, populate the process-local cache, schedule shadow validation, or emit the shadow value-age observation. When request-local caching is enabled, the recovered value is memoized only in that outer `enable()` scope. This prevents an outage response from becoming a new shared cache value. + +Default coalescing applies to the whole sequence, so same-key followers share one ordinary read, one source rejection, and one recovery read. With `coalesce: false`, each concurrent caller instead performs its own ordinary read, source call, independent fallback deadline, and possible recovery read; request-local memoization can still serve later sequential calls after a recovered value settles. -For the stamp and invalidation scripts, node-redis computes each script's SHA, uses `EVALSHA`, and retries with `EVAL` after `NOSCRIPT`; its cluster client routes commands by their first key and performs that fallback on the selected shard. That retry likewise extends the unreadable-placeholder gap of a tracked write by one round trip on a cold script cache. The GLIDE adapter batches the tracked write's `SET` with an `EVALSHA` of the stamp script — routing cluster write batches to the slot primary — and recovers from a flushed script cache by re-sending the stamp as `EVAL` with its source, which the server caches under the same SHA1, so the first tracked write against a cold script cache pays one extra round trip. A late stamp stays paired to its own placeholder through the nonce; if the placeholder is gone by then, the write fails rather than publishing. Invalidation dispatches as `EVALSHA` by the script's source SHA1 on both adapters, and both retry a rejected dispatch once by re-sending the source as `EVAL`: the invalidation script is idempotent — its watermark only advances and its TTL only widens — so a duplicate execution after an ambiguous failure is harmless, and the retry heals a flushed script cache and an `EVALSHA`-rejecting proxy without depending on error wording. Reply-domain violations are deterministic and are not retried. When the retry also fails, the surfaced error is the retry's. On GLIDE the original rejection is attached as the retry error's `cause` unless it already carries one, and a failing invalidation is bounded by roughly two `requestTimeout` windows. On node-redis the retry rejection surfaces unmodified and the original is discarded — the library rejects every command flushed by a single disconnect with one shared error instance, so the adapter never mutates it — and no per-command deadline exists: `disableOfflineQueue`, `commandsQueueMaxLength`, and `reconnectStrategy` bound queueing and dispatch (the setup snippet above disables the offline queue, which makes a disconnected retry fail fast instead of waiting for reconnect), but a command already written to a hung connection has no reply deadline. Its own `NOSCRIPT` recovery may also add one round trip before the adapter's retry. A retry that heals is indistinguishable from a first-attempt success in DialCache's metrics and logs. The genuinely silent regime is invalidation-dispatch healing: watch server-side `INFO commandstats` for `cmdstat_eval` calls rising in step with invalidation volume while `cmdstat_evalsha` stays flat (a proxy rejecting `EVALSHA` before it reaches Redis) or accrues `rejected_calls` (an ACL denial). A sustained stamp fault is loud by contrast — the ACL paragraph below describes its amplitude. +Each attempted recovery emits exactly one optional `staleRecovery` outcome: `served`, `miss`, `read_error`, `read_timeout`, or `deserialization_error`. Existing request, miss, read, serialization, compression, fallback-duration, and fallback-error telemetry remains unchanged, so a served recovery is still visibly paired with the source failure that caused it. Recovery never adds a raw exception or key to metric labels. -A tracked write rejected by an active future watermark uses `UNLINK` to remove the value key — the placeholder it just stored, along with any logically stale frame — without synchronously freeing it on Redis's command path. The mutation protocol therefore requires a server that implements `UNLINK` (Redis 4.0 or later, or a compatible Valkey release). Command-restricted Redis ACLs must allow the client to issue the native `GET`, `MGET`, and `SET` commands — `SET` newly carries every write, where the previous protocol wrote only through scripts — plus `EVALSHA` (the steady-state dispatch for both mutation scripts) and `EVAL` (both adapters recover a flushed script cache by re-sending script sources, never via `SCRIPT LOAD`). Server versions differ on whether script-invoked commands are also checked against the invoking user, so grant what the mutation scripts invoke as well: `TIME`, `GET`, `SET`, and `PTTL` (both scripts), plus the stamp's `PEXPIRE`, `UNLINK`, `GETRANGE`, and `SETRANGE`. Verify those grants before upgrading, because the failure amplitude of a persistent stamp fault changed. A sustained stamp failure (denied command, a proxy rejecting `EVALSHA`) still lands every paired `SET`, so each tracked write replaces the last served value with an unreadable placeholder while also suppressing process-local publication — within one TTL horizon the source absorbs full traffic, where the previous protocol degraded to serving stale values until expiry. DialCache's integration matrix covers Redis 6.2 and Valkey 8. +This feature keeps the existing frame-v1 Redis keys; it does not create a second stale key. Roll it out readers first: every reader must enforce logical `F` with Redis server time before any writer begins retaining values to `M`. An older reader relies on physical expiry and could otherwise serve a retained frame normally between `F` and `M`. Once any write uses physical `M`, treat that deployment as a downgrade barrier until all such keys have expired or been explicitly removed; disabling recovery on new readers is safe because they still enforce `F`, but reintroducing an older reader is not. The same readers-first rule applies to custom adapters, which must declare `enforcesMaxAge: true` and implement the required `RedisReadRequest.maxAgeMs` contract. #### Remote read deadlines and async liveness @@ -420,7 +459,7 @@ DialCache bounds every active Redis read. The effective timeout is resolved per When the deadline expires, DialCache aborts the optional `RedisReadContext.signal`, records one `cache_read_timeout` error, logs a `RedisReadTimeoutError`, and starts the source fallback. Late read fulfillment or rejection is consumed and ignored. A read failure or timeout never triggers a post-fallback Redis write; an untracked active process-local miss may retain the source value, while a tracked key suppresses local publication because the failed read did not establish watermark safety. -Same-key followers share the leader's remaining remote-read budget. The timer covers only the semantic Redis read, not config resolution, serializer load, fallback work, Redis writes, or invalidation. `fallbackTimeoutMs` starts separately when the source fallback begins. +Same-key followers share the leader's remaining remote-read budget. The timer covers only one semantic Redis read, not config resolution, serializer load, fallback work, Redis writes, or invalidation. `fallbackTimeoutMs` starts separately when the source fallback begins. When stale-on-error is enabled and that fallback rejects, its recovery read starts a new full remote-read budget rather than inheriting the initial read's elapsed time. The bundled node-redis adapter passes the signal through per-command options, which can remove queued work where supported. Aborting after dispatch does not unsend a command or prove that Redis stopped executing it. GLIDE's current adapter commands have no per-invocation signal, so a read may continue after DialCache has fallen back. Keep client-native connection, retry, queue, and response budgets in place; they bound underlying resource lifetime while DialCache's deadline bounds caller wait time. @@ -428,20 +467,19 @@ Writes, invalidations, async `cacheConfigProvider` calls, and custom serializer #### Serialization -The core Redis boundary is the client-agnostic `DialCacheRedisClient` interface. Writes accept serialized values as `string | Buffer`; reads return a `DecodedRedisFrame` — the decoded `string | Buffer` payload plus the frame header's `createdAtMs` (Redis server time for tracked frames, the writer's informational client clock for untracked ones) — and the interface does not expose client commands or wire encodings. The shared `encodeRedisFrame`, `encodeTrackedRedisPlaceholder`, `decodeRedisFrame`, and `decodeTrackedRedisFrame` helpers, the `resolveTrackedRedisWriteReply`, `validateRedisSetReply`, and `validateRedisScriptInvalidationReply` reply helpers, the `ceilSupportedCacheTtlMs` TTL guard, and the tracked stamp and invalidation Lua sources are available from `dialcache/redis-protocol`, so custom adapters can reuse the bundled adapters' exact framing, miss, watermark-fencing, TTL-domain, and reply rules. A custom tracked write must pass the stamp script `KEYS = [valueKey, watermarkKey]` and `ARGV = [cacheTtlMs, nonce]`, running `cacheTtlMs` through `ceilSupportedCacheTtlMs` and using the result for both the paired `SET`'s `PX` and `ARGV[1]` (the stamp script re-validates the same domain server-side as defense in depth), with the nonce from the same `encodeTrackedRedisPlaceholder` call; `resolveTrackedRedisWriteReply` maps the reply, failing the write with the root-exported `DialCacheRedisPlaceholderLostError` when the stamp replies `2`. Custom adapters can throw the root-exported `DialCacheRedisPayloadError`, `DialCacheRedisPayloadEncodingError`, `DialCacheRedisProtocolError`, and `DialCacheRedisPlaceholderLostError` classes to distinguish malformed replies, unsupported encodings, reply-domain violations, and lost placeholders in logs. DialCache records bounded `cache_read`, `cache_write`, or `invalidation` metrics by failure site. +The core Redis boundary is the client-agnostic `DialCacheRedisClient` interface. Writes accept serialized values as `string | Buffer`; reads require `maxAgeMs` and return a `DecodedRedisFrame` — the decoded payload plus its Redis-server `createdAtMs` — only when the frame is logically young enough. The interface does not expose client commands or wire encodings. The shared `encodeRedisFrame`, `encodeTrackedRedisPlaceholder`, `decodeRedisFrame`, `decodeTrackedRedisFrame`, `decodeRedisServerTime`, and `isRedisFrameWithinMaxAge` helpers; both stamp reply resolvers; the mutation reply validators; the `ceilSupportedCacheTtlMs` TTL guard; and both stamp and invalidation Lua sources are available from `dialcache/redis-protocol`, so custom adapters can reuse the bundled adapters' exact framing, logical-age, miss, watermark-fencing, TTL-domain, and reply rules. A custom write must use one `encodeTrackedRedisPlaceholder` result for its native `SET` and matching stamp. Untracked writes pass `KEYS = [valueKey]` and `ARGV = [nonce]` to `WRITE_UNTRACKED_STAMP_SCRIPT`; tracked writes pass `KEYS = [valueKey, watermarkKey]` and `ARGV = [cacheTtlMs, nonce]` to `WRITE_TRACKED_STAMP_SCRIPT`. Run `cacheTtlMs` through `ceilSupportedCacheTtlMs` and use it for the paired `SET`'s `PX` and, for tracked writes, `ARGV[1]`. The corresponding resolver maps the reply and fails with the root-exported `DialCacheRedisPlaceholderLostError` when a stamp replies `2`. Custom adapters can throw the root-exported `DialCacheRedisPayloadError`, `DialCacheRedisPayloadEncodingError`, `DialCacheRedisProtocolError`, and `DialCacheRedisPlaceholderLostError` classes to distinguish malformed replies, unsupported encodings, reply-domain violations, and lost placeholders in logs. DialCache records bounded `cache_read`, `cache_write`, or `invalidation` metrics by failure site. Redis values use a compact binary frame: ```text -byte 1 format version: 1 = servable, 0 = unreadable tracked placeholder -bytes 2-9 uint64 big-endian: Redis server time for a promoted tracked frame, - informational client time for an untracked frame, or the random - per-write nonce while a tracked placeholder awaits its stamp +byte 1 format version: 1 = servable, 0 = unreadable write placeholder +bytes 2-9 uint64 big-endian: Redis server time for a promoted frame, or the + random per-write nonce while a placeholder awaits its stamp byte 10 payload encoding (0 = UTF-8, 1 = raw binary) bytes 11... serialized payload (optionally zstd-compressed; see Compression) ``` -Adapters build frames in the Node process. Untracked frames come from `encodeRedisFrame` and carry an informational client-clock timestamp that untracked reads never consult. Tracked frames start as `encodeTrackedRedisPlaceholder` output — version byte `0`, with a random per-write nonce in the timestamp bytes — which no read path serves; the stamp script verifies the nonce and promotes the frame to version `1` with Redis server time using Lua's `struct` library, and adapters decode it with Node's buffer primitives. Redis TTL is authoritative, so expiry metadata is not duplicated in the frame. `payload` is produced by the operation's serializer, or by `JsonSerializer` by default. Custom serializers can return either `string` or `Buffer`. Payloads stored raw keep their exact serialized bytes: strings are stored as UTF-8 and Buffers byte-for-byte without base64 expansion, except that binary output beginning with a [compression envelope byte](#compression) (`0x00`–`0x02`) gains a one-byte escape prefix on the wire. Payloads at or above the compression threshold may instead be stored as a zstd envelope (see [Compression](#compression)), so wire bytes for large values are not the serializer's output. Adapters return the frame payload as-is; the envelope — including restoring a compressed string's representation before `serializer.load` — is interpreted by the core above them. +Adapters build placeholders in the Node process with `encodeTrackedRedisPlaceholder`: version byte `0`, a random nonce in the timestamp bytes, and the serialized payload. No read path serves version `0`. The matching stamp verifies the nonce and promotes the frame to version `1` with Redis server time using Lua's `struct` library; adapters decode it with Node's buffer primitives and compare that timestamp with a following Redis `TIME`. Redis physical TTL is authoritative for retention, while the frame timestamp is authoritative for logical `F`/`M` age. `payload` is produced by the operation's serializer, or by `JsonSerializer` by default. Custom serializers can return either `string` or `Buffer`. Payloads stored raw keep their exact serialized bytes: strings are stored as UTF-8 and Buffers byte-for-byte without base64 expansion, except that binary output beginning with a [compression envelope byte](#compression) (`0x00`–`0x02`) gains a one-byte escape prefix on the wire. Payloads at or above the compression threshold may instead be stored as a zstd envelope (see [Compression](#compression)), so wire bytes for large values are not the serializer's output. Adapters return the frame payload as-is; the envelope — including restoring a compressed string's representation before `serializer.load` — is interpreted by the core above them. DialCache uses native `JSON.stringify` and `JSON.parse` by default. There is no runtime validation pass, so the default adds no traversal beyond JSON serialization itself. A top-level `undefined` result is supported with an internal sentinel. @@ -562,7 +600,7 @@ The detached job uses this bounded algorithm: Here a clean miss means the semantic Redis read returned `null`; it does not include a non-null payload that later fails deserialization. A caller fallback rejection or timeout never becomes accepted `S` and never starts the fill. -Both detached Redis reads use the effective `remoteReadTimeoutMs` and the key's normal protocol. Every clean-miss fill uses the same serializer, TTL, and timestamp semantics as an ordinary fill — server time for tracked fills, informational client time for untracked ones. A tracked fill blanks the key with its placeholder before publishing, so a lost or raced stamp can leave a previously readable value unreadable until the value TTL, and `fill_error` includes that benign lost-placeholder outcome. Tracked `C0` and `C1` reads remain watermark-aware and are routed to primaries by the bundled adapters, while tracked fills also retain the ordinary invalidation watermark. Untracked reads use the ordinary one-key read route, which has no shadow-specific primary guarantee, and untracked fills use the ordinary TTL write without a watermark. Strings compare exactly, Buffers compare by bytes, and string/Buffer pairs compare by their UTF-8 bytes. DialCache does not deserialize `C1`, compare it with `S`, or chase another version. +Both detached Redis reads use the effective `remoteReadTimeoutMs`, enforce the key's logical remote TTL, and follow the normal native-read-plus-`TIME` protocol. Every clean-miss fill uses the same serializer, physical retention, and Redis-server timestamp semantics as an ordinary fill; when stale-on-error is active, that retention is `M` while reads still enforce `F`. Any fill blanks the key with its placeholder before publishing, so a lost or raced stamp can leave a previously readable value unreadable until the value TTL, and `fill_error` includes that benign lost-placeholder outcome. Tracked `C0` and `C1` reads remain watermark-aware, and tracked fills retain the ordinary invalidation watermark; untracked fills use the ordinary stamp without a watermark. Both bundled adapters route all reads and their following `TIME` to the same primary. Strings compare exactly, Buffers compare by bytes, and string/Buffer pairs compare by their UTF-8 bytes. DialCache does not deserialize `C1`, compare it with `S`, or chase another version. The detached scheduler, Redis-read deadline timers, and overall shadow deadline timer are unreferenced, so they do not keep an otherwise idle process alive. Detachment is asynchronous work on the Node event loop, not a worker thread: synchronous source, serializer, or comparator work can still occupy the event loop after the request path has been released. @@ -586,7 +624,7 @@ The effective serializer's `load` method therefore runs a second time for a samp Shadow metrics use the bounded outcomes `match`, `mismatch`, `superseded`, `filled`, `fill_blocked`, `fill_error`, `redis_error`, `source_error`, `deserialization_error`, `comparison_error`, `confirmation_error`, `timeout`, and `dropped`. `redis_error` applies to the initial shadow-only `C0` read; `confirmation_error` applies to `C1`; `fill_blocked` applies only when a tracked watermark rejects the write. A clean `C0` miss is an ordinary `miss{layer="remote_shadow"}` and terminates with a fill, source, or timeout outcome rather than a second shadow outcome for the miss itself. Labels never contain cache ids, values, payloads, Redis keys, or raw exception text. -A `match` or `mismatch` verdict additionally records the validated value's age through the optional `observeShadowValueAge` adapter hook: the observing process's epoch clock minus the validated frame's `createdAtMs` (the served `C0` frame, or the detached `C0` frame on a ramped-down path), in seconds, clamped at zero. The age is captured at verdict time, so a mismatch age lands one confirmation read after the comparison itself. A confirmed `mismatch` age therefore measures how long the stale value had been readable when validation caught it. Tracked frames are stamped with Redis server time and untracked frames with the writer's client clock, so the age mixes clocks and is coarse operational evidence rather than a precise measurement. Outcomes that deliver no verdict on a retained value — including `superseded`, `filled`, and every error or timeout outcome — record no age. The hook does not gate shadow eligibility; only `shadowValidation` does. +A `match` or `mismatch` verdict additionally records the validated value's age through the optional `observeShadowValueAge` adapter hook: the observing process's epoch clock minus the validated frame's Redis-server `createdAtMs` (the served `C0` frame, or the detached `C0` frame on a ramped-down path), in seconds, clamped at zero. The age is captured at verdict time, so a mismatch age lands one confirmation read after the comparison itself. A confirmed `mismatch` age therefore measures how long the stale value had been readable when validation caught it. Both tracked and untracked frames are Redis-server-stamped; the observation still mixes that server clock with the observing process clock, so it remains coarse operational evidence rather than a precise measurement. Outcomes that deliver no verdict on a retained value — including `superseded`, `filled`, and every error or timeout outcome — record no age. The hook does not gate shadow eligibility; only `shadowValidation` does. Confirmed-mismatch logging is separately opt-in through `shadow.logMismatches`; it never replaces the required `shadowValidation` metric or emits for a mismatch candidate that becomes `superseded`. The single warning contains `cacheNamespace`, `useCase`, `keyType`, `outcome: "mismatch"`, `cacheKey`, `cachedValueJson`, and `sourceValueJson`. `cacheKey` is the logical DialCache URN capped at 2 KiB, not the physical Redis storage key. DialCache independently applies native `JSON.stringify` to the deserialized cached snapshot and raw source value supplied to the comparator, then caps each resulting string at 8 KiB. A byte-clipped field ends in `...[truncated]`, counted inside its cap. If native JSON throws or returns `undefined`, the corresponding JSON field is `null`; the other side is still attempted. DialCache does not compute a textual diff or call the configured serializer again for logging. @@ -596,7 +634,7 @@ The byte caps apply before logger framing or escaping, so they do not guarantee Detached Redis reads, serializer loads/dumps, payload sizes, and read/write errors use the existing layer label with `layer="remote_shadow"`. This distinguishes non-serving Redis cost from caller-path `layer="remote"` telemetry without adding a metric or label. The established `observeGet{layer="remote"}` boundary includes caller-path deserialization, while `observeGet{layer="remote_shadow"}` ends when the deadline-bounded Redis read result settles; detached serializer work and any later raw-client settlement are outside that timer. The request-path read that supplied a served `C0` keeps `layer="remote"`, and a ramped-down caller keeps `disabled{layer="remote", reason="ramped_down"}`. No `disabled{layer="remote_shadow"}` event is emitted for ineligible or dropped work; `dropped` remains the terminal shadow outcome. Confirmation reads use the same `remote_shadow` value, with `superseded` or `confirmation_error` describing their role. -The command amplification is bounded: a selected served hit adds one SoT read and adds `C1` only for a semantic mismatch candidate; a selected ramped-down hit adds detached `C0`, reuses the caller's existing SoT read, and likewise adds `C1` only for a candidate; a selected ramped-down miss adds detached `C0` and at most one fill in the key's existing mode. `superseded` means only that the original observation could not be confirmed. `mismatch` means the exact `C0` payload survived another Redis read after the SoT disagreement; it is not a cross-system atomic snapshot or a guarantee that the mismatch persists. For an untracked key it is also not proof of primary freshness or invalidation safety. +The command amplification is bounded: a selected served hit adds one SoT read and adds `C1` only for a semantic mismatch candidate; a selected ramped-down hit adds detached `C0`, reuses the caller's existing SoT read, and likewise adds `C1` only for a candidate; a selected ramped-down miss adds detached `C0` and at most one fill in the key's existing mode. `superseded` means only that the original observation could not be confirmed. `mismatch` means the exact `C0` payload survived another Redis read after the SoT disagreement; it is not a cross-system atomic snapshot or a guarantee that the mismatch persists. For an untracked key it is also not proof of invalidation safety because no watermark participates. The initial `C0` read and later fill are not atomic. The fill is a normal overwrite, not a compare-and-set or write-if-still-missing operation: another writer can populate Redis after the clean miss and be overwritten by the shadow fill. Tracked invalidation watermarks still fence tracked writes using Redis time, so size `futureBufferMs` to cover the complete SoT, serialization, client queue, network, and write interval when stale-publication protection matters. An untracked shadow fill has no such fence and retains the ordinary TTL-based last-writer-wins contract; because it is detached, an older accepted source value may be written after a concurrent source mutation and remain until expiry. Shadow mode never repairs a non-null `C0`, refreshes its TTL, invalidates, evicts local state, or changes the value returned to the caller. @@ -796,6 +834,7 @@ The Prometheus adapter emits: | `dialcache_coalesced_counter` | Counter | `cache_namespace`, `use_case`, `key_type`, `scope` | Coalesced requests split by `request_local` or `process` scope | | `dialcache_shadow_validation_counter` | Counter | `cache_namespace`, `use_case`, `key_type`, `outcome` | Sampled Redis shadow-job outcomes | | `dialcache_shadow_value_age_histogram` | Histogram | `cache_namespace`, `use_case`, `key_type`, `outcome` | Age in seconds of the validated Redis value at shadow verdict time, recorded for `match` and `mismatch` | +| `dialcache_stale_recovery_counter` | Counter | `cache_namespace`, `use_case`, `key_type`, `outcome` | Stale-on-error recovery attempts by bounded terminal outcome | | `dialcache_compression_counter` | Counter | `cache_namespace`, `use_case`, `key_type`, `layer`, `outcome` | Payload compression outcomes: writes record `compressed`, `below_threshold`, `not_smaller`, or `write_over_limit`; reads record `decompressed`, `fallback_raw`, or `read_over_limit` | | `dialcache_get_timer` | Histogram | `cache_namespace`, `use_case`, `key_type`, `layer` | Cache get latency in seconds | | `dialcache_fallback_timer` | Histogram | `cache_namespace`, `use_case`, `key_type`, `layer` | Elapsed time until the underlying function settles or timeout rejection is delivered | @@ -807,7 +846,7 @@ The Prometheus adapter emits: `policy_disabled` means that a process-local or Redis layer has no effective TTL after runtime overlays are applied. It is an intentional policy outcome, including the default when `defaultConfig` is omitted, rather than a configuration-loading failure. -Every metric carries `cache_namespace`, including disabled-context, key-construction, coalescing, shadow-validation, and invalidation paths that do not have a constructed key. Its value is `DialCacheConfig.namespace`, defaulting to `urn`. The `layer` label is `request_local`, `local` (process-local), `remote` (caller-serving Redis), or `remote_shadow` (detached, non-serving Redis work); `noop` means no cache layer was reached. Detached reads, serializer work, payload sizes, and Redis read/write errors use `remote_shadow`, while the dedicated bounded shadow `outcome` records the terminal job result. The bounded `scope` label on `dialcache_coalesced_counter` distinguishes request-local from instance-scoped single-flight work. `scope="process"` coordinates calls only within one `DialCache` instance; separate instances in the same process do not share in-flight state. +Every metric carries `cache_namespace`, including disabled-context, key-construction, coalescing, shadow-validation, stale-recovery, and invalidation paths that do not have a constructed key. Its value is `DialCacheConfig.namespace`, defaulting to `urn`. The `layer` label is `request_local`, `local` (process-local), `remote` (caller-serving Redis), or `remote_shadow` (detached, non-serving Redis work); `noop` means no cache layer was reached. Detached reads, serializer work, payload sizes, and Redis read/write errors use `remote_shadow`, while the dedicated bounded shadow `outcome` records the terminal job result. Stale recovery is always caller-serving Redis work, so its dedicated counter needs no `layer` label. The bounded `scope` label on `dialcache_coalesced_counter` distinguishes request-local from instance-scoped single-flight work. `scope="process"` coordinates calls only within one `DialCache` instance; separate instances in the same process do not share in-flight state. ### Datadog @@ -859,6 +898,7 @@ The Datadog adapter emits exact increments of `1` for counters and preserves sec | `dialcache.coalesced.count` | Count | `cache_namespace`, `use_case`, `key_type`, `scope` | Coalesced requests by sharing scope | | `dialcache.shadow.count` | Count | `cache_namespace`, `use_case`, `key_type`, `outcome` | Sampled Redis shadow-job outcomes | | `dialcache.shadow.value_age` | Distribution or histogram | `cache_namespace`, `use_case`, `key_type`, `outcome` | Age in seconds of the validated Redis value at shadow verdict time, recorded for `match` and `mismatch` | +| `dialcache.stale_recovery.count` | Count | `cache_namespace`, `use_case`, `key_type`, `outcome` | Stale-on-error recovery attempts by bounded terminal outcome | | `dialcache.compression.count` | Count | `cache_namespace`, `use_case`, `key_type`, `layer`, `outcome` | Payload compression outcomes: writes record `compressed`, `below_threshold`, `not_smaller`, or `write_over_limit`; reads record `decompressed`, `fallback_raw`, or `read_over_limit` | | `dialcache.get.duration` | Distribution or histogram | `cache_namespace`, `use_case`, `key_type`, `layer` | Cache get latency in seconds | | `dialcache.fallback.duration` | Distribution or histogram | `cache_namespace`, `use_case`, `key_type`, `layer` | Elapsed time until the underlying function settles or timeout rejection is delivered | @@ -892,7 +932,7 @@ These values are defined by the backend-neutral core and are identical for every ### Custom adapters -For other telemetry backends, implement `DialCacheMetricsAdapter` and pass the adapter through `new DialCache({ metrics })`. Every backend-neutral label object exposes the logical namespace as camel-case `cacheNamespace`; adapters should map it to their backend's `cache_namespace` label/tag. This field is present even when no key or cache layer was reached. Implement the optional `shadowValidation` method to enable shadow work as well as record its outcomes; omitting it leaves all shadow work disabled even when `shadow.ramp` is nonzero or mismatch logging is enabled. The optional `observeShadowValueAge` method records the validated value's age in seconds for `match` and `mismatch` outcomes; omitting it skips only that observation without affecting shadow eligibility. Every metrics callback is fire-and-forget: DialCache isolates synchronous throws and consumes rejections from returned promises or thenables, but never awaits or drains observer work. Omit `metrics` to disable metrics. +For other telemetry backends, implement `DialCacheMetricsAdapter` and pass the adapter through `new DialCache({ metrics })`. Every backend-neutral label object exposes the logical namespace as camel-case `cacheNamespace`; adapters should map it to their backend's `cache_namespace` label/tag. This field is present even when no key or cache layer was reached. Implement the optional `shadowValidation` method to enable shadow work as well as record its outcomes; omitting it leaves all shadow work disabled even when `shadow.ramp` is nonzero or mismatch logging is enabled. The optional `observeShadowValueAge` method records the validated value's age in seconds for `match` and `mismatch` outcomes; omitting it skips only that observation without affecting shadow eligibility. The optional `staleRecovery` method records one bounded terminal outcome for each attempted recovery; omitting it disables only that observation, not recovery itself. Every metrics callback is fire-and-forget: DialCache isolates synchronous throws and consumes rejections from returned promises or thenables, but never awaits or drains observer work. Omit `metrics` to disable metrics. ## Maintainers @@ -916,6 +956,16 @@ pnpm benchmark:redis-write The command builds `dist`, then runs sequential tracked and untracked writes at 100 B, 10 KiB, 100 KiB, and 1 MiB payloads, reporting server-side command cost per write from `INFO commandstats` (the `EVALSHA` entry envelopes the stamp script's internal calls) and client-side p50/p95 latency. Like the cache-path benchmark it is a maintainer tool, is not part of the published package, and asserts no timing thresholds — absolute numbers depend on the machine, engine, and load, so compare runs only within one environment. Scale iteration counts with `DIALCACHE_BENCH_WRITE_SCALE`. +### Stale-on-error benchmark + +With Redis reachable at `REDIS_URL`, exercise the current native-read-plus-`TIME` design and a representative compressible payload: + +```bash +pnpm benchmark:stale-on-error +``` + +The benchmark warms isolated keys, verifies that physical retention uses `M`, and reports fresh end-to-end hits, logical stale misses at `F`, end-to-end stale recovery, and same-key coalesced recovery. It snapshots `INFO commandstats` and network byte counters around each scenario without resetting shared server statistics, and reports command, server-CPU, network, and client-throughput signals per operation. Semantic assertions cover compression, exact source/recovery counts, and returned values; elapsed time remains informational with no pass/fail threshold. Override work sizes with `DIALCACHE_BENCH_STALE_ITERATIONS`, `DIALCACHE_BENCH_STALE_FANOUT`, and `DIALCACHE_BENCH_STALE_PAYLOAD_BYTES`. + ### Releasing Publishing starts by manually running the `Release` workflow from current `main`. After the package checks pass, Semantic Release selects the next version from Conventional Commits since the highest stable `vX.Y.Z` tag. While the package is pre-1.0, breaking changes bump minor — their `BREAKING CHANGE:` footers still drive full release notes without forcing 1.0.0 — `feat` bumps minor, and every other normal PR-title type (`fix`, `perf`, `docs`, `style`, `refactor`, `test`, `build`, `chore`, `ci`, and `revert`) bumps patch. The highest required bump wins. Major bumps return when 1.0.0 is cut; `release.config.mjs` implements this table and must change together with this section. diff --git a/package.json b/package.json index 9a5fc01..2cd6a44 100644 --- a/package.json +++ b/package.json @@ -78,6 +78,7 @@ "scripts": { "benchmark:request-local": "pnpm build && node scripts/benchmark-request-local.mjs", "benchmark:redis-write": "pnpm build && node scripts/benchmark-redis-write.mjs", + "benchmark:stale-on-error": "pnpm build && node scripts/benchmark-stale-on-error.mjs", "build": "tsup src/index.ts src/datadog.ts src/node-redis.ts src/prometheus.ts src/redis-protocol.ts src/valkey-glide.ts --format esm,cjs --dts --clean", "check": "pnpm typecheck && pnpm test && pnpm build && pnpm test:package", "typecheck": "tsc --noEmit", diff --git a/scripts/benchmark-request-local.mjs b/scripts/benchmark-request-local.mjs index 8a0c3b9..7d1e841 100644 --- a/scripts/benchmark-request-local.mjs +++ b/scripts/benchmark-request-local.mjs @@ -244,11 +244,12 @@ async function benchmarkRedisReadDeadlineCoalescing(fanout) { const originalSetTimeout = globalThis.setTimeout; const originalClearTimeout = globalThis.clearTimeout; const redisClient = { - async read() { + enforcesMaxAge: true, + async read({ maxAgeMs }) { redisReadCalls += 1; started.resolve(); await gate.promise; - return JSON.stringify("shared"); + return freshFrame(JSON.stringify("shared"), maxAgeMs); }, async write() { return true; @@ -319,10 +320,11 @@ async function benchmarkSequentialTrackedRedisHits(iterations, { scenario, useCa let redisWriteCalls = 0; let redisInvalidationCalls = 0; const redisClient = { - async read({ watermarkKey }) { + enforcesMaxAge: true, + async read({ watermarkKey, maxAgeMs }) { assert.equal(typeof watermarkKey, "string", "the benchmark must exercise tracked Redis reads"); redisReadCalls += 1; - return JSON.stringify("shared"); + return freshFrame(JSON.stringify("shared"), maxAgeMs); }, async write() { redisWriteCalls += 1; @@ -392,10 +394,11 @@ async function benchmarkDarkShadowDetachment() { const cachedValue = { source: "redis" }; const sourceValue = { source: "truth" }; const redisClient = { - async read({ watermarkKey }) { + enforcesMaxAge: true, + async read({ watermarkKey, maxAgeMs }) { assert.equal(typeof watermarkKey, "string", "dark shadow reads must remain tracked"); redisReadCalls += 1; - return await readGate.promise; + return freshFrame(await readGate.promise, maxAgeMs); }, async write() { redisWriteCalls += 1; @@ -476,8 +479,10 @@ async function benchmarkDarkShadowFillDetachment() { let redisWriteCalls = 0; const sourceValue = { source: "truth" }; const redisClient = { - async read({ watermarkKey }) { + enforcesMaxAge: true, + async read({ watermarkKey, maxAgeMs }) { assert.equal(typeof watermarkKey, "string", "dark shadow reads must remain tracked"); + assert(Number.isSafeInteger(maxAgeMs) && maxAgeMs > 0, "reads must request a positive max age"); redisReadCalls += 1; return null; }, @@ -558,6 +563,12 @@ function deferred() { return { promise, resolve }; } +function freshFrame(payload, maxAgeMs) { + assert(Number.isSafeInteger(maxAgeMs) && maxAgeMs > 0, "reads must request a positive max age"); + const createdAtMs = Date.now(); + return Date.now() - createdAtMs < maxAgeMs ? { payload, createdAtMs } : null; +} + function readPositiveInteger(name, fallback) { const raw = process.env[name]; if (raw === undefined) { diff --git a/scripts/benchmark-stale-on-error.mjs b/scripts/benchmark-stale-on-error.mjs new file mode 100644 index 0000000..46985c5 --- /dev/null +++ b/scripts/benchmark-stale-on-error.mjs @@ -0,0 +1,378 @@ +// Maintainer benchmark for the stale-on-error Redis path. It uses the public +// node-redis adapter against a live Redis, keeps payload I/O native, and +// reports client latency plus INFO commandstats/network deltas. Results have +// no pass/fail timing threshold; compare runs only on the same environment. +// +// Requires Redis, e.g.: docker run --rm -p 6379:6379 redis:6.2 +// Usage: pnpm benchmark:stale-on-error (REDIS_URL to override) +// Optional sizing: DIALCACHE_BENCH_STALE_ITERATIONS, +// DIALCACHE_BENCH_STALE_FANOUT, and DIALCACHE_BENCH_STALE_PAYLOAD_BYTES. +import assert from "node:assert/strict"; +import { performance } from "node:perf_hooks"; + +import { createClient } from "redis"; + +import { + CacheLayer, + DialCache, + DialCacheKey, + DialCacheKeyConfig, +} from "../dist/index.js"; +import { + createNodeRedisDialCacheClient, + dialcacheRedisScripts, +} from "../dist/node-redis.js"; + +const redisUrl = process.env.REDIS_URL ?? "redis://127.0.0.1:6379"; +const iterations = readPositiveInteger("DIALCACHE_BENCH_STALE_ITERATIONS", 200); +const fanout = readPositiveInteger("DIALCACHE_BENCH_STALE_FANOUT", 500); +const payloadBytes = readPositiveInteger("DIALCACHE_BENCH_STALE_PAYLOAD_BYTES", 64 * 1024); +const freshAgeSec = 60; +const logicalFreshAgeSec = 1; +const staleMaxAgeSec = 60; +const namespace = `dialcache-stale-benchmark-${process.pid}-${Date.now()}`; +const keyType = "benchmark_id"; +const id = "shared"; +const sourceError = new Error("benchmark source unavailable"); +const payloadPattern = "dialcache-stale-on-error-compressible-payload-"; +const payload = { + id, + // Repeated text intentionally exercises the default zstd path instead of + // benchmarking an unrealistically tiny raw JSON value. + body: payloadPattern.repeat( + Math.ceil(payloadBytes / payloadPattern.length), + ).slice(0, payloadBytes), +}; + +const redis = createClient({ + url: redisUrl, + scripts: dialcacheRedisScripts, + disableOfflineQueue: true, + commandsQueueMaxLength: 1_000, + socket: { connectTimeout: 2_000 }, +}); +redis.on("error", () => undefined); + +try { + await redis.connect(); +} catch (error) { + console.error( + `Could not reach Redis at ${redisUrl}; start one first, e.g. docker run --rm -p 6379:6379 redis:6.2`, + ); + throw error; +} + +const adapter = createNodeRedisDialCacheClient(redis); +const staleOutcomes = new Map(); +const noOpMetrics = { + request() {}, + miss() {}, + disabled() {}, + error() {}, + invalidation() {}, + coalesced() {}, + shadowValidation() {}, + staleRecovery({ outcome }) { + staleOutcomes.set(outcome, (staleOutcomes.get(outcome) ?? 0) + 1); + }, + compression() {}, + observeGet() {}, + observeFallback() {}, + observeSerialization() {}, + observeSize() {}, + observeStoredSize() {}, + observeCompressionRatio() {}, + observeCompression() {}, +}; +const dialcache = new DialCache({ + namespace, + redis: { + client: adapter, + readTimeoutMs: 2_000, + compression: { thresholdBytes: 1_024, level: 3 }, + }, + metrics: noOpMetrics, + logger: { debug() {}, warn() {}, error() {} }, +}); + +let freshSourceCalls = 0; +let freshWarmed = false; +const freshUseCase = "BenchmarkFreshHit"; +const loadFresh = dialcache.cached( + async () => { + freshSourceCalls += 1; + if (freshWarmed) { + throw new Error("fresh benchmark unexpectedly reached the source"); + } + return payload; + }, + { + keyType, + useCase: freshUseCase, + cacheKey: () => id, + defaultConfig: new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: freshAgeSec }, + }), + }, +); + +let staleSourceCalls = 0; +let staleSourceMode = "warm"; +let staleSourceGate; +const staleUseCase = "BenchmarkStaleRecovery"; +const loadStale = dialcache.cached( + async () => { + staleSourceCalls += 1; + if (staleSourceMode === "warm") { + return payload; + } + if (staleSourceMode === "gated-rejection") { + await staleSourceGate.promise; + } + throw sourceError; + }, + { + keyType, + useCase: staleUseCase, + cacheKey: () => id, + defaultConfig: new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: logicalFreshAgeSec }, + staleOnErrorMaxAgeSec: staleMaxAgeSec, + }), + }, +); + +const freshValueKey = redisValueKey(namespace, keyType, id, freshUseCase); +const staleValueKey = redisValueKey(namespace, keyType, id, staleUseCase); + +try { + assert.deepEqual(await dialcache.enable(async () => await loadFresh()), payload); + freshWarmed = true; + assert.deepEqual(await dialcache.enable(async () => await loadStale()), payload); + staleSourceMode = "rejection"; + + const storedBytes = await redis.strLen(staleValueKey); + const serializedBytes = Buffer.byteLength(JSON.stringify(payload)); + const physicalTtlMs = await redis.pTTL(staleValueKey); + assert(storedBytes > 10, "the stale benchmark frame must contain a payload"); + assert( + storedBytes < serializedBytes / 2, + "the representative payload should be materially compressed on Redis", + ); + assert( + physicalTtlMs <= staleMaxAgeSec * 1_000 + && physicalTtlMs >= staleMaxAgeSec * 1_000 - 5_000, + `stale-on-error writes must retain the frame near M; observed ${physicalTtlMs} ms`, + ); + + const rows = []; + rows.push(await measureScenario({ + name: "fresh end-to-end hit", + operations: iterations, + redis, + run: async () => { + for (let index = 0; index < iterations; index += 1) { + assert.deepEqual(await dialcache.enable(async () => await loadFresh()), payload); + } + }, + })); + assert.equal(freshSourceCalls, 1, "fresh hits must not reach the source after warmup"); + + // Writes are stamped with Redis TIME. Waiting beyond F makes the retained + // frame a deterministic normal miss while it remains physically present to M. + await wait(logicalFreshAgeSec * 1_000 + 100); + assert((await redis.pTTL(staleValueKey)) > 0, "the logically stale frame must remain retained"); + + rows.push(await measureScenario({ + name: "logical stale adapter miss", + operations: iterations, + redis, + run: async () => { + for (let index = 0; index < iterations; index += 1) { + assert.equal( + await adapter.read({ valueKey: staleValueKey, maxAgeMs: logicalFreshAgeSec * 1_000 }), + null, + ); + } + }, + })); + assert.notEqual( + await adapter.read({ valueKey: staleValueKey, maxAgeMs: staleMaxAgeSec * 1_000 }), + null, + "the same frame must remain eligible at M", + ); + + const recoveryOutcomesBefore = staleOutcomes.get("served") ?? 0; + const recoverySourceCallsBefore = staleSourceCalls; + rows.push(await measureScenario({ + name: "end-to-end stale recovery", + operations: iterations, + redis, + run: async () => { + for (let index = 0; index < iterations; index += 1) { + assert.deepEqual(await dialcache.enable(async () => await loadStale()), payload); + } + }, + })); + assert.equal(staleSourceCalls - recoverySourceCallsBefore, iterations); + assert.equal((staleOutcomes.get("served") ?? 0) - recoveryOutcomesBefore, iterations); + + staleSourceGate = deferred(); + staleSourceMode = "gated-rejection"; + const coalescedSourceCallsBefore = staleSourceCalls; + const coalescedOutcomesBefore = staleOutcomes.get("served") ?? 0; + const coalesced = await measureScenario({ + name: "coalesced stale recovery", + operations: fanout, + redis, + run: async () => { + const pending = Array.from( + { length: fanout }, + () => dialcache.enable(async () => await loadStale()), + ); + await waitFor(() => staleSourceCalls > coalescedSourceCallsBefore); + staleSourceGate.resolve(); + const values = await Promise.all(pending); + for (const value of values) { + assert.deepEqual(value, payload); + } + }, + }); + rows.push(coalesced); + assert.equal( + staleSourceCalls - coalescedSourceCallsBefore, + 1, + "same-key coalescing must share one source rejection", + ); + assert.equal( + (staleOutcomes.get("served") ?? 0) - coalescedOutcomesBefore, + 1, + "same-key coalescing must share one recovery read", + ); + + const serverInfo = parseInfo(await redis.sendCommand(["INFO", "server"])); + console.log( + `Stale-on-error benchmark — ${redisUrl} (${serverInfo.redis_version ?? "unknown engine"})`, + ); + console.log( + `payload JSON=${serializedBytes.toLocaleString("en-US")} B, stored frame=${storedBytes.toLocaleString("en-US")} B, F=${logicalFreshAgeSec}s, M=${staleMaxAgeSec}s`, + ); + console.table(rows.map((row) => ({ + scenario: row.name, + operations: row.operations, + "elapsed (ms)": row.elapsedMs.toFixed(2), + "ops/sec": Math.round((row.operations / row.elapsedMs) * 1_000).toLocaleString("en-US"), + "GET/op": perOperation(row.commands.get, row.operations), + "TIME/op": perOperation(row.commands.time, row.operations), + "server us/op": perOperation(row.serverUsec, row.operations), + "net in B/op": perOperation(row.netInputBytes, row.operations), + "net out B/op": perOperation(row.netOutputBytes, row.operations), + }))); + console.log( + "Semantic assertions passed. INFO deltas are observational and include small snapshot-query overhead; no timing threshold is applied.", + ); +} finally { + await redis.del([freshValueKey, staleValueKey]).catch(() => undefined); + await redis.quit().catch(() => redis.disconnect()); +} + +async function measureScenario({ name, operations, redis: client, run }) { + const before = await redisSnapshot(client); + const start = performance.now(); + await run(); + const elapsedMs = performance.now() - start; + const after = await redisSnapshot(client); + const commands = {}; + let serverUsec = 0; + for (const command of ["get", "mget", "time", "set", "evalsha", "eval"]) { + const calls = (after.commands[command]?.calls ?? 0) - (before.commands[command]?.calls ?? 0); + const usec = (after.commands[command]?.usec ?? 0) - (before.commands[command]?.usec ?? 0); + commands[command] = calls; + serverUsec += usec; + } + return { + name, + operations, + elapsedMs, + commands, + serverUsec, + netInputBytes: after.netInputBytes - before.netInputBytes, + netOutputBytes: after.netOutputBytes - before.netOutputBytes, + }; +} + +async function redisSnapshot(client) { + const commandInfo = String(await client.sendCommand(["INFO", "commandstats"])); + const statsInfo = parseInfo(await client.sendCommand(["INFO", "stats"])); + const commands = {}; + for (const line of commandInfo.split("\n")) { + const match = /^cmdstat_([a-z0-9_-]+):calls=(\d+),usec=(\d+)/.exec(line.trim()); + if (match !== null) { + commands[match[1]] = { calls: Number(match[2]), usec: Number(match[3]) }; + } + } + return { + commands, + netInputBytes: Number(statsInfo.total_net_input_bytes ?? 0), + netOutputBytes: Number(statsInfo.total_net_output_bytes ?? 0), + }; +} + +function parseInfo(raw) { + const values = {}; + for (const line of String(raw).split("\n")) { + const separator = line.indexOf(":"); + if (separator > 0 && line[0] !== "#") { + values[line.slice(0, separator)] = line.slice(separator + 1).trim(); + } + } + return values; +} + +function redisValueKey(cacheNamespace, cacheKeyType, cacheId, useCase) { + const key = new DialCacheKey({ + namespace: cacheNamespace, + keyType: cacheKeyType, + id: cacheId, + useCase, + }); + return `${key.urn}:dialcache-frame-v1`; +} + +function deferred() { + let resolve; + const promise = new Promise((resolvePromise) => { + resolve = resolvePromise; + }); + return { promise, resolve }; +} + +async function waitFor(predicate) { + for (let attempt = 0; attempt < 1_000; attempt += 1) { + if (predicate()) { + return; + } + await wait(1); + } + throw new Error("Timed out waiting for the benchmark source call"); +} + +function wait(milliseconds) { + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +} + +function perOperation(value, operations) { + return (value / operations).toFixed(2); +} + +function readPositiveInteger(name, fallback) { + const raw = process.env[name]; + if (raw === undefined) { + return fallback; + } + const parsed = Number(raw); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + throw new RangeError(`${name} must be a positive safe integer`); + } + return parsed; +} diff --git a/scripts/test-package.mjs b/scripts/test-package.mjs index 03b00ed..0465b34 100644 --- a/scripts/test-package.mjs +++ b/scripts/test-package.mjs @@ -43,27 +43,35 @@ const rootConsumer = `import { type RedisConfig, type RedisInvalidationRequest, type RedisReadContext, + type RedisReadRequest, type RedisWriteRequest, type Serializer, type ShadowComparator, type ShadowConfig, type ShadowValidationMetricLabels, type ShadowValidationOutcome, + type StaleRecoveryMetricLabels, + type StaleRecoveryOutcome, } from "dialcache"; // @ts-expect-error The unused MissingKeyConfigError class was removed instead of deprecated. import { MissingKeyConfigError } from "dialcache"; import { DialCacheRedisPlaceholderLostError } from "dialcache"; import { createNodeRedisDialCacheClient, dialcacheRedisScripts } from "dialcache/node-redis"; import { + assertValidRedisMaxAgeMs, ceilSupportedCacheTtlMs, decodeRedisFrame, + decodeRedisServerTime, decodeTrackedRedisFrame, encodeRedisFrame, encodeTrackedRedisPlaceholder, + isRedisFrameWithinMaxAge, + resolveUntrackedRedisWriteReply, resolveTrackedRedisWriteReply, validateRedisScriptInvalidationReply, validateRedisSetReply, WRITE_TRACKED_STAMP_SCRIPT, + WRITE_UNTRACKED_STAMP_SCRIPT, type DecodedRedisFrame, type TrackedRedisPlaceholder, } from "dialcache/redis-protocol"; @@ -77,7 +85,7 @@ import { REDIS_ENCODING_BINARY } from "dialcache/redis-protocol"; import { READ_CACHE_SCRIPT } from "dialcache/redis-protocol"; // @ts-expect-error Tracked read Lua was removed from the mutation-only Redis protocol. import { READ_TRACKED_CACHE_SCRIPT } from "dialcache/redis-protocol"; -// @ts-expect-error The untracked write Lua was replaced by a native client-framed SET. +// @ts-expect-error The monolithic untracked write Lua was replaced by native SET plus a stamp script. import { WRITE_CACHE_SCRIPT } from "dialcache/redis-protocol"; // @ts-expect-error The tracked write Lua was replaced by a native SET plus the stamp script. import { WRITE_TRACKED_CACHE_SCRIPT } from "dialcache/redis-protocol"; @@ -117,6 +125,13 @@ const shadowMetrics: DialCacheMetricsAdapter = { void outcome; }, }; +const staleMetrics: DialCacheMetricsAdapter = { + ...metrics, + staleRecovery: (labels: StaleRecoveryMetricLabels) => { + const outcome: StaleRecoveryOutcome = labels.outcome; + void outcome; + }, +}; const shadowOutcomes: Readonly> = { match: true, mismatch: true, @@ -133,6 +148,21 @@ const shadowOutcomes: Readonly> = { dropped: true, }; void shadowOutcomes; +const staleRecoveryOutcomes: Readonly> = { + served: true, + miss: true, + read_error: true, + read_timeout: true, + deserialization_error: true, +}; +const staleRecoveryLabels: StaleRecoveryMetricLabels = { + cacheNamespace: "consumer-cache", + useCase: "Load", + keyType: "id", + outcome: "served", +}; +void staleRecoveryOutcomes; +void staleRecoveryLabels; const metricLayers: Readonly> = { [CacheLayer.LOCAL]: true, [CacheLayer.REMOTE]: true, @@ -152,6 +182,11 @@ const shadowConfig: ShadowConfig = { logMismatches: true, }; const shadowKeyConfig = new DialCacheKeyConfig({ shadow: shadowConfig }); +const staleKeyConfig = new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 60 }, + staleOnErrorMaxAgeSec: 300, +}); +const staleRecoveryMaxAgeSec: number | undefined = staleKeyConfig.staleOnErrorMaxAgeSec; const dogStatsDClient: DatadogDogStatsDClient = { increment: () => undefined, histogram: () => undefined, @@ -176,14 +211,29 @@ const decodedStaleRedisFrame: DecodedRedisFrame | null = decodeTrackedRedisFrame emptyRedisFrame, Buffer.from("1"), ); +const decodedRedisServerTimeMs: number = decodeRedisServerTime([ + Buffer.from("1"), + Buffer.from("500000"), +]); +assertValidRedisMaxAgeMs(1_500); +const frameWithinMaxAge: boolean = decodedEmptyRedisFrame === null + ? false + : isRedisFrameWithinMaxAge(decodedEmptyRedisFrame, 1_500, 1_500); const placeholderRedisFrame: Buffer = encodeRedisFrame("pending", 0); const trackedRedisPlaceholder: TrackedRedisPlaceholder = encodeTrackedRedisPlaceholder("pending"); const stampReplyResolution: boolean = resolveTrackedRedisWriteReply(1); +const untrackedStampReplyResolution: true = resolveUntrackedRedisWriteReply(1); const setReplyValidation: void = validateRedisSetReply("OK"); const invalidationReplyValidation: 1 = validateRedisScriptInvalidationReply(1); const ceiledCacheTtlMs: number = ceilSupportedCacheTtlMs(1_000.5); const placeholderLostError = new DialCacheRedisPlaceholderLostError("lost"); const stampScriptSource: string = WRITE_TRACKED_STAMP_SCRIPT; +const untrackedStampScriptSource: string = WRITE_UNTRACKED_STAMP_SCRIPT; +const untrackedStampArguments: Array = + dialcacheRedisScripts.dialcacheWriteUntrackedStamp.transformArguments( + "untracked:{id}:value", + trackedRedisPlaceholder.nonce, + ); const stampArguments: Array = dialcacheRedisScripts.dialcacheWriteTrackedStamp.transformArguments( "tracked:{id}:value", "tracked:{id}:watermark", @@ -401,12 +451,24 @@ const compressionOperationMetricLabels: CompressionOperationMetricLabels = { const unboundedCompressionOutcome: CompressionOutcome = "inflated"; const customRedisClient: DialCacheRedisClient = { - // The optional second read argument preserves one-argument custom clients. - read: async () => ({ payload: Buffer.from([0, 255]), createdAtMs: 1 }), + enforcesMaxAge: true, + read: async ({ maxAgeMs }) => { + const createdAtMs = Date.now(); + return Date.now() - createdAtMs < maxAgeMs + ? { payload: Buffer.from([0, 255]), createdAtMs } + : null; + }, write: async ({ value }) => typeof value === "string" || Buffer.isBuffer(value), invalidate: async () => undefined, }; +const redisReadRequest: RedisReadRequest = { + valueKey: "untracked:{id}:value", + maxAgeMs: 1_000, +}; +// @ts-expect-error Semantic reads must always supply a logical maximum age. +const legacyRedisReadRequest: RedisReadRequest = { valueKey: "untracked:{id}:value" }; const redisClientMethods: Readonly> = { + enforcesMaxAge: true, read: true, write: true, invalidate: true, @@ -502,6 +564,9 @@ void coalesceFlag; void structuralConfigProvider; void shadowCache; void shadowKeyConfig; +void staleMetrics; +void staleKeyConfig; +void staleRecoveryMaxAgeSec; void requestLocalCoalescingLabels; void cacheMetricLabels; void invalidationMetricLabels; @@ -530,15 +595,18 @@ void redisConfigAcceptsCompressionOptOut; void createNodeRedisDialCacheClient; void decodedEmptyRedisFrame; void decodedStaleRedisFrame; +void decodedRedisServerTimeMs; +void frameWithinMaxAge; // @ts-expect-error Native reads removed the legacy node-redis registration. void dialcacheRedisScripts.dialcacheRead; // @ts-expect-error Native tracked reads removed the legacy node-redis registration. void dialcacheRedisScripts.dialcacheReadTracked; -// @ts-expect-error Native SET writes removed the legacy node-redis registration. +// @ts-expect-error Native placeholder SET plus stamp removed the legacy registration. void dialcacheRedisScripts.dialcacheWrite; // @ts-expect-error The stamp protocol removed the legacy tracked-write registration. void dialcacheRedisScripts.dialcacheWriteTracked; void dialcacheRedisScripts.dialcacheWriteTrackedStamp; +void dialcacheRedisScripts.dialcacheWriteUntrackedStamp; void READ_CACHE_SCRIPT; void READ_TRACKED_CACHE_SCRIPT; void WRITE_CACHE_SCRIPT; @@ -546,14 +614,19 @@ void WRITE_TRACKED_CACHE_SCRIPT; void placeholderRedisFrame; void trackedRedisPlaceholder; void stampReplyResolution; +void untrackedStampReplyResolution; void setReplyValidation; void placeholderLostError; void REDIS_FRAME_VERSION; void REDIS_ENCODING_UTF8; void REDIS_ENCODING_BINARY; void stampScriptSource; +void untrackedStampScriptSource; +void untrackedStampArguments; void stampArguments; void customRedisClient; +void redisReadRequest; +void legacyRedisReadRequest; const globalSerializer: Serializer = { dump: () => "global", load: () => ({ source: "global" }), @@ -730,7 +803,8 @@ const redisProtocol = await import("dialcache/redis-protocol"); // Each bundle embeds its own copy of the Lua sources; a divergence forks the // protocol (different SHA1s) without failing any behavioral test. if ( - nodeRedis.dialcacheRedisScripts.dialcacheWriteTrackedStamp.SCRIPT !== redisProtocol.WRITE_TRACKED_STAMP_SCRIPT + nodeRedis.dialcacheRedisScripts.dialcacheWriteUntrackedStamp.SCRIPT !== redisProtocol.WRITE_UNTRACKED_STAMP_SCRIPT + || nodeRedis.dialcacheRedisScripts.dialcacheWriteTrackedStamp.SCRIPT !== redisProtocol.WRITE_TRACKED_STAMP_SCRIPT || nodeRedis.dialcacheRedisScripts.dialcacheInvalidate.SCRIPT !== redisProtocol.INVALIDATE_CACHE_SCRIPT ) { throw new Error("The packed ESM node-redis Lua sources diverged from the redis-protocol entry"); @@ -773,6 +847,14 @@ try { throw new Error("The node-redis protocol error does not match the root ESM export"); } } +try { + nodeRedis.dialcacheRedisScripts.dialcacheWriteUntrackedStamp.transformReply(0); + throw new Error("Expected an invalid untracked node-redis script reply to fail"); +} catch (error) { + if (!(error instanceof root.DialCacheRedisProtocolError)) { + throw new Error("The untracked node-redis protocol error does not match the root ESM export"); + } +} if ("MissingKeyConfigError" in root) { throw new Error("The removed MissingKeyConfigError class must not be exported from the root ESM entry"); } @@ -800,13 +882,25 @@ if ( ) { throw new Error("The removed write scripts must not be exported by the packed ESM Redis protocol entry"); } -if (typeof redisProtocol.WRITE_TRACKED_STAMP_SCRIPT !== "string") { - throw new Error("The packed ESM Redis protocol entry must export the tracked stamp script source"); +if ( + typeof redisProtocol.WRITE_UNTRACKED_STAMP_SCRIPT !== "string" + || typeof redisProtocol.WRITE_TRACKED_STAMP_SCRIPT !== "string" +) { + throw new Error("The packed ESM Redis protocol entry must export both stamp script sources"); } const esmRoundTrip = redisProtocol.decodeRedisFrame(redisProtocol.encodeRedisFrame("value", 1)); if (esmRoundTrip?.payload !== "value" || esmRoundTrip.createdAtMs !== 1) { throw new Error("The packed ESM Redis protocol encoder did not round-trip through the decoder"); } +const esmServerNowMs = redisProtocol.decodeRedisServerTime([Buffer.from("1"), Buffer.from("500000")]); +redisProtocol.assertValidRedisMaxAgeMs(1_500); +if ( + esmServerNowMs !== 1500 + || !redisProtocol.isRedisFrameWithinMaxAge(esmRoundTrip, esmServerNowMs, 1500) + || redisProtocol.isRedisFrameWithinMaxAge(esmRoundTrip, esmServerNowMs, 1499) +) { + throw new Error("The packed ESM Redis logical-age helpers did not enforce the exact boundary"); +} if (redisProtocol.decodeTrackedRedisFrame(redisProtocol.encodeRedisFrame("pending", 0), Buffer.from("0")) !== null) { throw new Error("The packed ESM Redis protocol encoder did not produce a fenced placeholder frame"); } @@ -832,6 +926,17 @@ if ( ) { throw new Error("The packed ESM stamp reply resolver did not map replies 0 and 1"); } +if (redisProtocol.resolveUntrackedRedisWriteReply(1) !== true) { + throw new Error("The packed ESM untracked stamp reply resolver did not accept reply 1"); +} +try { + redisProtocol.resolveUntrackedRedisWriteReply(2); + throw new Error("Expected an untracked lost-placeholder stamp reply to fail"); +} catch (error) { + if (!(error instanceof root.DialCacheRedisPlaceholderLostError)) { + throw new Error("The untracked lost-placeholder error does not match the root ESM export"); + } +} try { redisProtocol.resolveTrackedRedisWriteReply(2); throw new Error("Expected a lost-placeholder stamp reply to fail"); @@ -908,6 +1013,7 @@ const esmDisabledOverlay = root.DialCacheKeyConfig.disabled(); if ( esmDisabledOverlay.requestLocal !== false || esmDisabledOverlay.coalesce !== undefined + || esmDisabledOverlay.staleOnErrorMaxAgeSec !== 0 || esmDisabledOverlay.shadow?.ramp !== 0 || esmDisabledOverlay.shadow.logMismatches !== false || esmDisabledOverlay.ramp[root.CacheLayer.LOCAL] !== 0 @@ -1015,7 +1121,11 @@ console.log("${observerIsolationMarker}");`, let payload = Buffer.alloc(4 * 1024 * 1024, 1); const payloadReference = new WeakRef(payload); const redis = { - read: async () => ({ payload, createdAtMs: 1 }), + enforcesMaxAge: true, + read: async ({ maxAgeMs }) => { + const createdAtMs = Date.now(); + return Date.now() - createdAtMs < maxAgeMs ? { payload, createdAtMs } : null; + }, write: async () => true, invalidate: async () => undefined, }; @@ -1103,7 +1213,8 @@ const redisProtocol = require("dialcache/redis-protocol"); // CommonJS bundles duplicate the Lua sources per entry point; a divergence // forks the protocol (different SHA1s) without failing any behavioral test. if ( - nodeRedis.dialcacheRedisScripts.dialcacheWriteTrackedStamp.SCRIPT !== redisProtocol.WRITE_TRACKED_STAMP_SCRIPT + nodeRedis.dialcacheRedisScripts.dialcacheWriteUntrackedStamp.SCRIPT !== redisProtocol.WRITE_UNTRACKED_STAMP_SCRIPT + || nodeRedis.dialcacheRedisScripts.dialcacheWriteTrackedStamp.SCRIPT !== redisProtocol.WRITE_TRACKED_STAMP_SCRIPT || nodeRedis.dialcacheRedisScripts.dialcacheInvalidate.SCRIPT !== redisProtocol.INVALIDATE_CACHE_SCRIPT ) { throw new Error("The packed CommonJS node-redis Lua sources diverged from the redis-protocol entry"); @@ -1148,6 +1259,14 @@ try { throw new Error("The node-redis protocol error does not match the root CommonJS export"); } } +try { + nodeRedis.dialcacheRedisScripts.dialcacheWriteUntrackedStamp.transformReply(0); + throw new Error("Expected an invalid untracked node-redis script reply to fail"); +} catch (error) { + if (!(error instanceof root.DialCacheRedisProtocolError)) { + throw new Error("The untracked node-redis protocol error does not match the root CommonJS export"); + } +} if ("MissingKeyConfigError" in root) { throw new Error("The removed MissingKeyConfigError class must not be exported from the root CommonJS entry"); } @@ -1175,13 +1294,25 @@ if ( ) { throw new Error("The removed write scripts must not be exported by the packed CommonJS Redis protocol entry"); } -if (typeof redisProtocol.WRITE_TRACKED_STAMP_SCRIPT !== "string") { - throw new Error("The packed CommonJS Redis protocol entry must export the tracked stamp script source"); +if ( + typeof redisProtocol.WRITE_UNTRACKED_STAMP_SCRIPT !== "string" + || typeof redisProtocol.WRITE_TRACKED_STAMP_SCRIPT !== "string" +) { + throw new Error("The packed CommonJS Redis protocol entry must export both stamp script sources"); } const cjsRoundTrip = redisProtocol.decodeRedisFrame(redisProtocol.encodeRedisFrame("value", 1)); if (cjsRoundTrip?.payload !== "value" || cjsRoundTrip.createdAtMs !== 1) { throw new Error("The packed CommonJS Redis protocol encoder did not round-trip through the decoder"); } +const cjsServerNowMs = redisProtocol.decodeRedisServerTime([Buffer.from("1"), Buffer.from("500000")]); +redisProtocol.assertValidRedisMaxAgeMs(1_500); +if ( + cjsServerNowMs !== 1500 + || !redisProtocol.isRedisFrameWithinMaxAge(cjsRoundTrip, cjsServerNowMs, 1500) + || redisProtocol.isRedisFrameWithinMaxAge(cjsRoundTrip, cjsServerNowMs, 1499) +) { + throw new Error("The packed CommonJS Redis logical-age helpers did not enforce the exact boundary"); +} if (redisProtocol.decodeTrackedRedisFrame(redisProtocol.encodeRedisFrame("pending", 0), Buffer.from("0")) !== null) { throw new Error("The packed CommonJS Redis protocol encoder did not produce a fenced placeholder frame"); } @@ -1207,6 +1338,17 @@ if ( ) { throw new Error("The packed CommonJS stamp reply resolver did not map replies 0 and 1"); } +if (redisProtocol.resolveUntrackedRedisWriteReply(1) !== true) { + throw new Error("The packed CommonJS untracked stamp reply resolver did not accept reply 1"); +} +try { + redisProtocol.resolveUntrackedRedisWriteReply(2); + throw new Error("Expected an untracked lost-placeholder stamp reply to fail"); +} catch (error) { + if (!(error instanceof root.DialCacheRedisPlaceholderLostError)) { + throw new Error("The untracked lost-placeholder error does not match the root CommonJS export"); + } +} try { redisProtocol.resolveTrackedRedisWriteReply(2); throw new Error("Expected a lost-placeholder stamp reply to fail"); @@ -1283,6 +1425,7 @@ const cjsDisabledOverlay = root.DialCacheKeyConfig.disabled(); if ( cjsDisabledOverlay.requestLocal !== false || cjsDisabledOverlay.coalesce !== undefined + || cjsDisabledOverlay.staleOnErrorMaxAgeSec !== 0 || cjsDisabledOverlay.shadow?.ramp !== 0 || cjsDisabledOverlay.shadow.logMismatches !== false || cjsDisabledOverlay.ramp[root.CacheLayer.LOCAL] !== 0 diff --git a/src/config.ts b/src/config.ts index 5b89c81..9b3ba8f 100644 --- a/src/config.ts +++ b/src/config.ts @@ -40,6 +40,14 @@ export class DialCacheKeyConfig { * independent fallback deadline, and its own cache writes. */ readonly coalesce?: boolean; + /** + * Absolute Redis-frame age in seconds through which a retained value may be + * returned after the source of truth rejects. Omission disables recovery by + * default and inherits in runtime overlays; zero explicitly disables an + * inherited policy. A positive value requires a smaller positive remote TTL + * and may not exceed 31,536,000 seconds (365 days). + */ + readonly staleOnErrorMaxAgeSec?: number; /** * Maximum time DialCache waits for a remote read before failing open to the * source of truth. Overrides the instance default for this use case. @@ -52,6 +60,7 @@ export class DialCacheKeyConfig { shadow?: ShadowConfig; requestLocal?: boolean; coalesce?: boolean; + staleOnErrorMaxAgeSec?: number; remoteReadTimeoutMs?: number; }) { if (config === null || typeof config !== "object" || Array.isArray(config)) { @@ -78,6 +87,11 @@ export class DialCacheKeyConfig { if (config.coalesce !== undefined) { this.coalesce = config.coalesce; } + // Like ttlSec/ramp leaves, validation is deferred to static-default capture + // or runtime resolution so malformed runtime policy can fail open narrowly. + if (config.staleOnErrorMaxAgeSec !== undefined) { + this.staleOnErrorMaxAgeSec = config.staleOnErrorMaxAgeSec; + } if (config.remoteReadTimeoutMs !== undefined) { assertValidDeadlineMs(config.remoteReadTimeoutMs, "DialCache remoteReadTimeoutMs"); this.remoteReadTimeoutMs = config.remoteReadTimeoutMs; @@ -107,6 +121,7 @@ export class DialCacheKeyConfig { static disabled(): DialCacheKeyConfig { return new DialCacheKeyConfig({ requestLocal: false, + staleOnErrorMaxAgeSec: 0, shadow: { ramp: 0, logMismatches: false, diff --git a/src/datadog.ts b/src/datadog.ts index 1a28fda..7d5018d 100644 --- a/src/datadog.ts +++ b/src/datadog.ts @@ -9,6 +9,7 @@ import type { InvalidationMetricLabels, SerializationMetricLabels, ShadowValidationMetricLabels, + StaleRecoveryMetricLabels, } from "./metrics.js"; export type DatadogObservationMetricType = "histogram" | "distribution"; @@ -49,6 +50,7 @@ const METRIC_SUFFIXES = { coalesced: "coalesced.count", shadowValidation: "shadow.count", shadowValueAge: "shadow.value_age", + staleRecovery: "stale_recovery.count", compression: "compression.count", get: "get.duration", fallback: "fallback.duration", @@ -128,6 +130,15 @@ export class DatadogDialCacheMetrics implements DialCacheMetricsAdapter { this.observe(this.metricNames.shadowValueAge, seconds, shadowValidationTags(labels)); } + staleRecovery(labels: StaleRecoveryMetricLabels): void { + return this.increment(this.metricNames.staleRecovery, { + cache_namespace: labels.cacheNamespace, + use_case: labels.useCase, + key_type: labels.keyType, + outcome: labels.outcome, + }); + } + compression(labels: CompressionMetricLabels): void { this.increment(this.metricNames.compression, { ...cacheTags(labels), outcome: labels.outcome }); } diff --git a/src/dialcache.ts b/src/dialcache.ts index afd8468..71c9403 100644 --- a/src/dialcache.ts +++ b/src/dialcache.ts @@ -37,9 +37,10 @@ import { deterministicShadowRampSample } from "./internal/ramp.js"; import { RedisCache } from "./internal/redis-cache.js"; import { fetchKeyConfig, - resolveLayerConfigResult, + resolveRemoteLayerConfigResult, type LayerConfigResolution, type ResolvedLayerConfig, + type ResolvedRemoteLayerConfig, } from "./internal/runtime-config.js"; import { shadowMismatchLogDetails } from "./internal/shadow-log-json.js"; @@ -229,23 +230,27 @@ interface ShadowMismatchDetails { } type ShadowValidationStart = - | { readonly kind: "retained"; readonly frame: DecodedRedisFrame } + | { + readonly kind: "retained"; + readonly frame: DecodedRedisFrame; + readonly remoteConfig: ResolvedRemoteLayerConfig; + } | { readonly kind: "redis"; /** The caller-owned, fallback-deadline-bounded SoT operation. */ readonly source: Promise; /** Valid remote policy retained even though its serving ramp excluded this key. */ - readonly remoteConfig: ResolvedLayerConfig; + readonly remoteConfig: ResolvedRemoteLayerConfig; /** Includes synchronous SoT work that ran before shadow admission. */ readonly startedAtMs: number | null; }; type ShadowValidationRunStart = - | { readonly kind: "retained" } + | { readonly kind: "retained"; readonly remoteConfig: ResolvedRemoteLayerConfig } | { readonly kind: "redis"; readonly source: Promise; - readonly remoteConfig: ResolvedLayerConfig; + readonly remoteConfig: ResolvedRemoteLayerConfig; }; const DEFAULT_LOCAL_MAX_SIZE = 10_000; @@ -721,7 +726,7 @@ export class DialCache { key: DialCacheKey, keyConfig: DialCacheKeyConfig | null, local: CacheGetResult | null, - remoteConfig: ResolvedLayerConfig, + remoteConfig: ResolvedRemoteLayerConfig, fallbackLabels: CacheMetricLabels, fallback: () => Promise, shadowValidation: ShadowValidationPlan, @@ -754,20 +759,22 @@ export class DialCache { remote: RemoteCacheGetResult, fallback: () => Promise, shadowValidation: ShadowValidationPlan, - resolvedRemoteConfig?: ResolvedLayerConfig, + resolvedRemoteConfig?: ResolvedRemoteLayerConfig, ): Promise { if (remote.status === "hit") { if (local.status === "miss") { await this.putLocalFailOpen(key, remote.value, local.config); } - this.scheduleShadowValidation( - redisCache, - key, - keyConfig, - { kind: "retained", frame: remote.frame }, - shadowValidation, - keyConfig?.remoteReadTimeoutMs ?? redisCache.readTimeoutMs, - ); + if (resolvedRemoteConfig !== undefined) { + this.scheduleShadowValidation( + redisCache, + key, + keyConfig, + { kind: "retained", frame: remote.frame, remoteConfig: resolvedRemoteConfig }, + shadowValidation, + keyConfig?.remoteReadTimeoutMs ?? redisCache.readTimeoutMs, + ); + } return remote.value; } @@ -780,9 +787,37 @@ export class DialCache { } const remoteErrored = remote.status === "disabled" && remote.reason === "config_error"; - const remoteWriteConfig = remote.status === "miss" ? remote.config : remoteErrored ? resolvedRemoteConfig : undefined; + const remoteWriteConfig = remote.status === "miss" || remoteErrored ? resolvedRemoteConfig : undefined; const fallbackLayer = remote.status === "miss" || remoteErrored ? CacheLayer.REMOTE : CacheLayer.LOCAL; - const value = await this.callFallback(labelsFor(key, fallbackLayer), fallback); + let value: T; + try { + value = await this.callFallback(labelsFor(key, fallbackLayer), fallback); + } catch (fallbackError) { + if ( + remote.status === "miss" + && remote.skipStaleRecovery !== true + && resolvedRemoteConfig !== undefined + && resolvedRemoteConfig.staleOnErrorMaxAgeSec !== null + ) { + try { + const recovered = await redisCache.recoverWithResolvedConfig( + key, + resolvedRemoteConfig, + keyConfig?.remoteReadTimeoutMs ?? redisCache.readTimeoutMs, + ); + if (recovered.status === "hit") { + return recovered.value; + } + if (recovered.status === "error") { + this.logger.warn("Error getting value from Redis cache during stale recovery", recovered.error); + } + } catch (recoveryError) { + // Recovery is subordinate to the source rejection and must never replace it. + this.logger.warn("Error getting value from Redis cache during stale recovery", recoveryError); + } + } + throw fallbackError; + } const skipCacheWrite = (remote.status === "miss" || remote.status === "disabled") && remote.skipCacheWrite === true; let suppressCacheWrite = skipCacheWrite; if (!suppressCacheWrite && remoteWriteConfig !== undefined) { @@ -855,7 +890,7 @@ export class DialCache { : performance.now(), }; const runStart: ShadowValidationRunStart = start.kind === "retained" - ? { kind: "retained" } + ? { kind: "retained", remoteConfig: start.remoteConfig } : { kind: "redis", source: start.source, remoteConfig: start.remoteConfig }; this.shadowFlights.set(key.urn, flight); this.deferShadowValidation( @@ -943,7 +978,11 @@ export class DialCache { maybeRelease(); }; const readShadowFrame = (): Promise => { - const read = redisCache.startPayloadReadForShadow(key, readTimeoutMs); + const read = redisCache.startPayloadReadForShadow( + key, + start.remoteConfig.ttlSec, + readTimeoutMs, + ); pendingRedisReads.add(read.settled); void read.settled.then(() => { pendingRedisReads.delete(read.settled); @@ -981,7 +1020,7 @@ export class DialCache { return "timeout"; } - let shadowFillConfig: ResolvedLayerConfig | null = null; + let shadowFillConfig: ResolvedRemoteLayerConfig | null = null; if (start.kind === "redis") { let frame: DecodedRedisFrame | null; try { @@ -1203,11 +1242,13 @@ export class DialCache { private async resolveRemoteLayerConfig(key: DialCacheKey, keyConfig: DialCacheKeyConfig | null) { try { - const result = resolveLayerConfigResult({ + const result = resolveRemoteLayerConfigResult({ config: keyConfig, key, - layer: CacheLayer.REMOTE, }); + if (result.staleOnErrorConfigError === true) { + this.recordError(key, CacheLayer.REMOTE, "config_resolution"); + } if (result.status === "disabled") { this.metrics?.disabled({ ...labelsFor(key, CacheLayer.REMOTE), reason: result.reason }); this.recordInvalidLeaf(key, CacheLayer.REMOTE, result.reason); @@ -1224,7 +1265,7 @@ export class DialCache { private async readRemoteWithResolvedConfig( redisCache: RedisCache, key: DialCacheKey, - layerConfig: ResolvedLayerConfig, + layerConfig: ResolvedRemoteLayerConfig, readTimeoutMs: number, ): Promise> { try { @@ -1400,6 +1441,7 @@ function snapshotDefaultConfig(config: DialCacheKeyConfig | null | undefined): D const shadowConfig = config.shadow; const requestLocal = config.requestLocal; const coalesce = config.coalesce; + const staleOnErrorMaxAgeSec = config.staleOnErrorMaxAgeSec; const remoteReadTimeoutMs = config.remoteReadTimeoutMs; if (requestLocal !== undefined && typeof requestLocal !== "boolean") { throw new TypeError("DialCache defaultConfig requestLocal must be a boolean"); @@ -1417,6 +1459,7 @@ function snapshotDefaultConfig(config: DialCacheKeyConfig | null | undefined): D ramp: rampConfig, ...(requestLocal === undefined ? {} : { requestLocal }), ...(coalesce === undefined ? {} : { coalesce }), + ...(staleOnErrorMaxAgeSec === undefined ? {} : { staleOnErrorMaxAgeSec }), ...(remoteReadTimeoutMs === undefined ? {} : { remoteReadTimeoutMs }), ...(shadowConfig === undefined ? {} : { shadow: shadowConfig }), }); @@ -1445,6 +1488,31 @@ function snapshotDefaultConfig(config: DialCacheKeyConfig | null | undefined): D } } + if (snapshot.staleOnErrorMaxAgeSec !== undefined) { + const maxAgeSec = snapshot.staleOnErrorMaxAgeSec; + if (typeof maxAgeSec !== "number") { + throw new TypeError("DialCache defaultConfig staleOnErrorMaxAgeSec must be a number"); + } + if (!Number.isSafeInteger(maxAgeSec) || maxAgeSec < 0 || maxAgeSec > MAX_CACHE_TTL_SEC) { + throw new RangeError( + `DialCache defaultConfig staleOnErrorMaxAgeSec must be a nonnegative safe integer no greater than ${MAX_CACHE_TTL_SEC}`, + ); + } + if (maxAgeSec > 0) { + const remoteTtlSec = snapshot.ttlSec[CacheLayer.REMOTE]; + if (remoteTtlSec === undefined) { + throw new RangeError( + "DialCache defaultConfig staleOnErrorMaxAgeSec requires ttlSec.remote", + ); + } + if (maxAgeSec <= remoteTtlSec) { + throw new RangeError( + "DialCache defaultConfig staleOnErrorMaxAgeSec must be greater than ttlSec.remote", + ); + } + } + } + if (snapshot.shadow !== undefined) { if (snapshot.shadow.ramp !== undefined) { if (typeof snapshot.shadow.ramp !== "number") { @@ -1543,6 +1611,12 @@ function safeMetrics(metrics: DialCacheMetricsAdapter | null): DialCacheMetricsA callObserver(() => metrics.shadowValidation!(labels)), } : {}), + ...(typeof metrics.staleRecovery === "function" + ? { + staleRecovery: (labels) => + callObserver(() => metrics.staleRecovery!(labels)), + } + : {}), observeShadowValueAge: (labels, seconds) => callObserver(() => metrics.observeShadowValueAge?.(labels, seconds)), observeGet: (labels, seconds) => callObserver(() => metrics.observeGet(labels, seconds)), @@ -1574,11 +1648,11 @@ function resolveShadowComparator( return comparator ?? isDeepStrictEqual; } -// Frame stamps are epoch-based (Redis server time for tracked writes, writer -// client clock for untracked), so the age uses the epoch clock and clamps -// negative cross-clock skew to zero. A custom client that violates the decode -// contract can hand over a non-finite stamp; recording it would permanently -// poison backend histogram sums, so the observation is skipped instead. +// Frame stamps are epoch-based Redis server time, so the age uses the epoch +// clock and clamps negative reader/server skew to zero. A custom client that +// violates the decode contract can hand over a non-finite stamp; recording it +// would permanently poison backend histogram sums, so the observation is +// skipped instead. function shadowValueAgeSeconds(createdAtMs: number): number | undefined { const ageSeconds = (Date.now() - createdAtMs) / 1000; if (!Number.isFinite(ageSeconds)) { diff --git a/src/index.ts b/src/index.ts index cc10a23..b52e594 100644 --- a/src/index.ts +++ b/src/index.ts @@ -18,6 +18,8 @@ export type { SerializationMetricLabels, ShadowValidationMetricLabels, ShadowValidationOutcome, + StaleRecoveryMetricLabels, + StaleRecoveryOutcome, } from "./metrics.js"; export { DialCacheError, diff --git a/src/internal/cache-result.ts b/src/internal/cache-result.ts index 9abae50..a4cdfaa 100644 --- a/src/internal/cache-result.ts +++ b/src/internal/cache-result.ts @@ -1,4 +1,4 @@ -import type { ResolvedLayerConfig } from "./runtime-config.js"; +import type { ResolvedLayerConfig, ResolvedRemoteLayerConfig } from "./runtime-config.js"; import type { DisabledReason } from "../metrics.js"; import type { DecodedRedisFrame } from "../redis-client.js"; @@ -13,7 +13,14 @@ export type CacheGetResult = export type RedisCacheGetResult = | { readonly status: "hit"; readonly value: T; readonly frame: DecodedRedisFrame } - | Exclude, { readonly status: "hit" }>; + | { + readonly status: "miss"; + readonly config: ResolvedRemoteLayerConfig; + readonly skipCacheWrite?: boolean; + /** The present payload failed normal decoding and cannot later qualify as stale. */ + readonly skipStaleRecovery?: boolean; + } + | Extract, { readonly status: "disabled" }>; export type RemoteCacheGetResult = | RedisCacheGetResult diff --git a/src/internal/redis-cache.ts b/src/internal/redis-cache.ts index ad25fe8..779e44d 100644 --- a/src/internal/redis-cache.ts +++ b/src/internal/redis-cache.ts @@ -9,6 +9,7 @@ import { type DialCacheMetricsAdapter, type MetricErrorKind, type MetricLayer, + type StaleRecoveryOutcome, } from "../metrics.js"; import type { DecodedRedisFrame, DialCacheRedisClient, RedisCachePayload } from "../redis-client.js"; import { JsonSerializer, type Serializer } from "../serializer.js"; @@ -22,7 +23,11 @@ import { } from "./compression.js"; import { assertValidDeadlineMs, withMonotonicDeadline } from "./deadline.js"; import { cacheTtlSecToMs } from "./duration.js"; -import { fetchKeyConfig, resolveLayerConfigResult, type ResolvedLayerConfig } from "./runtime-config.js"; +import { + fetchKeyConfig, + resolveRemoteLayerConfigResult, + type ResolvedRemoteLayerConfig, +} from "./runtime-config.js"; export interface RedisConfig { /** @@ -58,6 +63,11 @@ interface StartedRedisRead { readonly settled: Promise; } +type RedisStaleRecoveryResult = + | { readonly status: "hit"; readonly value: T } + | { readonly status: "miss" } + | { readonly status: "error"; readonly error: unknown }; + const defaultSerializer = new JsonSerializer(); const REDIS_FRAME_KEY_SUFFIX = ":dialcache-frame-v1"; const DEFAULT_REMOTE_READ_TIMEOUT_MS = 50; @@ -95,6 +105,9 @@ export class RedisCache { if (options.redis.client === undefined) { throw new TypeError("Redis config requires client"); } + if (options.redis.client.enforcesMaxAge !== true) { + throw new TypeError("DialCache Redis client must declare enforcesMaxAge: true"); + } this.client = options.redis.client; } @@ -119,7 +132,7 @@ export class RedisCache { async getWithResolvedConfig( key: DialCacheKey, - layerConfig: ResolvedLayerConfig, + layerConfig: ResolvedRemoteLayerConfig, readTimeoutMs = this.readTimeoutMs, ): Promise> { const metricLayer = CacheLayer.REMOTE; @@ -128,7 +141,12 @@ export class RedisCache { try { let frame: DecodedRedisFrame | null; try { - frame = await this.startPayloadRead(key, readTimeoutMs, false).result; + frame = await this.startPayloadRead( + key, + cacheTtlSecToMs(layerConfig.ttlSec), + readTimeoutMs, + false, + ).result; } catch (error) { this.recordError( key, @@ -147,7 +165,7 @@ export class RedisCache { return { status: "hit", value, frame }; } catch { this.recordMetric((metrics) => metrics.miss(labelsFor(key, metricLayer))); - return { status: "miss", config: layerConfig }; + return { status: "miss", config: layerConfig, skipStaleRecovery: true }; } } finally { // Preserve the established caller-serving boundary: Redis read plus load. @@ -155,6 +173,64 @@ export class RedisCache { } } + /** + * Reread a definitive normal miss after the source rejects, using the + * configured absolute recovery age. Every failure is contained so it cannot + * replace the original source rejection held by the caller. + */ + async recoverWithResolvedConfig( + key: DialCacheKey, + layerConfig: ResolvedRemoteLayerConfig, + readTimeoutMs: number, + ): Promise> { + const metricLayer = CacheLayer.REMOTE; + const maxAgeSec = layerConfig.staleOnErrorMaxAgeSec; + if (maxAgeSec === null) { + throw new Error("DialCache stale recovery requires an enabled maximum age"); + } + + const start = performance.now(); + this.recordMetric((metrics) => metrics.request(labelsFor(key, metricLayer))); + try { + let frame: DecodedRedisFrame | null; + try { + frame = await this.startPayloadRead( + key, + cacheTtlSecToMs(maxAgeSec), + readTimeoutMs, + false, + ).result; + } catch (error) { + const outcome = error instanceof RedisReadTimeoutError ? "read_timeout" : "read_error"; + this.recordError( + key, + metricLayer, + error instanceof RedisReadTimeoutError ? "cache_read_timeout" : "cache_read", + ); + this.recordStaleRecovery(key, outcome); + return { status: "error", error }; + } + + if (frame === null) { + this.recordMetric((metrics) => metrics.miss(labelsFor(key, metricLayer))); + this.recordStaleRecovery(key, "miss"); + return { status: "miss" }; + } + + try { + const value = await this.deserializePayload(key, frame.payload, metricLayer); + this.recordStaleRecovery(key, "served"); + return { status: "hit", value }; + } catch { + this.recordMetric((metrics) => metrics.miss(labelsFor(key, metricLayer))); + this.recordStaleRecovery(key, "deserialization_error"); + return { status: "miss" }; + } + } finally { + this.recordMetric((metrics) => metrics.observeGet(labelsFor(key, metricLayer), elapsedSeconds(start))); + } + } + /** * Decode the retained Redis payload again for detached semantic comparison, * recording it separately from caller-serving Redis work. @@ -171,18 +247,22 @@ export class RedisCache { */ startPayloadReadForShadow( key: DialCacheKey, + maxAgeSec: number, readTimeoutMs: number, ): StartedRedisRead { return this.startMeasuredPayloadRead( key, + cacheTtlSecToMs(maxAgeSec), readTimeoutMs, REMOTE_SHADOW_CACHE_LAYER, true, ); } - async put(key: DialCacheKey, value: T, config?: { readonly ttlSec: number }): Promise { - const ttlSec = config?.ttlSec ?? await this.resolveRemoteTtlSec(key); + async put(key: DialCacheKey, value: T, config?: ResolvedRemoteLayerConfig): Promise { + const ttlSec = config === undefined + ? await this.resolveRemoteRetentionTtlSec(key) + : retentionTtlSecFor(config); if (ttlSec === null) { return true; } @@ -193,13 +273,13 @@ export class RedisCache { async putForShadow( key: DialCacheKey, value: T, - config: { readonly ttlSec: number }, + config: ResolvedRemoteLayerConfig, shouldWrite: () => boolean, ): Promise { return await this.putWithLayer( key, value, - config.ttlSec, + retentionTtlSecFor(config), REMOTE_SHADOW_CACHE_LAYER, shouldWrite, ); @@ -306,6 +386,7 @@ export class RedisCache { private startPayloadRead( key: DialCacheKey, + maxAgeMs: number, readTimeoutMs: number, unrefTimer: boolean, ): StartedRedisRead { @@ -315,6 +396,7 @@ export class RedisCache { { valueKey: this.redisKey(key), ...(key.trackForInvalidation ? { watermarkKey: this.redisWatermarkKeyFromKey(key) } : {}), + maxAgeMs, }, { timeoutMs: readTimeoutMs, signal: abortController.signal }, ) @@ -337,13 +419,14 @@ export class RedisCache { private startMeasuredPayloadRead( key: DialCacheKey, + maxAgeMs: number, readTimeoutMs: number, metricLayer: MetricLayer, unrefTimer: boolean, ): StartedRedisRead { const start = performance.now(); this.recordMetric((metrics) => metrics.request(labelsFor(key, metricLayer))); - const read = this.startPayloadRead(key, readTimeoutMs, unrefTimer); + const read = this.startPayloadRead(key, maxAgeMs, readTimeoutMs, unrefTimer); const result = read.result.then( (frame) => { if (frame === null) { @@ -399,16 +482,15 @@ export class RedisCache { private async resolveRemoteLayerConfig(key: DialCacheKey, keyConfig?: DialCacheKeyConfig | null) { const config = keyConfig === undefined ? await fetchKeyConfig(this.configProvider, key) : keyConfig; - return resolveLayerConfigResult({ + return resolveRemoteLayerConfigResult({ config, key, - layer: CacheLayer.REMOTE, }); } - private async resolveRemoteTtlSec(key: DialCacheKey): Promise { + private async resolveRemoteRetentionTtlSec(key: DialCacheKey): Promise { const layerConfig = await this.resolveRemoteLayerConfig(key); - return layerConfig.status === "enabled" ? layerConfig.config.ttlSec : null; + return layerConfig.status === "enabled" ? retentionTtlSecFor(layerConfig.config) : null; } private recordMetric(record: (metrics: DialCacheMetricsAdapter) => void): void { @@ -425,6 +507,19 @@ export class RedisCache { private recordError(key: DialCacheKey, layer: MetricLayer, kind: MetricErrorKind): void { this.recordMetric((metrics) => metrics.error({ ...labelsFor(key, layer), error: kind, inFallback: false })); } + + private recordStaleRecovery(key: DialCacheKey, outcome: StaleRecoveryOutcome): void { + this.recordMetric((metrics) => metrics.staleRecovery?.({ + cacheNamespace: key.namespace, + useCase: key.useCase, + keyType: key.keyType, + outcome, + })); + } +} + +function retentionTtlSecFor(config: ResolvedRemoteLayerConfig): number { + return config.staleOnErrorMaxAgeSec ?? config.ttlSec; } function payloadSize(payload: string | Buffer): number { diff --git a/src/internal/redis-payload.ts b/src/internal/redis-payload.ts index fbc3f8e..e21cbc3 100644 --- a/src/internal/redis-payload.ts +++ b/src/internal/redis-payload.ts @@ -6,11 +6,12 @@ import { type DecodedRedisFrame, type RedisCachePayload, } from "../redis-client.js"; +import { MAX_SUPPORTED_DURATION_MS } from "./duration.js"; export const REDIS_FRAME_VERSION = 1; const REDIS_ENCODING_UTF8 = 0; const REDIS_ENCODING_BINARY = 1; -/** Version byte of a tracked-write placeholder; no read path serves it. */ +/** Version byte of a write placeholder; no read path serves it. */ export const REDIS_FRAME_PLACEHOLDER_VERSION = 0; const REDIS_FRAME_TIMESTAMP_OFFSET = 1; export const REDIS_FRAME_TIMESTAMP_BYTES = 8; @@ -79,12 +80,10 @@ function encodeFrameBytes(payload: RedisCachePayload, version: number, stampByte /** * Encode a serializer payload into a servable DialCache Redis frame. * - * Untracked writes stamp a client-clock `createdAtMs`. Untracked reads never - * consult the stamp for serving or miss decisions, but they surface it as the - * decoded frame's `createdAtMs`, which feeds the shadow value-age - * observation — so stamp real client time, not a constant. Tracked writes - * must not use this directly — they pair `encodeTrackedRedisPlaceholder` - * with `WRITE_TRACKED_STAMP_SCRIPT` instead. + * This fully stamped encoder is available for protocol tooling. Bundled + * adapters do not use it for writes: they pair + * `encodeTrackedRedisPlaceholder` with the appropriate server-time stamp + * script so logical age never depends on the writer's wall clock. */ export function encodeRedisFrame(payload: RedisCachePayload, createdAtMs: number): Buffer { if (!Number.isSafeInteger(createdAtMs) || createdAtMs < 0) { @@ -98,13 +97,14 @@ export function encodeRedisFrame(payload: RedisCachePayload, createdAtMs: number export interface TrackedRedisPlaceholder { /** Version-0 frame that no read path serves until the stamp promotes it. */ readonly frame: Buffer; - /** Per-write identity passed to `WRITE_TRACKED_STAMP_SCRIPT` as its nonce argument. */ + /** Per-write identity passed to the selected stamp script as its nonce argument. */ readonly nonce: Buffer; } /** - * Encode the placeholder frame a tracked write pairs with - * `WRITE_TRACKED_STAMP_SCRIPT`. + * Encode the placeholder frame a write pairs with its tracked or untracked + * server-time stamp script. The historical public name is retained because + * tracked writes introduced this wire shape. * * The frame carries the placeholder version byte, so both read paths treat it * as a miss, and a fresh random nonce where a stamped frame carries its @@ -121,20 +121,23 @@ export function encodeTrackedRedisPlaceholder(payload: RedisCachePayload): Track } /** - * Decode an untracked DialCache frame returned as a Redis bulk string into - * its serializer payload and header creation time (the writer's informational - * client clock). Missing, short, and unsupported-version frames are cache - * misses. Invalid runtime reply types and unsupported payload encodings throw - * typed errors. + * Decode an untracked DialCache frame returned as a Redis bulk string into its + * serializer payload and Redis-server creation time. Missing, short, + * unsupported-version, and unsafe-timestamp frames are cache misses. Invalid + * runtime reply types and unsupported payload encodings throw typed errors. */ export function decodeRedisFrame(raw: unknown): DecodedRedisFrame | null { const frame = validateRedisBulkStringReply(raw); if (!isSupportedRedisFrame(frame)) { return null; } + const createdAtMs = readFrameCreatedAtMs(frame); + if (createdAtMs === null) { + return null; + } return { payload: decodeRedisPayload(frame.subarray(REDIS_FRAME_HEADER_BYTES)), - createdAtMs: readFrameCreatedAtMs(frame), + createdAtMs, }; } @@ -159,7 +162,7 @@ export function decodeTrackedRedisFrame( return null; } const createdAtMs = readFrameCreatedAtMs(frame); - return createdAtMs <= watermark + return createdAtMs === null || createdAtMs <= watermark ? null : { payload: decodeRedisPayload(frame.subarray(REDIS_FRAME_HEADER_BYTES)), @@ -167,6 +170,69 @@ export function decodeTrackedRedisFrame( }; } -function readFrameCreatedAtMs(frame: Buffer): number { - return Number(frame.readBigUInt64BE(REDIS_FRAME_TIMESTAMP_OFFSET)); +/** + * Decode the native Redis `TIME` reply into epoch milliseconds. With binary + * replies enabled, Redis returns two bulk strings: whole seconds and the + * microsecond offset within that second. Any malformed or unsafe value is a + * payload protocol error rather than an imprecise clock reading. + */ +export function decodeRedisServerTime(raw: unknown): number { + if (!Array.isArray(raw) || raw.length !== 2) { + throw invalidRedisTimeReply(); + } + const [rawSeconds, rawMicroseconds] = raw; + if (!Buffer.isBuffer(rawSeconds) || !Buffer.isBuffer(rawMicroseconds)) { + throw invalidRedisTimeReply(); + } + const secondsText = rawSeconds.toString("utf8"); + const microsecondsText = rawMicroseconds.toString("utf8"); + if (!/^[0-9]+$/.test(secondsText) || !/^[0-9]+$/.test(microsecondsText)) { + throw invalidRedisTimeReply(); + } + + const seconds = BigInt(secondsText); + const microseconds = BigInt(microsecondsText); + if (microseconds > 999_999n) { + throw invalidRedisTimeReply(); + } + const serverNowMs = seconds * 1_000n + microseconds / 1_000n; + if (serverNowMs > BigInt(Number.MAX_SAFE_INTEGER)) { + throw invalidRedisTimeReply(); + } + return Number(serverNowMs); +} + +/** Validate the semantic read age before an adapter dispatches any commands. */ +export function assertValidRedisMaxAgeMs(maxAgeMs: number): void { + if ( + !Number.isSafeInteger(maxAgeMs) + || maxAgeMs <= 0 + || maxAgeMs > MAX_SUPPORTED_DURATION_MS + ) { + throw new RangeError( + `DialCache Redis maxAgeMs must be a positive safe integer no greater than ${MAX_SUPPORTED_DURATION_MS}`, + ); + } +} + +/** Return whether a decoded frame is strictly younger than the requested age. */ +export function isRedisFrameWithinMaxAge( + frame: DecodedRedisFrame, + serverNowMs: number, + maxAgeMs: number, +): boolean { + assertValidRedisMaxAgeMs(maxAgeMs); + const ageMs = serverNowMs - frame.createdAtMs; + return ageMs >= 0 && ageMs < maxAgeMs; +} + +function invalidRedisTimeReply(): DialCacheRedisPayloadError { + return new DialCacheRedisPayloadError( + "Invalid DialCache Redis TIME reply; expected two unsigned decimal bulk strings", + ); +} + +function readFrameCreatedAtMs(frame: Buffer): number | null { + const createdAtMs = frame.readBigUInt64BE(REDIS_FRAME_TIMESTAMP_OFFSET); + return createdAtMs <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(createdAtMs) : null; } diff --git a/src/internal/redis-script-reply.ts b/src/internal/redis-script-reply.ts index b018aab..cfec6c7 100644 --- a/src/internal/redis-script-reply.ts +++ b/src/internal/redis-script-reply.ts @@ -36,6 +36,26 @@ export function resolveTrackedRedisWriteReply(reply: unknown): boolean { return stamp === 1; } +/** + * Resolve the untracked stamp's narrower reply domain: 1 is promoted and 2 + * means the paired placeholder disappeared. Untracked writes have no + * invalidation-fenced false outcome, so every other reply is a protocol + * violation. + */ +export function resolveUntrackedRedisWriteReply(reply: unknown): true { + if (reply === 1) { + return true; + } + if (reply === 2) { + throw new DialCacheRedisPlaceholderLostError( + "DialCache untracked write lost its placeholder before the stamp; the SET was rejected, overwritten, or expired", + ); + } + throw new DialCacheRedisProtocolError( + "Invalid DialCache Redis untracked write reply; expected integer 1 or 2", + ); +} + export function validateRedisScriptInvalidationReply(reply: unknown): 1 { if (reply !== 1) { throw new DialCacheRedisProtocolError("Invalid DialCache Redis invalidate reply; expected integer 1"); diff --git a/src/internal/redis-scripts.ts b/src/internal/redis-scripts.ts index 1fcfcd7..551e0ea 100644 --- a/src/internal/redis-scripts.ts +++ b/src/internal/redis-scripts.ts @@ -38,6 +38,20 @@ end`; const REDIS_TIME_LUA = String.raw`local redis_time = redis.call("TIME") local now_ms = tonumber(redis_time[1]) * 1000 + math.floor(tonumber(redis_time[2]) / 1000)`; +export const WRITE_UNTRACKED_STAMP_SCRIPT = [ + String.raw`if string.len(ARGV[1]) ~= ${REDIS_FRAME_TIMESTAMP_BYTES} then + return redis.error_reply("ERR invalid DialCache stamp nonce") +end`, + REDIS_TIME_LUA, + String.raw`if redis.call("GETRANGE", KEYS[1], 0, ${REDIS_FRAME_HEADER_BYTES - 1}) == string.char(${REDIS_FRAME_PLACEHOLDER_VERSION}) .. ARGV[1] then + redis.call("SETRANGE", KEYS[1], 0, string.char(${REDIS_FRAME_VERSION}) .. struct.pack(">I8", now_ms)) + return 1 +end + +-- Never promote a frame or placeholder this stamp does not own. +return 2`, +].join("\n\n"); + export const WRITE_TRACKED_STAMP_SCRIPT = [ PARSE_WATERMARK_LUA, CEIL_FINITE_NUMBER_LUA, diff --git a/src/internal/runtime-config.ts b/src/internal/runtime-config.ts index fdabb70..e3c4b1b 100644 --- a/src/internal/runtime-config.ts +++ b/src/internal/runtime-config.ts @@ -15,25 +15,43 @@ export interface ResolvedLayerConfig { readonly ramp: number; } -export type LayerConfigResolution = - | { readonly status: "enabled"; readonly config: ResolvedLayerConfig } +/** Remote-only policy resolved against the same invocation snapshot as its TTL. */ +export interface ResolvedRemoteLayerConfig extends ResolvedLayerConfig { + readonly staleOnErrorMaxAgeSec: number | null; +} + +export type LayerConfigResolution = + | { readonly status: "enabled"; readonly config: Config } | { readonly status: "disabled"; readonly reason: "ramped_down"; /** Valid policy retained even though its ramp excluded this key. */ - readonly config: ResolvedLayerConfig; + readonly config: Config; } | { readonly status: "disabled"; readonly reason: Exclude; }; +/** + * A malformed optional stale policy is diagnostic-only: the valid remote layer + * remains available with recovery disabled. + */ +export type RemoteLayerConfigResolution = LayerConfigResolution & { + readonly staleOnErrorConfigError?: true; +}; + interface ResolveLayerConfigOptions { readonly config: DialCacheKeyConfig | null; readonly key: DialCacheKey; readonly layer: CacheLayer; } +interface ResolveRemoteLayerConfigOptions { + readonly config: DialCacheKeyConfig | null; + readonly key: DialCacheKey; +} + export async function fetchKeyConfig( configProvider: CacheConfigProvider, key: DialCacheKey, @@ -91,6 +109,48 @@ export function resolveLayerConfigResult(options: ResolveLayerConfigOptions): La : { status: "disabled", reason: "ramped_down", config: { ttlSec, ramp } }; } +export function resolveRemoteLayerConfigResult( + options: ResolveRemoteLayerConfigOptions, +): RemoteLayerConfigResolution { + const resolution = resolveLayerConfigResult({ + ...options, + layer: CacheLayer.REMOTE, + }); + const configuredMaxAge: unknown = options.config?.staleOnErrorMaxAgeSec; + if (!("config" in resolution)) { + if ( + resolution.reason === "policy_disabled" + && configuredMaxAge !== undefined + && configuredMaxAge !== 0 + ) { + return { ...resolution, staleOnErrorConfigError: true }; + } + return resolution; + } + + if (configuredMaxAge === undefined || configuredMaxAge === 0) { + return { + ...resolution, + config: { ...resolution.config, staleOnErrorMaxAgeSec: null }, + }; + } + if ( + !isSupportedCacheTtlSec(configuredMaxAge) + || configuredMaxAge <= resolution.config.ttlSec + ) { + return { + ...resolution, + config: { ...resolution.config, staleOnErrorMaxAgeSec: null }, + staleOnErrorConfigError: true, + }; + } + + return { + ...resolution, + config: { ...resolution.config, staleOnErrorMaxAgeSec: configuredMaxAge }, + }; +} + function mergeKeyConfig( defaultConfig: DialCacheKeyConfig | null, runtimeConfig: DialCacheKeyConfig | null | undefined, @@ -112,6 +172,9 @@ function mergeKeyConfig( const remoteReadTimeoutMs = overlay?.remoteReadTimeoutMs !== undefined ? overlay.remoteReadTimeoutMs : defaultConfig?.remoteReadTimeoutMs; + const staleOnErrorMaxAgeSec = overlay?.staleOnErrorMaxAgeSec !== undefined + ? overlay.staleOnErrorMaxAgeSec + : defaultConfig?.staleOnErrorMaxAgeSec; const shadow = mergeShadowConfig(defaultConfig?.shadow, overlay?.shadow); return new DialCacheKeyConfig({ @@ -119,6 +182,7 @@ function mergeKeyConfig( ramp: mergeLayerConfig(defaultConfig?.ramp, overlay?.ramp, "ramp"), ...(requestLocal === undefined ? {} : { requestLocal }), ...(coalesce === undefined ? {} : { coalesce }), + ...(staleOnErrorMaxAgeSec === undefined ? {} : { staleOnErrorMaxAgeSec }), ...(remoteReadTimeoutMs === undefined ? {} : { remoteReadTimeoutMs }), ...(shadow === undefined ? {} : { shadow }), }); diff --git a/src/metrics.ts b/src/metrics.ts index 998469f..f22f334 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -25,6 +25,13 @@ export type ShadowValidationOutcome = | "confirmation_error" | "timeout" | "dropped"; +/** Bounded terminal outcomes for an attempted stale-on-error Redis recovery. */ +export type StaleRecoveryOutcome = + | "served" + | "miss" + | "read_error" + | "read_timeout" + | "deserialization_error"; /** * Bounded compression outcomes. Writes record compressed, below_threshold, * not_smaller, or write_over_limit (serialized form exceeds the decompression @@ -106,6 +113,13 @@ export interface ShadowValidationMetricLabels { readonly outcome: ShadowValidationOutcome; } +export interface StaleRecoveryMetricLabels { + readonly cacheNamespace: string; + readonly useCase: string; + readonly keyType: string; + readonly outcome: StaleRecoveryOutcome; +} + export interface DialCacheMetricsAdapter { request(labels: CacheMetricLabels): void; miss(labels: CacheMetricLabels): void; @@ -121,14 +135,16 @@ export interface DialCacheMetricsAdapter { * a mismatch, after the confirming re-read): the observing process's epoch * clock minus the validated frame's `createdAtMs`, clamped at zero. * Emitted only alongside terminal `match` and `mismatch` outcomes; other - * outcomes deliver no verdict on a retained value. Tracked frames are - * stamped with Redis server time and untracked frames with the writer's - * client clock, so the age mixes clocks and is coarse operational - * evidence, not a precise measurement. Optional so existing custom - * adapters keep compiling without changes. + * outcomes deliver no verdict on a retained value. Bundled adapters stamp + * both tracked and untracked frames with Redis server time. The observing + * process still computes this metric with its own clock, so the value is + * coarse operational evidence rather than a precise measurement. Optional + * so existing custom adapters keep compiling without changes. */ observeShadowValueAge?(labels: ShadowValidationMetricLabels, seconds: number): void; // Optional so existing custom adapters keep compiling without changes. + staleRecovery?(labels: StaleRecoveryMetricLabels): void; + // Optional so existing custom adapters keep compiling without changes. compression?(labels: CompressionMetricLabels): void; observeGet(labels: CacheMetricLabels, seconds: number): void; observeFallback(labels: CacheMetricLabels, seconds: number): void; diff --git a/src/node-redis.ts b/src/node-redis.ts index 4c80358..91f5b27 100644 --- a/src/node-redis.ts +++ b/src/node-redis.ts @@ -3,16 +3,20 @@ import { commandOptions, defineScript } from "redis"; import { INVALIDATE_CACHE_SCRIPT, WRITE_TRACKED_STAMP_SCRIPT, + WRITE_UNTRACKED_STAMP_SCRIPT, } from "./internal/redis-scripts.js"; import { + assertValidRedisMaxAgeMs, + decodeRedisServerTime, decodeRedisFrame, decodeTrackedRedisFrame, - encodeRedisFrame, encodeTrackedRedisPlaceholder, + isRedisFrameWithinMaxAge, } from "./internal/redis-payload.js"; import { ceilSupportedCacheTtlMs } from "./internal/duration.js"; import { resolveTrackedRedisWriteReply, + resolveUntrackedRedisWriteReply, validateRedisScriptInvalidationReply, validateRedisScriptWriteReply, validateRedisSetReply, @@ -32,6 +36,13 @@ type BufferReplyOptions = ReturnType< // Redis bulk strings are binary data; decoding them as UTF-8 would corrupt arbitrary serializer output. const bufferReplyOptions: BufferReplyOptions = commandOptions({ returnBuffers: true }); const writeReply = (reply: number): number => validateRedisScriptWriteReply(reply); +const untrackedWriteReply = (reply: number): number => { + if (reply !== 1 && reply !== 2) { + // Reuse the public resolver as the single source of the reply-domain error. + resolveUntrackedRedisWriteReply(reply); + } + return reply; +}; const invalidationReply = (reply: number): number => validateRedisScriptInvalidationReply(reply); type NodeRedisArgument = string | Buffer; @@ -55,13 +66,18 @@ function defineDialCacheScript, Reply>( /** * DialCache's client wiring, not a write API: the registered methods return - * raw script replies. `dialcacheWriteTrackedStamp` replies `0 | 1 | 2`, and - * `2` means the placeholder was lost — not success. Code invoking these - * methods directly must map stamp replies through - * `resolveTrackedRedisWriteReply` from `dialcache/redis-protocol`, which - * throws `DialCacheRedisPlaceholderLostError` on `2`. + * raw script replies. `dialcacheWriteUntrackedStamp` replies `1 | 2` and + * `dialcacheWriteTrackedStamp` replies `0 | 1 | 2`; `2` means the placeholder + * was lost — not success. Code invoking these methods directly must map stamp + * replies through the corresponding resolver from + * `dialcache/redis-protocol`, which throws + * `DialCacheRedisPlaceholderLostError` on `2`. */ export type DialCacheNodeRedisScripts = { + readonly dialcacheWriteUntrackedStamp: NodeRedisScript< + [valueKey: string, nonce: Buffer], + number + >; readonly dialcacheWriteTrackedStamp: NodeRedisScript< [valueKey: string, watermarkKey: string, cacheTtlMs: number, nonce: Buffer], number @@ -74,6 +90,16 @@ export type DialCacheNodeRedisScripts = { /** See {@link DialCacheNodeRedisScripts}: wiring for the adapter, not a direct write API. */ export const dialcacheRedisScripts: DialCacheNodeRedisScripts = { + dialcacheWriteUntrackedStamp: defineDialCacheScript({ + SCRIPT: WRITE_UNTRACKED_STAMP_SCRIPT, + NUMBER_OF_KEYS: 1, + FIRST_KEY_INDEX: 0, + IS_READ_ONLY: false, + transformArguments(valueKey: string, nonce: Buffer): Array { + return [valueKey, nonce]; + }, + transformReply: untrackedWriteReply, + }), dialcacheWriteTrackedStamp: defineDialCacheScript({ SCRIPT: WRITE_TRACKED_STAMP_SCRIPT, NUMBER_OF_KEYS: 2, @@ -102,6 +128,7 @@ export const dialcacheRedisScripts: DialCacheNodeRedisScripts = { }; interface NodeRedisWriteClient { + dialcacheWriteUntrackedStamp(valueKey: string, nonce: Buffer): Promise; dialcacheWriteTrackedStamp( valueKey: string, watermarkKey: string, @@ -112,7 +139,6 @@ interface NodeRedisWriteClient { } interface NodeRedisStandaloneClient extends NodeRedisWriteClient { - get(options: BufferReplyOptions, valueKey: string): Promise; sendCommand( args: Array, options: BufferReplyOptions, @@ -122,7 +148,6 @@ interface NodeRedisStandaloneClient extends NodeRedisWriteClient { interface NodeRedisClusterClient extends NodeRedisWriteClient { /** Public node-redis Cluster topology view, used only to distinguish its sendCommand overload. */ readonly masters: ReadonlyArray; - get(options: BufferReplyOptions, valueKey: string): Promise; sendCommand( firstKey: string, isReadonly: false, @@ -150,8 +175,9 @@ function validateRedisMGetReply(reply: unknown): [unknown, unknown] { } // Keyed commands route to the slot primary in cluster mode (isReadonly=false), -// so tracked reads observe the latest invalidation watermark even when the -// caller configured node-redis Cluster with useReplicas. +// keeping each native read and TIME on the same server. Tracked reads also +// observe the latest invalidation watermark even when the caller configured +// node-redis Cluster with useReplicas. function sendKeyedCommand( client: NodeRedisClient, firstKey: string, @@ -163,14 +189,29 @@ function sendKeyedCommand( : client.sendCommand(args, options); } -async function readTracked( +function readTracked( client: NodeRedisClient, options: BufferReplyOptions, valueKey: string, watermarkKey: string, -): Promise<[unknown, unknown]> { - const raw = await sendKeyedCommand(client, valueKey, ["MGET", valueKey, watermarkKey], options); - return validateRedisMGetReply(raw); +): Promise { + return sendKeyedCommand(client, valueKey, ["MGET", valueKey, watermarkKey], options); +} + +async function readFrameWithServerTime( + client: NodeRedisClient, + options: BufferReplyOptions, + valueKey: string, + watermarkKey?: string, +): Promise { + const readPromise: Promise = watermarkKey === undefined + ? sendKeyedCommand(client, valueKey, ["GET", valueKey], options) + : readTracked(client, options, valueKey, watermarkKey); + // Observe the read unconditionally so a synchronous throw while issuing + // TIME cannot leave the already-enqueued command's rejection unhandled. + readPromise.catch(() => undefined); + const timePromise = sendKeyedCommand(client, valueKey, ["TIME"], options); + return await Promise.all([readPromise, timePromise]); } function sendFrameSet( @@ -191,9 +232,10 @@ function sendFrameSet( * Create a resource-free semantic view over a caller-owned node-redis client. * Read signals are passed to node-redis so queued commands can be removed when * supported. Aborting after dispatch does not unsend a command or prove the - * server stopped executing it. Tracked writes enqueue their placeholder SET - * and stamp script in one synchronous tick, so node-redis pipelines them in - * order on one connection (per slot node in cluster mode). Invalidation + * server stopped executing it. Reads enqueue their native GET/MGET and TIME, + * and writes enqueue their placeholder SET and stamp script, in one + * synchronous tick so node-redis pipelines each pair in order on one + * connection (per slot node in cluster mode). Invalidation * retries any dispatch rejection other than a reply-domain violation once by * re-sending the script source as EVAL — the script is idempotent, so a * duplicate run is harmless — and a failed retry surfaces unmodified, with @@ -206,7 +248,8 @@ function sendFrameSet( */ export function createNodeRedisDialCacheClient(client: NodeRedisClient): DialCacheRedisClient { if ( - typeof client.dialcacheWriteTrackedStamp !== "function" + typeof client.dialcacheWriteUntrackedStamp !== "function" + || typeof client.dialcacheWriteTrackedStamp !== "function" || typeof client.dialcacheInvalidate !== "function" ) { throw new TypeError( @@ -214,30 +257,29 @@ export function createNodeRedisDialCacheClient(client: NodeRedisClient): DialCac ); } return { - async read({ valueKey, watermarkKey }, context) { + enforcesMaxAge: true, + async read({ valueKey, watermarkKey, maxAgeMs }, context) { + assertValidRedisMaxAgeMs(maxAgeMs); const options: BufferReplyOptions = context === undefined ? bufferReplyOptions : commandOptions({ returnBuffers: true, signal: context.signal }); - if (watermarkKey === undefined) { - return decodeRedisFrame(await client.get(options, valueKey)); - } - const [rawValue, rawWatermark] = await readTracked( + const [rawRead, rawTime] = await readFrameWithServerTime( client, options, valueKey, watermarkKey, ); - return decodeTrackedRedisFrame(rawValue, rawWatermark); + const frame = watermarkKey === undefined + ? decodeRedisFrame(rawRead) + : decodeTrackedRedisFrame(...validateRedisMGetReply(rawRead)); + const serverNowMs = decodeRedisServerTime(rawTime); + return frame !== null && isRedisFrameWithinMaxAge(frame, serverNowMs, maxAgeMs) + ? frame + : null; }, async write(request) { const { valueKey, watermarkKey, value } = request; const cacheTtlMs = ceilSupportedCacheTtlMs(request.cacheTtlMs); - if (watermarkKey === undefined) { - validateRedisSetReply( - await sendFrameSet(client, valueKey, encodeRedisFrame(value, Date.now()), cacheTtlMs), - ); - return true; - } const { frame, nonce } = encodeTrackedRedisPlaceholder(value); // Both commands must enqueue in this synchronous tick so they pipeline // in order; an await between them would allow reordering around them. @@ -245,7 +287,9 @@ export function createNodeRedisDialCacheClient(client: NodeRedisClient): DialCac // Observe the SET unconditionally so a synchronous throw before // allSettled cannot leave its rejection unhandled. setPromise.catch(() => undefined); - const stampPromise = client.dialcacheWriteTrackedStamp(valueKey, watermarkKey, cacheTtlMs, nonce); + const stampPromise = watermarkKey === undefined + ? client.dialcacheWriteUntrackedStamp(valueKey, nonce) + : client.dialcacheWriteTrackedStamp(valueKey, watermarkKey, cacheTtlMs, nonce); const [setResult, stampResult] = await Promise.allSettled([setPromise, stampPromise]); // A failed SET is the write outcome even when the stamp settled. if (setResult.status === "rejected") { @@ -255,7 +299,9 @@ export function createNodeRedisDialCacheClient(client: NodeRedisClient): DialCac if (stampResult.status === "rejected") { throw stampResult.reason; } - return resolveTrackedRedisWriteReply(stampResult.value); + return watermarkKey === undefined + ? resolveUntrackedRedisWriteReply(stampResult.value) + : resolveTrackedRedisWriteReply(stampResult.value); }, async invalidate({ watermarkKey, futureBufferMs }) { let raw: unknown; diff --git a/src/prometheus.ts b/src/prometheus.ts index 7674675..0a4d55c 100644 --- a/src/prometheus.ts +++ b/src/prometheus.ts @@ -11,6 +11,7 @@ import type { InvalidationMetricLabels, SerializationMetricLabels, ShadowValidationMetricLabels, + StaleRecoveryMetricLabels, } from "./metrics.js"; export interface PrometheusMetricsOptions { @@ -27,6 +28,7 @@ type SerializationLabels = CounterLabels | "operation"; type InvalidationLabels = "cache_namespace" | "key_type" | "layer"; type CoalescedLabels = "cache_namespace" | "use_case" | "key_type" | "scope"; type ShadowValidationLabels = "cache_namespace" | "use_case" | "key_type" | "outcome"; +type StaleRecoveryLabels = "cache_namespace" | "use_case" | "key_type" | "outcome"; type CompressionLabels = CounterLabels | "outcome"; interface BaseCollectorConfig { @@ -70,6 +72,7 @@ export class PrometheusDialCacheMetrics implements DialCacheMetricsAdapter { private readonly coalescedCounter: Counter; private readonly shadowValidationCounter: Counter; private readonly shadowValueAgeHistogram: Histogram; + private readonly staleRecoveryCounter: Counter; private readonly compressionCounter: Counter; private readonly getTimer: Histogram; private readonly fallbackTimer: Histogram; @@ -93,6 +96,7 @@ export class PrometheusDialCacheMetrics implements DialCacheMetricsAdapter { this.coalescedCounter = counter(registry, collectors.coalescedCounter); this.shadowValidationCounter = counter(registry, collectors.shadowValidationCounter); this.shadowValueAgeHistogram = histogram(registry, collectors.shadowValueAgeHistogram); + this.staleRecoveryCounter = counter(registry, collectors.staleRecoveryCounter); this.compressionCounter = counter(registry, collectors.compressionCounter); this.getTimer = histogram(registry, collectors.getTimer); this.fallbackTimer = histogram(registry, collectors.fallbackTimer); @@ -148,6 +152,15 @@ export class PrometheusDialCacheMetrics implements DialCacheMetricsAdapter { this.shadowValueAgeHistogram.observe(shadowValidationLabels(labels), seconds); } + staleRecovery(labels: StaleRecoveryMetricLabels): void { + this.staleRecoveryCounter.inc({ + cache_namespace: labels.cacheNamespace, + use_case: labels.useCase, + key_type: labels.keyType, + outcome: labels.outcome, + }); + } + compression(labels: CompressionMetricLabels): void { this.compressionCounter.inc({ ...cacheLabels(labels), outcome: labels.outcome }); } @@ -254,6 +267,12 @@ function collectorConfigs(prefix: string) { labelNames: ["cache_namespace", "use_case", "key_type", "outcome"], buckets: VALUE_AGE_BUCKETS, }, + staleRecoveryCounter: { + type: "counter", + name: `${prefix}dialcache_stale_recovery_counter`, + help: "DialCache stale-on-error Redis recovery outcomes.", + labelNames: ["cache_namespace", "use_case", "key_type", "outcome"], + }, compressionCounter: { type: "counter", name: `${prefix}dialcache_compression_counter`, diff --git a/src/redis-client.ts b/src/redis-client.ts index 1b58f3d..852e270 100644 --- a/src/redis-client.ts +++ b/src/redis-client.ts @@ -60,11 +60,13 @@ export class DialCacheRedisProtocolError extends Error { } /** - * A tracked write's stamp found no placeholder carrying its nonce: the paired + * A write's stamp found no placeholder carrying its nonce: the paired * SET was rejected, overwritten by a concurrent writer, expired, or removed - * by a fenced write. The value was not published, and DialCache suppresses the - * corresponding process-local publication. Same-key write contention produces - * a benign floor of these, concentrated on hot keys at TTL expiry. + * by a fenced write. The value was not published. For tracked writes, + * DialCache also suppresses the corresponding process-local publication; + * untracked writes retain their existing fail-open local-fill behavior. + * Same-key write contention produces a benign floor of these, concentrated on + * hot keys at TTL expiry. */ export class DialCacheRedisPlaceholderLostError extends Error { static [Symbol.hasInstance](value: unknown): boolean { @@ -91,12 +93,11 @@ export type RedisCachePayload = string | Buffer; * A served Redis frame: the payload bytes past the frame header plus the * header's creation time. The payload is the serializer output, possibly * still wrapped in a compression envelope that DialCache core interprets - * above the adapter (see the `dialcache/redis-protocol` module doc). Tracked - * frames carry Redis server time written by the stamp script; untracked - * frames carry the writer's client clock. DialCache consumes `createdAtMs` - * only for observability (the shadow value-age observation) — tracked - * watermark fencing already happened inside the decoder — so it never - * affects serving decisions. + * above the adapter (see the `dialcache/redis-protocol` module doc). Frames + * carry Redis server time written by the stamp scripts. Adapters use + * `createdAtMs` to enforce the requested logical maximum age, and DialCache + * also consumes it for the shadow value-age observation. Tracked watermark + * fencing already happened inside the decoder. */ export interface DecodedRedisFrame { readonly payload: RedisCachePayload; @@ -108,11 +109,16 @@ interface RedisValueRequest { readonly valueKey: string; } -interface TrackedRedisValueRequest extends RedisValueRequest { +interface RedisReadBase extends RedisValueRequest { + /** Positive integer no greater than 31,536,000,000 (365 days). */ + readonly maxAgeMs: number; +} + +interface TrackedRedisValueRequest extends RedisReadBase { readonly watermarkKey: string; } -interface UntrackedRedisValueRequest extends RedisValueRequest { +interface UntrackedRedisValueRequest extends RedisReadBase { readonly watermarkKey?: never; } @@ -133,8 +139,8 @@ interface RedisWriteBase extends RedisValueRequest { readonly value: RedisCachePayload; } -type TrackedRedisWriteRequest = RedisWriteBase & TrackedRedisValueRequest; -type UntrackedRedisWriteRequest = RedisWriteBase & UntrackedRedisValueRequest; +type TrackedRedisWriteRequest = RedisWriteBase & { readonly watermarkKey: string }; +type UntrackedRedisWriteRequest = RedisWriteBase & { readonly watermarkKey?: never }; export type RedisWriteRequest = TrackedRedisWriteRequest | UntrackedRedisWriteRequest; @@ -161,10 +167,19 @@ export interface RedisInvalidationRequest { */ export interface DialCacheRedisClient { /** - * Read a DialCache Redis frame and return its decoded serializer payload - * together with the frame header's creation time. Implementations must use - * `decodeRedisFrame` / `decodeTrackedRedisFrame` from - * `dialcache/redis-protocol`, or preserve their exact behavior. + * Safety capability marker. Custom clients must explicitly attest that every + * read enforces `RedisReadRequest.maxAgeMs` using Redis server time. + * DialCache also checks this marker at runtime for JavaScript clients. + */ + readonly enforcesMaxAge: true; + /** + * Read a DialCache Redis frame whose Redis-server age is strictly less than + * `maxAgeMs`, returning its decoded serializer payload together with the + * frame header's creation time. Implementations must validate the request + * with `assertValidRedisMaxAgeMs` before dispatch and use + * `decodeRedisFrame` / `decodeTrackedRedisFrame`, `decodeRedisServerTime`, + * and `isRedisFrameWithinMaxAge` from `dialcache/redis-protocol`, or preserve + * their exact behavior. * * Raw values are Redis bulk strings (`Buffer`) or null. A missing value, a * frame shorter than the version/timestamp/encoding header, or an @@ -174,8 +189,9 @@ export interface DialCacheRedisClient { * `createdAt <= watermark` is fenced. Unsupported payload encodings and * non-bulk runtime replies are payload protocol errors rather than misses. * - * Tracked implementations must read the value and watermark atomically from - * one authoritative snapshot; replica lag must not hide an invalidation. + * Implementations must compare against Redis server time. Tracked + * implementations must read the value and watermark atomically from one + * authoritative snapshot; replica lag must not hide an invalidation. * * A non-null frame is transferred to DialCache. A returned Buffer payload * must remain stable and must not be mutated, pooled, or reused after this @@ -188,12 +204,12 @@ export interface DialCacheRedisClient { * Write a DialCache Redis frame using the `dialcache/redis-protocol` * encoders, or preserve their exact behavior. * - * Untracked writes are one native `SET valueKey frame PX cacheTtlMs` whose - * frame comes from `encodeRedisFrame` with a client-clock `createdAtMs`. - * Untracked reads never consult that stamp for serving or miss decisions, - * but they do surface it as the decoded frame's `createdAtMs`, where it - * feeds the shadow value-age observation — so untracked writers must stamp - * real client time, not a constant. + * Untracked writes issue two commands ordered on one connection without a + * transaction: a native `SET` of an `encodeTrackedRedisPlaceholder` frame, + * followed by `WRITE_UNTRACKED_STAMP_SCRIPT` with `KEYS = [valueKey]` and + * `ARGV = [nonce]`. The script promotes exactly that placeholder to a + * served frame with Redis-server `createdAt` (reply 1), or reports the + * placeholder gone (reply 2). * * Tracked writes issue two commands ordered on one connection without a * transaction: a native `SET` of an `encodeTrackedRedisPlaceholder` frame, @@ -213,7 +229,7 @@ export interface DialCacheRedisClient { * placeholder remains subject to the invalidation future buffer, like any * in-flight write. * - * Implementations must not reorder the pair, must mint one placeholder per + * Implementations must not reorder either pair, must mint one placeholder per * logical write so client-level retries stay paired with their stamp, and * must surface a SET failure as the write error even when the stamp settled * (in that case the stamp may have promoted the landed SET, leaving the diff --git a/src/redis-protocol.ts b/src/redis-protocol.ts index 10642c0..3554223 100644 --- a/src/redis-protocol.ts +++ b/src/redis-protocol.ts @@ -1,13 +1,13 @@ /** * Public frame protocol surface for adapter authors and out-of-band tooling. * - * These exports encode frames and mint tracked placeholders (use + * These exports encode frames and mint write placeholders (use * `encodeRedisFrame` and `encodeTrackedRedisPlaceholder` rather than * reimplementing them — see the latter's JSDoc for the nonce contract), * decode a frame into its payload bytes and header creation time, resolve * and validate mutation replies, guard the write-TTL acceptance domain, and - * carry the tracked stamp and invalidation Lua sources the bundled adapters - * dispatch. The + * carry both stamp sources and the invalidation Lua source the bundled + * adapters dispatch. The * payload region past the header is opaque at this layer: entries written by * DialCache releases with payload compression may begin with a compression * envelope byte (0x00 escape, 0x01/0x02 zstd; see the README Compression @@ -18,17 +18,22 @@ export { ceilSupportedCacheTtlMs } from "./internal/duration.js"; export { INVALIDATE_CACHE_SCRIPT, WRITE_TRACKED_STAMP_SCRIPT, + WRITE_UNTRACKED_STAMP_SCRIPT, } from "./internal/redis-scripts.js"; export { + assertValidRedisMaxAgeMs, + decodeRedisServerTime, decodeRedisFrame, decodeTrackedRedisFrame, encodeRedisFrame, encodeTrackedRedisPlaceholder, + isRedisFrameWithinMaxAge, type TrackedRedisPlaceholder, } from "./internal/redis-payload.js"; export type { DecodedRedisFrame } from "./redis-client.js"; export { resolveTrackedRedisWriteReply, + resolveUntrackedRedisWriteReply, validateRedisScriptInvalidationReply, validateRedisSetReply, } from "./internal/redis-script-reply.js"; diff --git a/src/valkey-glide.ts b/src/valkey-glide.ts index 7b33915..19c01ce 100644 --- a/src/valkey-glide.ts +++ b/src/valkey-glide.ts @@ -2,17 +2,21 @@ import { createHash } from "node:crypto"; import { ceilSupportedCacheTtlMs } from "./internal/duration.js"; import { + assertValidRedisMaxAgeMs, decodeRedisFrame, + decodeRedisServerTime, decodeTrackedRedisFrame, - encodeRedisFrame, encodeTrackedRedisPlaceholder, + isRedisFrameWithinMaxAge, } from "./internal/redis-payload.js"; import { INVALIDATE_CACHE_SCRIPT, WRITE_TRACKED_STAMP_SCRIPT, + WRITE_UNTRACKED_STAMP_SCRIPT, } from "./internal/redis-scripts.js"; import { resolveTrackedRedisWriteReply, + resolveUntrackedRedisWriteReply, validateRedisScriptInvalidationReply, validateRedisSetReply, } from "./internal/redis-script-reply.js"; @@ -24,6 +28,9 @@ type ValkeyGlideString = string | Buffer; // definition the ones the EVALSHA dispatches must use and the ones the EVAL // recoveries repopulate. const WRITE_TRACKED_STAMP_SHA1 = createHash("sha1").update(WRITE_TRACKED_STAMP_SCRIPT).digest("hex"); +const WRITE_UNTRACKED_STAMP_SHA1 = createHash("sha1") + .update(WRITE_UNTRACKED_STAMP_SCRIPT) + .digest("hex"); const INVALIDATE_CACHE_SHA1 = createHash("sha1").update(INVALIDATE_CACHE_SCRIPT).digest("hex"); // Matches the server's raw NOSCRIPT reply and GLIDE's mapped NoScriptError @@ -34,6 +41,7 @@ function isNoScriptError(error: Error): boolean { interface ValkeyGlideBatch { customCommand(args: ValkeyGlideString[]): ValkeyGlideBatch; + get(key: ValkeyGlideString): ValkeyGlideBatch; mget(keys: ValkeyGlideString[]): ValkeyGlideBatch; } @@ -45,10 +53,6 @@ export interface ValkeyGlideScriptingClient { route?: { type: "primarySlotKey"; key: string }; }, ): Promise; - get( - key: ValkeyGlideString, - options: { decoder: TDecoder }, - ): Promise; exec( batch: ValkeyGlideBatch, raiseOnError: boolean, @@ -126,21 +130,22 @@ function classifyValkeyGlideClient( * wrappers should implement DialCacheRedisClient directly. A request * timeout bounds client waiting but is not server-side command cancellation. * GLIDE's current command API has no per-invocation signal, so DialCache's core - * read deadline may return before this adapter's invocation settles. Tracked - * standalone reads use a one-command primary batch, while tracked cluster - * reads route MGET explicitly to the slot primary, so replica lag cannot hide - * an invalidation watermark. Both mutation scripts dispatch as EVALSHA by - * their source SHA1 and recover a flushed script cache by re-sending the - * source as EVAL — which the server caches under that same SHA1 — so the - * first mutation against a cold script cache pays one extra round trip. - * Tracked writes batch a native placeholder SET with the stamp EVALSHA; - * cluster write batches route to the slot primary. Batches are deliberately - * non-atomic: MGET and SET are atomic themselves, an interleaved stamp is - * safe by design, and MULTI/EXEC would consume caller-owned WATCH state. - * Recovery differs by script: the stamp is retried only on NOSCRIPT, while - * invalidation retries any rejection once with EVAL by source. When that - * retry also fails, the original rejection is attached as the retry error's - * `cause` unless it already carries one. + * read deadline may return before this adapter's invocation settles. Reads + * batch a native GET or atomic MGET followed by TIME on one connection, and + * cluster reads route that batch to the value's slot primary. That preserves + * invalidation fencing and gives logical-age decisions an authoritative + * server-clock snapshot ordered after the value snapshot. Stamp scripts + * dispatch as EVALSHA by their source SHA1 and recover a flushed script cache + * only after NOSCRIPT by re-sending the source as EVAL — which the server + * caches under that same SHA1 — so the first mutation against a cold script + * cache pays one extra round trip. Both tracked and untracked writes batch a + * native placeholder SET with their stamp EVALSHA; cluster write batches + * route to the slot primary. Batches are deliberately non-atomic: GET/MGET + * and SET are atomic themselves, interleaving around a stamp is safe by + * design, and MULTI/EXEC would consume caller-owned WATCH state. Invalidation + * retries any rejection once with EVAL by source. When that retry also fails, + * the original rejection is attached as the retry error's `cause` unless it + * already carries one. */ export function createValkeyGlideDialCacheClient( client: ValkeyGlideScriptingClient, @@ -162,30 +167,34 @@ export function createValkeyGlideDialCacheClient( : { decoder: glide.Decoder.Bytes }; return { - async read({ valueKey, watermarkKey }) { + enforcesMaxAge: true, + async read({ valueKey, watermarkKey, maxAgeMs }) { + assertValidRedisMaxAgeMs(maxAgeMs); + const batch = isCluster ? new glide.ClusterBatch(false) : new glide.Batch(false); if (watermarkKey === undefined) { - const raw = await client.get(valueKey, { decoder: glide.Decoder.Bytes }); - return decodeRedisFrame(raw); + batch.get(valueKey); + } else { + batch.mget([valueKey, watermarkKey]); } - - let pair: unknown; - if (isCluster) { - pair = await client.customCommand( - ["MGET", valueKey, watermarkKey], - keyedOptions(valueKey), - ); + batch.customCommand(["TIME"]); + const raw = await client.exec(batch, true, keyedOptions(valueKey)); + if (!Array.isArray(raw) || raw.length !== 2) { + throw new DialCacheRedisPayloadError("Invalid DialCache Redis payload reply"); + } + const [rawValue, rawTime] = raw; + let frame; + if (watermarkKey === undefined) { + frame = decodeRedisFrame(rawValue); } else { - const batch = new glide.Batch(false).mget([valueKey, watermarkKey]); - const raw = await client.exec(batch, true, { decoder: glide.Decoder.Bytes }); - if (!Array.isArray(raw) || raw.length !== 1) { + if (!Array.isArray(rawValue) || rawValue.length !== 2) { throw new DialCacheRedisPayloadError("Invalid DialCache Redis payload reply"); } - pair = raw[0]; + frame = decodeTrackedRedisFrame(rawValue[0], rawValue[1]); } - if (!Array.isArray(pair) || pair.length !== 2) { - throw new DialCacheRedisPayloadError("Invalid DialCache Redis payload reply"); - } - return decodeTrackedRedisFrame(pair[0], pair[1]); + const serverNowMs = decodeRedisServerTime(rawTime); + return frame !== null && isRedisFrameWithinMaxAge(frame, serverNowMs, maxAgeMs) + ? frame + : null; }, async write(request) { const { valueKey, watermarkKey, value } = request; @@ -193,11 +202,37 @@ export function createValkeyGlideDialCacheClient( const execOptions = keyedOptions(valueKey); if (watermarkKey === undefined) { - const frame = encodeRedisFrame(value, Date.now()); - validateRedisSetReply( - await client.customCommand(["SET", valueKey, frame, "PX", String(cacheTtlMs)], execOptions), - ); - return true; + const { frame, nonce } = encodeTrackedRedisPlaceholder(value); + const batch = (isCluster ? new glide.ClusterBatch(false) : new glide.Batch(false)) + .customCommand(["SET", valueKey, frame, "PX", String(cacheTtlMs)]) + .customCommand([ + "EVALSHA", + WRITE_UNTRACKED_STAMP_SHA1, + "1", + valueKey, + nonce, + ]); + const replies = await client.exec(batch, false, execOptions); + if (!Array.isArray(replies) || replies.length !== 2) { + throw new DialCacheRedisPayloadError("Invalid DialCache Redis write reply"); + } + const [setReply, rawStamp] = replies as [unknown, unknown]; + // A failed SET is the write outcome even when the stamp settled. + if (setReply instanceof Error) { + throw setReply; + } + validateRedisSetReply(setReply); + let stampReply: unknown = rawStamp; + if (rawStamp instanceof Error) { + if (!isNoScriptError(rawStamp)) { + throw rawStamp; + } + stampReply = await client.customCommand( + ["EVAL", WRITE_UNTRACKED_STAMP_SCRIPT, "1", valueKey, nonce], + execOptions, + ); + } + return resolveUntrackedRedisWriteReply(stampReply); } const { frame, nonce } = encodeTrackedRedisPlaceholder(value); diff --git a/test/datadog.test.ts b/test/datadog.test.ts index fbb6b93..2c1ad5f 100644 --- a/test/datadog.test.ts +++ b/test/datadog.test.ts @@ -11,6 +11,7 @@ import { type MetricErrorKind, type MetricLayer, type ShadowValidationOutcome, + type StaleRecoveryOutcome, } from "../src/index.js"; import { NO_CACHE_LAYER, @@ -120,6 +121,14 @@ const SHADOW_VALIDATION_OUTCOMES: Readonly dropped: true, }; const shadowValidationOutcomes = Object.keys(SHADOW_VALIDATION_OUTCOMES) as ShadowValidationOutcome[]; +const STALE_RECOVERY_OUTCOMES: Readonly> = { + served: true, + miss: true, + read_error: true, + read_timeout: true, + deserialization_error: true, +}; +const staleRecoveryOutcomes = Object.keys(STALE_RECOVERY_OUTCOMES) as StaleRecoveryOutcome[]; const COMPRESSION_OUTCOMES: Readonly> = { compressed: true, below_threshold: true, @@ -160,6 +169,12 @@ describe("Datadog metrics adapter", () => { keyType: "user_id", outcome: "match", }); + metrics.staleRecovery({ + cacheNamespace: cacheLabels.cacheNamespace, + useCase: "LoadUser", + keyType: "user_id", + outcome: "served", + }); metrics.observeShadowValueAge( { cacheNamespace: cacheLabels.cacheNamespace, @@ -212,6 +227,12 @@ describe("Datadog metrics adapter", () => { value: 1, tags: { cache_namespace: "users", use_case: "LoadUser", key_type: "user_id", outcome: "match" }, }, + { + method: "increment", + name: "dialcache.stale_recovery.count", + value: 1, + tags: { cache_namespace: "users", use_case: "LoadUser", key_type: "user_id", outcome: "served" }, + }, { method: "distribution", name: "dialcache.shadow.value_age", @@ -318,6 +339,14 @@ describe("Datadog metrics adapter", () => { outcome, }); } + for (const outcome of staleRecoveryOutcomes) { + metrics.staleRecovery({ + cacheNamespace: cacheLabels.cacheNamespace, + useCase: cacheLabels.useCase, + keyType: cacheLabels.keyType, + outcome, + }); + } for (const outcome of compressionOutcomes) { metrics.compression({ ...cacheLabels, outcome }); } @@ -355,6 +384,18 @@ describe("Datadog metrics adapter", () => { outcome, })), ); + expect( + client.calls + .filter(({ name }) => name === "dialcache.stale_recovery.count") + .map(({ tags }) => tags), + ).toEqual( + staleRecoveryOutcomes.map((outcome) => ({ + cache_namespace: cacheLabels.cacheNamespace, + use_case: cacheLabels.useCase, + key_type: cacheLabels.keyType, + outcome, + })), + ); expect( client.calls .filter(({ name }) => name === "dialcache.compression.count") @@ -455,6 +496,7 @@ describe("Datadog metrics adapter", () => { const rawErrorMessage = "Redis failed for a private cache key"; let redisValueKey = ""; const redis: DialCacheRedisClient = { + enforcesMaxAge: true, read: async ({ valueKey }) => { redisValueKey = valueKey; const error = new Error(`${rawErrorMessage}: ${valueKey}`); diff --git a/test/dialcache-config-ramp.test.ts b/test/dialcache-config-ramp.test.ts index eb0d8ae..c7edb85 100644 --- a/test/dialcache-config-ramp.test.ts +++ b/test/dialcache-config-ramp.test.ts @@ -9,6 +9,7 @@ import { type Serializer, } from "../src/index.js"; import { deterministicRampSample, deterministicShadowRampSample } from "../src/internal/ramp.js"; +import { fetchKeyConfig } from "../src/internal/runtime-config.js"; import { FakeRedis } from "./fake-redis.js"; const configFor = (ttlSec: Partial>, ramp: Partial>) => @@ -44,6 +45,12 @@ describe("DialCache runtime config and ramp controls", () => { expect(new DialCacheKeyConfig({ coalesce: true }).coalesce).toBe(true); }); + it("preserves stale-on-error omission and explicit disable values", () => { + expect(new DialCacheKeyConfig({}).staleOnErrorMaxAgeSec).toBeUndefined(); + expect(new DialCacheKeyConfig({ staleOnErrorMaxAgeSec: 0 }).staleOnErrorMaxAgeSec).toBe(0); + expect(new DialCacheKeyConfig({ staleOnErrorMaxAgeSec: 3_600 }).staleOnErrorMaxAgeSec).toBe(3_600); + }); + it("preserves shadow omission and explicit kill-switch values", () => { expect(new DialCacheKeyConfig({}).shadow).toBeUndefined(); expect(new DialCacheKeyConfig({ shadow: {} }).shadow).toEqual({}); @@ -77,13 +84,14 @@ describe("DialCache runtime config and ramp controls", () => { it("captures an immutable default policy snapshot when the use case is registered", async () => { const suppliedDefault = new DialCacheKeyConfig({ - ttlSec: { [CacheLayer.LOCAL]: 60 }, - ramp: { [CacheLayer.LOCAL]: 100 }, + ttlSec: { [CacheLayer.LOCAL]: 60, [CacheLayer.REMOTE]: 60 }, + ramp: { [CacheLayer.LOCAL]: 100, [CacheLayer.REMOTE]: 100 }, shadow: { ramp: 25, logMismatches: true, }, coalesce: false, + staleOnErrorMaxAgeSec: 3_600, }); const observedDefaults: Array = []; const dialcache = new DialCache({ @@ -103,6 +111,7 @@ describe("DialCache runtime config and ramp controls", () => { suppliedDefault.ttlSec[CacheLayer.LOCAL] = 0; suppliedDefault.ramp[CacheLayer.LOCAL] = 0; + (suppliedDefault as { staleOnErrorMaxAgeSec?: number }).staleOnErrorMaxAgeSec = 0; const mutableShadow = suppliedDefault.shadow as { ramp?: number; logMismatches?: boolean; @@ -119,6 +128,7 @@ describe("DialCache runtime config and ramp controls", () => { expect(observedDefaults[1]).toBe(observedDefaults[0]); expect(observedDefaults[0]?.ttlSec[CacheLayer.LOCAL]).toBe(60); expect(observedDefaults[0]?.ramp[CacheLayer.LOCAL]).toBe(100); + expect(observedDefaults[0]?.staleOnErrorMaxAgeSec).toBe(3_600); expect(observedDefaults[0]?.shadow).toEqual({ ramp: 25, logMismatches: true, @@ -360,6 +370,75 @@ describe("DialCache runtime config and ramp controls", () => { RangeError, `no greater than ${MAX_CACHE_TTL_SEC}`, ], + [ + "negative stale-on-error max age", + new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 60 }, + staleOnErrorMaxAgeSec: -1, + }), + RangeError, + "nonnegative safe integer", + ], + [ + "fractional stale-on-error max age", + new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 60 }, + staleOnErrorMaxAgeSec: 60.5, + }), + RangeError, + "nonnegative safe integer", + ], + [ + "non-finite stale-on-error max age", + new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 60 }, + staleOnErrorMaxAgeSec: Number.NaN, + }), + RangeError, + "nonnegative safe integer", + ], + [ + "unsafe stale-on-error max age", + new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 60 }, + staleOnErrorMaxAgeSec: Number.MAX_SAFE_INTEGER + 1, + }), + RangeError, + "nonnegative safe integer", + ], + [ + "over-maximum stale-on-error max age", + new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 60 }, + staleOnErrorMaxAgeSec: MAX_CACHE_TTL_SEC + 1, + }), + RangeError, + `no greater than ${MAX_CACHE_TTL_SEC}`, + ], + [ + "stale-on-error max age equal to the remote TTL", + new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 60 }, + staleOnErrorMaxAgeSec: 60, + }), + RangeError, + "must be greater than ttlSec.remote", + ], + [ + "positive stale-on-error max age without a remote TTL", + new DialCacheKeyConfig({ staleOnErrorMaxAgeSec: 3_600 }), + RangeError, + "requires ttlSec.remote", + ], + [ + "stale-on-error max age below the remote TTL", + new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 60 }, + staleOnErrorMaxAgeSec: 30, + }), + RangeError, + "must be greater than ttlSec.remote", + ], ["negative ramp", new DialCacheKeyConfig({ ramp: { [CacheLayer.LOCAL]: -1 } }), RangeError, "between 0 and 100"], ["over-100 ramp", new DialCacheKeyConfig({ ramp: { [CacheLayer.LOCAL]: 101 } }), RangeError, "between 0 and 100"], ["non-finite ramp", new DialCacheKeyConfig({ ramp: { [CacheLayer.LOCAL]: Number.POSITIVE_INFINITY } }), RangeError, "between 0 and 100"], @@ -405,6 +484,12 @@ describe("DialCache runtime config and ramp controls", () => { TypeError, "must be a boolean", ], + [ + "wrong-type stale-on-error max age", + new DialCacheKeyConfig({ staleOnErrorMaxAgeSec: "3600" as unknown as number }), + TypeError, + "must be a number", + ], ["primitive config", 42 as unknown as DialCacheKeyConfig, TypeError, "must be an object"], ["array config", [] as unknown as DialCacheKeyConfig, TypeError, "must be an object"], [ @@ -457,6 +542,35 @@ describe("DialCache runtime config and ramp controls", () => { })).not.toThrow(); }); + it("accepts disabled and exact-maximum static stale-on-error policy", () => { + const dialcache = new DialCache(); + + expect(() => dialcache.cached(async () => "value", { + keyType: "item_id", + useCase: "DisabledStaticStaleOnErrorWithoutRemote", + cacheKey: () => "000", + defaultConfig: new DialCacheKeyConfig({ staleOnErrorMaxAgeSec: 0 }), + })).not.toThrow(); + expect(() => dialcache.cached(async () => "value", { + keyType: "item_id", + useCase: "DisabledStaticStaleOnError", + cacheKey: () => "123", + defaultConfig: new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 60 }, + staleOnErrorMaxAgeSec: 0, + }), + })).not.toThrow(); + expect(() => dialcache.cached(async () => "value", { + keyType: "item_id", + useCase: "MaximumStaticStaleOnError", + cacheKey: () => "456", + defaultConfig: new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: MAX_CACHE_TTL_SEC - 1 }, + staleOnErrorMaxAgeSec: MAX_CACHE_TTL_SEC, + }), + })).not.toThrow(); + }); + it.each([ ["a primitive", 42], ["an array", []], @@ -487,6 +601,7 @@ describe("DialCache runtime config and ramp controls", () => { it("returns the explicit kill-switch overlay from DialCacheKeyConfig.disabled()", () => { expect(DialCacheKeyConfig.disabled()).toEqual(new DialCacheKeyConfig({ requestLocal: false, + staleOnErrorMaxAgeSec: 0, shadow: { ramp: 0, logMismatches: false, @@ -575,6 +690,135 @@ describe("DialCache runtime config and ramp controls", () => { expect(second.calls).toBe(2); }); + it.each([ + ["null", null], + ["negative", -1], + ["fractional", 60.5], + ["NaN", Number.NaN], + ["infinite", Number.POSITIVE_INFINITY], + ["far over maximum", Number.MAX_SAFE_INTEGER], + ["unsafe", Number.MAX_SAFE_INTEGER + 1], + ["over maximum", MAX_CACHE_TTL_SEC + 1], + ["equal to fresh TTL", 60], + ["below fresh TTL", 30], + ["wrong type", "3600"], + ] as const)( + "disables only stale recovery for an invalid runtime max age ($0)", + async (_name, configuredMaxAge) => { + const redis = new FakeRedis(); + const dialcache = new DialCache({ + redis: { client: redis, readTimeoutMs: 1_000 }, + cacheConfigProvider: async () => new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 60 }, + staleOnErrorMaxAgeSec: configuredMaxAge as unknown as number, + }), + }); + let calls = 0; + const getUser = dialcache.cached(async (userId: string) => ({ userId, calls: ++calls }), { + keyType: "user_id", + useCase: `InvalidRuntimeStaleMaxAge${String(_name)}`, + cacheKey: (userId) => userId, + }); + + const first = await dialcache.enable(async () => await getUser("123")); + const second = await dialcache.enable(async () => await getUser("123")); + + expect(second).toEqual(first); + expect(calls).toBe(1); + expect(redis.getCalls).toBe(2); + expect(redis.setCalls).toBe(1); + }, + ); + + it("inherits, overrides, and explicitly disables stale-on-error runtime policy", async () => { + const defaultConfig = new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 60 }, + staleOnErrorMaxAgeSec: 3_600, + }); + const key = new DialCacheKey({ + keyType: "user_id", + id: "123", + useCase: "StaleOnErrorOverlay", + defaultConfig, + }); + + await expect(fetchKeyConfig(async () => new DialCacheKeyConfig({}), key)).resolves.toMatchObject({ + staleOnErrorMaxAgeSec: 3_600, + }); + await expect( + fetchKeyConfig(async () => new DialCacheKeyConfig({ staleOnErrorMaxAgeSec: 7_200 }), key), + ).resolves.toMatchObject({ staleOnErrorMaxAgeSec: 7_200 }); + await expect( + fetchKeyConfig(async () => new DialCacheKeyConfig({ staleOnErrorMaxAgeSec: 0 }), key), + ).resolves.toMatchObject({ staleOnErrorMaxAgeSec: 0 }); + }); + + it.each([ + [ + "an explicit stale-on-error zero", + () => new DialCacheKeyConfig({ staleOnErrorMaxAgeSec: 0 }), + 1, + ], + [ + "an invalid stale-on-error maximum", + () => new DialCacheKeyConfig({ staleOnErrorMaxAgeSec: 1 }), + 1, + ], + ["the complete disabled overlay", () => DialCacheKeyConfig.disabled(), 0], + ] as const)( + "does not recover a retained stale value after $0", + async (_name, disabledOverlay, expectedRemoteReads) => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-02T12:00:00.000Z")); + try { + const useCase = `RetainedStaleRuntimeDisable${expectedRemoteReads}`; + const redis = new FakeRedis(); + const sourceError = Object.freeze({ code: "SOURCE_UNAVAILABLE" }); + const source = vi.fn<() => Promise<{ readonly id: string; readonly version: number }>>() + .mockResolvedValueOnce({ id: "123", version: 1 }) + .mockRejectedValueOnce(sourceError); + let runtimeConfig = new DialCacheKeyConfig({}); + const dialcache = new DialCache({ + redis: { client: redis, readTimeoutMs: 1_000 }, + cacheConfigProvider: async () => runtimeConfig, + }); + const getUser = dialcache.cached(source, { + keyType: "user_id", + useCase, + cacheKey: () => "123", + defaultConfig: new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 1 }, + ramp: { [CacheLayer.REMOTE]: 100 }, + staleOnErrorMaxAgeSec: 10, + }), + }); + const valueKey = `${new DialCacheKey({ + keyType: "user_id", + id: "123", + useCase, + }).urn}:dialcache-frame-v1`; + + await expect(dialcache.enable(async () => await getUser())).resolves.toEqual({ + id: "123", + version: 1, + }); + expect(redis.ttlMs(valueKey)).toBe(10_000); + await vi.advanceTimersByTimeAsync(2_000); + runtimeConfig = disabledOverlay(); + const readsBeforeDisabledCall = redis.getCalls; + + await expect(dialcache.enable(async () => await getUser())).rejects.toBe(sourceError); + + expect(redis.getCalls - readsBeforeDisabledCall).toBe(expectedRemoteReads); + expect(redis.setCalls).toBe(1); + expect(redis.ttlMs(valueKey)).toBe(8_000); + expect(source).toHaveBeenCalledTimes(2); + } finally { + vi.useRealTimers(); + } + }, + ); + it("applies runtime config changes to subsequent calls", async () => { // Given a provider whose config can change without redeploying the cached function. let runtimeConfig: DialCacheKeyConfig | null = DialCacheKeyConfig.enabled(60); diff --git a/test/dialcache-liveness.test.ts b/test/dialcache-liveness.test.ts index 8f31305..203fe83 100644 --- a/test/dialcache-liveness.test.ts +++ b/test/dialcache-liveness.test.ts @@ -543,6 +543,7 @@ describe("DialCache fallback liveness", () => { const fallback = vi.fn(async () => "value"); const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout"); const redis: DialCacheRedisClient = { + enforcesMaxAge: true, read: async () => { readStarted.resolve(); return await readGate.promise; @@ -588,6 +589,7 @@ describe("DialCache fallback liveness", () => { }, }; const redis: DialCacheRedisClient = { + enforcesMaxAge: true, read: async () => ({ payload: "stored", createdAtMs: Date.now() }), write: async () => true, invalidate: async () => undefined, @@ -632,6 +634,7 @@ describe("DialCache fallback liveness", () => { load: (value) => value.toString(), }; const redis: DialCacheRedisClient = { + enforcesMaxAge: true, read: async () => null, write: async () => { writeStarted.resolve(); diff --git a/test/dialcache-logger.test.ts b/test/dialcache-logger.test.ts index a7df3e1..06cc93e 100644 --- a/test/dialcache-logger.test.ts +++ b/test/dialcache-logger.test.ts @@ -159,6 +159,7 @@ describe("DialCache logger isolation", () => { const logger = throwingLogger(); const invalidationError = new Error("invalidation failed"); const redis = { + enforcesMaxAge: true, read: vi.fn(async () => null), write: vi.fn(async () => true), invalidate: vi.fn(async () => { diff --git a/test/dialcache-metrics.test.ts b/test/dialcache-metrics.test.ts index 15f5fa1..09c2da7 100644 --- a/test/dialcache-metrics.test.ts +++ b/test/dialcache-metrics.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it, vi } from "vitest"; import { CacheLayer, DialCache, + DialCacheKey, DialCacheKeyConfig, type CacheMetricLabels, type CoalescedMetricLabels, @@ -14,8 +15,9 @@ import { type SerializationMetricLabels, type Serializer, type ShadowValidationMetricLabels, + type StaleRecoveryMetricLabels, } from "../src/index.js"; -import { FakeRedis } from "./fake-redis.js"; +import { encodeFrame, FakeRedis } from "./fake-redis.js"; class RecordingMetrics implements DialCacheMetricsAdapter { readonly events: Array<{ readonly name: string; readonly labels: Record; readonly value?: number }> = []; @@ -44,6 +46,10 @@ class RecordingMetrics implements DialCacheMetricsAdapter { this.record("coalesced", labels); } + staleRecovery(labels: StaleRecoveryMetricLabels): void { + this.record("staleRecovery", labels); + } + observeGet(labels: CacheMetricLabels, seconds: number): void { this.record("get", labels, seconds); } @@ -96,6 +102,7 @@ describe("DialCache observability metrics", () => { invalidation: vi.fn(() => thenable), coalesced: vi.fn(() => thenable), shadowValidation: vi.fn(() => thenable), + staleRecovery: vi.fn(() => thenable), observeGet: vi.fn(() => thenable), observeFallback: vi.fn(() => thenable), observeSerialization: vi.fn(() => thenable), @@ -133,6 +140,12 @@ describe("DialCache observability metrics", () => { keyType: "user_id", outcome: "match", } satisfies ShadowValidationMetricLabels); + isolatedMetrics.staleRecovery?.({ + cacheNamespace: "urn", + useCase: "RejectingMetricsThenable", + keyType: "user_id", + outcome: "served", + } satisfies StaleRecoveryMetricLabels); isolatedMetrics.observeGet(labels, 0); isolatedMetrics.observeFallback(labels, 0); isolatedMetrics.observeSerialization({ ...labels, operation: "dump" }, 0); @@ -140,7 +153,7 @@ describe("DialCache observability metrics", () => { expect(then).not.toHaveBeenCalled(); await tick(); - expect(then).toHaveBeenCalledTimes(11); + expect(then).toHaveBeenCalledTimes(12); }); it("includes the configured cache namespace on every metric path", async () => { @@ -410,6 +423,43 @@ describe("DialCache observability metrics", () => { expect(events(metrics, "error", { useCase: "DisabledByPolicy" })).toHaveLength(0); }); + it("reports invalid stale-on-error policy without disabling fresh Redis", async () => { + const metrics = new RecordingMetrics(); + const redis = new FakeRedis(); + const dialcache = new DialCache({ + metrics, + redis: { client: redis, readTimeoutMs: 1_000 }, + cacheConfigProvider: async () => new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 60 }, + staleOnErrorMaxAgeSec: 60, + }), + }); + let calls = 0; + const getUser = dialcache.cached(async (userId: string) => ({ userId, calls: ++calls }), { + keyType: "user_id", + useCase: "InvalidStaleOnErrorPolicy", + cacheKey: (userId) => userId, + }); + + const first = await dialcache.enable(async () => await getUser("123")); + const second = await dialcache.enable(async () => await getUser("123")); + + expect(second).toEqual(first); + expect(calls).toBe(1); + expect(redis.getCalls).toBe(2); + expect(redis.setCalls).toBe(1); + expect(events(metrics, "error", { + useCase: "InvalidStaleOnErrorPolicy", + layer: CacheLayer.REMOTE, + error: "config_resolution", + inFallback: false, + })).toHaveLength(2); + expect(events(metrics, "disabled", { + useCase: "InvalidStaleOnErrorPolicy", + layer: CacheLayer.REMOTE, + })).toHaveLength(0); + }); + it("labels cache errors separately from fallback errors", async () => { // Given cache and fallback errors carry caller-defined names containing dynamic identifiers. const metrics = new RecordingMetrics(); @@ -417,6 +467,7 @@ describe("DialCache observability metrics", () => { const cacheError = new Error("redis key urn:user_id:tenant-123 failed"); cacheError.name = "Tenant123RedisError"; const failingRedis: DialCacheRedisClient = { + enforcesMaxAge: true, read: vi.fn(async () => { throw cacheError; }), @@ -468,6 +519,87 @@ describe("DialCache observability metrics", () => { ); }); + it("records one complete telemetry trail when stale recovery serves a retained value", async () => { + const metrics = new RecordingMetrics(); + const redis = new FakeRedis(); + const useCase = "StaleRecoveryServedMetrics"; + const staleValue = { userId: "123", version: 1 }; + const key = new DialCacheKey({ keyType: "user_id", id: "123", useCase }); + redis.setRaw( + `${key.urn}:dialcache-frame-v1`, + encodeFrame(staleValue, Date.now() - 2_000), + 10_000, + ); + const source = vi.fn(async () => { + throw new Error("source unavailable"); + }); + const logger = { debug: vi.fn(), warn: vi.fn(), error: vi.fn() }; + const dialcache = new DialCache({ + metrics, + redis: { client: redis, readTimeoutMs: 1_000 }, + logger, + }); + const getUser = dialcache.cached(source, { + keyType: "user_id", + useCase, + cacheKey: () => "123", + defaultConfig: new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 1 }, + ramp: { [CacheLayer.REMOTE]: 100 }, + staleOnErrorMaxAgeSec: 10, + }), + }); + + await expect(dialcache.enable(async () => await getUser())).resolves.toEqual(staleValue); + + expect(source).toHaveBeenCalledOnce(); + expect(events(metrics, "error", { useCase })).toEqual([ + { + name: "error", + labels: { + cacheNamespace: "urn", + useCase, + keyType: "user_id", + layer: CacheLayer.REMOTE, + error: "fallback", + inFallback: true, + }, + }, + ]); + const remoteLabels = { + cacheNamespace: "urn", + useCase, + keyType: "user_id", + layer: CacheLayer.REMOTE, + }; + expect(events(metrics, "fallback", { useCase })).toEqual([ + { name: "fallback", labels: remoteLabels, value: expect.any(Number) }, + ]); + expect(events(metrics, "miss", { useCase })).toEqual([ + { name: "miss", labels: remoteLabels }, + ]); + expect(events(metrics, "request", { useCase })).toEqual([ + { name: "request", labels: remoteLabels }, + { name: "request", labels: remoteLabels }, + ]); + expect(events(metrics, "get", { useCase })).toEqual([ + { name: "get", labels: remoteLabels, value: expect.any(Number) }, + { name: "get", labels: remoteLabels, value: expect.any(Number) }, + ]); + expect(events(metrics, "staleRecovery", { useCase })).toEqual([ + { + name: "staleRecovery", + labels: { + cacheNamespace: "urn", + useCase, + keyType: "user_id", + outcome: "served", + }, + }, + ]); + expect(logger.warn).not.toHaveBeenCalled(); + }); + it("classifies config, Redis write, and serializer failures by stable operation", async () => { const metrics = new RecordingMetrics(); const logger = { debug: vi.fn(), warn: vi.fn(), error: vi.fn() }; diff --git a/test/dialcache-observability-internals.test.ts b/test/dialcache-observability-internals.test.ts index d6ec4a4..554f01a 100644 --- a/test/dialcache-observability-internals.test.ts +++ b/test/dialcache-observability-internals.test.ts @@ -3,7 +3,11 @@ import { describe, expect, it } from "vitest"; import { CacheLayer, DialCacheKey, DialCacheKeyConfig } from "../src/index.js"; import { LocalCache } from "../src/internal/local-cache.js"; import { RedisCache } from "../src/internal/redis-cache.js"; -import { fetchKeyConfig, resolveLayerConfig } from "../src/internal/runtime-config.js"; +import { + fetchKeyConfig, + resolveLayerConfig, + resolveRemoteLayerConfigResult, +} from "../src/internal/runtime-config.js"; import { encodeFrame, FakeRedis } from "./fake-redis.js"; const key = (defaultConfig: DialCacheKeyConfig | null = DialCacheKeyConfig.enabled(60)) => @@ -20,6 +24,7 @@ describe("DialCache observability internal compatibility paths", () => { ramp: 20, logMismatches: true, }, + staleOnErrorMaxAgeSec: 3_600, }); const cases = [ { @@ -39,6 +44,7 @@ describe("DialCache observability internal compatibility paths", () => { ramp: 80, logMismatches: true, }, + staleOnErrorMaxAgeSec: 3_600, }), }, { @@ -48,6 +54,7 @@ describe("DialCache observability internal compatibility paths", () => { shadow: { logMismatches: false, }, + staleOnErrorMaxAgeSec: 0, }), expected: new DialCacheKeyConfig({ requestLocal: true, @@ -58,10 +65,11 @@ describe("DialCache observability internal compatibility paths", () => { ramp: 20, logMismatches: false, }, + staleOnErrorMaxAgeSec: 0, }), }, { - runtime: new DialCacheKeyConfig({ shadow: {} }), + runtime: new DialCacheKeyConfig({ shadow: {}, staleOnErrorMaxAgeSec: 7_200 }), expected: new DialCacheKeyConfig({ requestLocal: true, coalesce: false, @@ -71,6 +79,7 @@ describe("DialCache observability internal compatibility paths", () => { ramp: 20, logMismatches: true, }, + staleOnErrorMaxAgeSec: 7_200, }), }, ]; @@ -164,4 +173,48 @@ describe("DialCache observability internal compatibility paths", () => { expect(noConfig).toBeNull(); expect(noRamp).toEqual({ ttlSec: 60, ramp: 100 }); }); + + it("keeps invalid stale-on-error policy diagnostic-only in remote resolution", () => { + const remoteKey = key(); + + expect(resolveRemoteLayerConfigResult({ + config: new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 60 }, + staleOnErrorMaxAgeSec: 3_600, + }), + key: remoteKey, + })).toEqual({ + status: "enabled", + config: { ttlSec: 60, ramp: 100, staleOnErrorMaxAgeSec: 3_600 }, + }); + expect(resolveRemoteLayerConfigResult({ + config: new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 60 }, + staleOnErrorMaxAgeSec: 60, + }), + key: remoteKey, + })).toEqual({ + status: "enabled", + config: { ttlSec: 60, ramp: 100, staleOnErrorMaxAgeSec: null }, + staleOnErrorConfigError: true, + }); + expect(resolveRemoteLayerConfigResult({ + config: new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 60 }, + staleOnErrorMaxAgeSec: 0, + }), + key: remoteKey, + })).toEqual({ + status: "enabled", + config: { ttlSec: 60, ramp: 100, staleOnErrorMaxAgeSec: null }, + }); + expect(resolveRemoteLayerConfigResult({ + config: new DialCacheKeyConfig({ staleOnErrorMaxAgeSec: 3_600 }), + key: remoteKey, + })).toEqual({ + status: "disabled", + reason: "policy_disabled", + staleOnErrorConfigError: true, + }); + }); }); diff --git a/test/dialcache-redis-read-deadline.test.ts b/test/dialcache-redis-read-deadline.test.ts index 211c56c..4c08388 100644 --- a/test/dialcache-redis-read-deadline.test.ts +++ b/test/dialcache-redis-read-deadline.test.ts @@ -65,6 +65,7 @@ function redisClient(read: DialCacheRedisClient["read"]): { const write = vi.fn(async () => true); return { client: { + enforcesMaxAge: true, read: readMock, write, invalidate: async () => undefined, @@ -180,6 +181,20 @@ describe("DialCache Redis read deadlines", () => { ).not.toThrow(); }); + it("rejects legacy semantic clients that do not attest max-age enforcement", () => { + const legacyClient = { + read: async () => null, + write: async () => true, + invalidate: async () => undefined, + }; + + expect( + () => new DialCache({ + redis: { client: legacyClient as unknown as DialCacheRedisClient }, + }), + ).toThrow(new TypeError("DialCache Redis client must declare enforcesMaxAge: true")); + }); + it("rejects invalid static use-case overrides before reserving the use-case name", () => { const client = redisClient(async () => null).client; const invalidValues: readonly unknown[] = [ diff --git a/test/dialcache-redis.test.ts b/test/dialcache-redis.test.ts index 3c9abfd..32503ba 100644 --- a/test/dialcache-redis.test.ts +++ b/test/dialcache-redis.test.ts @@ -300,13 +300,13 @@ describe("DialCache Redis TTL layer", () => { const payload = Buffer.from([0, 1, 2, 0xff]); await redis.write({ valueKey, cacheTtlMs: 60_000, value: payload }); - const firstRead = await redis.read({ valueKey }); + const firstRead = await redis.read({ valueKey, maxAgeMs: 60_000 }); if (!Buffer.isBuffer(firstRead?.payload)) { throw new Error("Expected a binary Redis payload"); } firstRead.payload[0] = 0xff; - expect((await redis.read({ valueKey }))?.payload).toEqual(payload); + expect((await redis.read({ valueKey, maxAgeMs: 60_000 }))?.payload).toEqual(payload); }); it("fails open when Redis serializer dump fails", async () => { @@ -421,6 +421,7 @@ describe("DialCache Redis TTL layer", () => { it("records a distinct metric label when a Redis adapter reports invalid payload encoding", async () => { const redisClient: DialCacheRedisClient = { + enforcesMaxAge: true, read: vi.fn(async () => { throw new DialCacheRedisPayloadEncodingError("Invalid DialCache Redis payload encoding"); }), diff --git a/test/dialcache-shadow-confirmation.test.ts b/test/dialcache-shadow-confirmation.test.ts index 0bd33b7..5e54a71 100644 --- a/test/dialcache-shadow-confirmation.test.ts +++ b/test/dialcache-shadow-confirmation.test.ts @@ -25,6 +25,7 @@ import { type SerializationMetricLabels, type Serializer, type ShadowValidationMetricLabels, + type StaleRecoveryMetricLabels, } from "../src/index.js"; import { deterministicRampSample, @@ -55,14 +56,14 @@ function deferred(): Deferred { type ReadStep = () => RedisCachePayload | null | Promise; -const SCRIPTED_FRAME_CREATED_AT_MS = 1_700_000_000_000; - class ScriptedRedis implements DialCacheRedisClient { + readonly enforcesMaxAge = true; readonly requests: RedisReadRequest[] = []; readonly contexts: Array = []; readonly write = vi.fn(async (_request: RedisWriteRequest): Promise => true); readonly invalidate = vi.fn(async (_request: RedisInvalidationRequest): Promise => undefined); - frameCreatedAtMs = SCRIPTED_FRAME_CREATED_AT_MS; + frameCreatedAtMs = Date.now(); + bypassMaxAge = false; constructor(private readonly steps: ReadStep[]) {} @@ -74,7 +75,13 @@ class ScriptedRedis implements DialCacheRedisClient { throw new Error("Unexpected Redis read"); } const payload = await step(); - return payload === null ? null : { payload, createdAtMs: this.frameCreatedAtMs }; + if (payload === null) { + return null; + } + if (!this.bypassMaxAge && Date.now() - this.frameCreatedAtMs >= request.maxAgeMs) { + return null; + } + return { payload, createdAtMs: this.frameCreatedAtMs }; } } @@ -104,6 +111,7 @@ class RecordingMetrics implements DialCacheMetricsAdapter { readonly ordinaryEvents: OrdinaryMetricEvent[] = []; readonly shadowEvents: ShadowValidationMetricLabels[] = []; readonly shadowAgeEvents: ShadowAgeEvent[] = []; + readonly staleRecoveryEvents: StaleRecoveryMetricLabels[] = []; request(labels: CacheMetricLabels): void { this.record("request", labels); @@ -137,6 +145,10 @@ class RecordingMetrics implements DialCacheMetricsAdapter { this.shadowAgeEvents.push({ labels: { ...labels }, seconds }); } + staleRecovery(labels: StaleRecoveryMetricLabels): void { + this.staleRecoveryEvents.push({ ...labels }); + } + observeGet(labels: CacheMetricLabels, _seconds: number): void { this.record("get", labels); } @@ -232,10 +244,13 @@ async function waitForShadowEvents(metrics: RecordingMetrics, count: number): Pr function expectTrackedReads( redis: ScriptedRedis, count: number, - options: { readonly singleWatermark?: boolean } = { singleWatermark: true }, + options: { readonly singleWatermark?: boolean; readonly maxAgeMs?: number } = { singleWatermark: true }, ): void { expect(redis.requests).toHaveLength(count); expect(redis.requests.every(({ watermarkKey }) => typeof watermarkKey === "string")).toBe(true); + if (options.maxAgeMs !== undefined) { + expect(redis.requests.every(({ maxAgeMs }) => maxAgeMs === options.maxAgeMs)).toBe(true); + } if (options.singleWatermark !== false) { expect(new Set(redis.requests.map(({ watermarkKey }) => watermarkKey)).size).toBe(1); } @@ -565,7 +580,7 @@ describe("DialCache Redis shadow confirmation", () => { return payload; }, ]); - redis.frameCreatedAtMs = nowMs - 90_000; + redis.frameCreatedAtMs = nowMs - 30_000; const metrics = new RecordingMetrics(); const dialcache = createCache(redis, metrics); const getUser = dialcache.cached(async () => ({ id: "123", version: 2 }), { @@ -578,7 +593,7 @@ describe("DialCache Redis shadow confirmation", () => { expect(metrics.shadowEvents.map(({ outcome }) => outcome)).toEqual(["mismatch"]); expect(metrics.shadowAgeEvents).toHaveLength(1); - expect(metrics.shadowAgeEvents[0]?.seconds).toBe(90); + expect(metrics.shadowAgeEvents[0]?.seconds).toBe(30); expect(metrics.shadowAgeEvents[0]?.labels).toMatchObject({ useCase: "ShadowMismatchValueAge", keyType: "user_id", @@ -593,6 +608,8 @@ describe("DialCache Redis shadow confirmation", () => { const payload = JSON.stringify({ id: "123", version: 1 }); const redis = new ScriptedRedis([() => payload]); redis.frameCreatedAtMs = Number.NaN; + // Deliberately violate the adapter contract to verify core metric isolation. + redis.bypassMaxAge = true; const metrics = new RecordingMetrics(); const dialcache = createCache(redis, metrics); const getUser = dialcache.cached(async () => ({ id: "123", version: 1 }), { @@ -860,6 +877,53 @@ describe("DialCache Redis shadow confirmation", () => { expectTrackedReads(redis, 1); }); + it("never serves retained stale data when remote serving is ramped down", async () => { + const stalePayload = JSON.stringify({ id: "123", source: "stale-cache" }); + const redis = new ScriptedRedis([() => stalePayload]); + redis.frameCreatedAtMs = Date.now() - 120_000; + const metrics = new RecordingMetrics(); + const sourceError = Object.freeze({ code: "SOURCE_UNAVAILABLE" }); + const source = vi.fn(async () => { + throw sourceError; + }); + const dialcache = createCache(redis, metrics); + const getUser = dialcache.cached(source, { + ...trackedOptions("ShadowDarkRetainedStale", new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 60 }, + ramp: { [CacheLayer.REMOTE]: 0 }, + staleOnErrorMaxAgeSec: 3_600, + shadow: { ramp: 100 }, + })), + cacheKey: () => "123", + }); + + await expect(dialcache.enable(async () => await getUser())).rejects.toBe(sourceError); + await waitForShadowEvents(metrics, 1); + + expect(source).toHaveBeenCalledOnce(); + expectTrackedReads(redis, 1, { maxAgeMs: 60_000 }); + expect(metrics.staleRecoveryEvents).toHaveLength(0); + expect(redis.write).not.toHaveBeenCalled(); + expect(redis.invalidate).not.toHaveBeenCalled(); + expect(metrics.shadowEvents).toEqual([{ + cacheNamespace: "urn", + useCase: "ShadowDarkRetainedStale", + keyType: "user_id", + outcome: "source_error", + }]); + expect(metrics.ordinaryEvents.filter(({ name, labels }) => + name === "request" && labels.layer === REMOTE_SHADOW_CACHE_LAYER + )).toHaveLength(1); + expect(metrics.ordinaryEvents.filter(({ name, labels }) => + name === "get" && labels.layer === REMOTE_SHADOW_CACHE_LAYER + )).toHaveLength(1); + expect(metrics.ordinaryEvents.filter(({ name, labels }) => + name === "disabled" + && labels.layer === CacheLayer.REMOTE + && labels.reason === "ramped_down" + )).toHaveLength(1); + }); + it("does not misclassify a source-propagated FallbackTimeoutError as its own timeout", async () => { const redis = new ScriptedRedis([() => JSON.stringify({ id: "123", source: "cache" })]); const metrics = new RecordingMetrics(); @@ -997,7 +1061,7 @@ describe("DialCache Redis shadow confirmation", () => { it.each([ { name: "tracked", tracked: true }, { name: "untracked", tracked: false }, - ])("fills a clean $name dark Redis miss and attributes the read and write to remote_shadow", async ({ + ])("retains a clean $name dark Redis miss through M and attributes the work to remote_shadow", async ({ name, tracked, }) => { @@ -1010,7 +1074,12 @@ describe("DialCache Redis shadow confirmation", () => { useCase: `ShadowDarkMissFill${name}`, cacheKey: () => "123", trackForInvalidation: tracked, - defaultConfig: remoteConfig(0), + defaultConfig: new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 60 }, + ramp: { [CacheLayer.REMOTE]: 0 }, + staleOnErrorMaxAgeSec: 3_600, + shadow: { ramp: 100 }, + }), }); await expect(dialcache.enable(async () => await getUser())).resolves.toEqual({ id: "123" }); @@ -1026,7 +1095,7 @@ describe("DialCache Redis shadow confirmation", () => { } expect(redis.write).toHaveBeenCalledOnce(); expect(redis.write).toHaveBeenCalledWith(expect.objectContaining({ - cacheTtlMs: 60_000, + cacheTtlMs: 3_600_000, value: JSON.stringify({ id: "123" }), })); expect(Object.hasOwn(redis.write.mock.calls[0]?.[0] ?? {}, "watermarkKey")).toBe(tracked); diff --git a/test/dialcache-shadow-validation.test.ts b/test/dialcache-shadow-validation.test.ts index 0eed650..5ead019 100644 --- a/test/dialcache-shadow-validation.test.ts +++ b/test/dialcache-shadow-validation.test.ts @@ -321,7 +321,7 @@ describe("DialCache Redis shadow validation", () => { id: "123", useCase, payload: JSON.stringify({ id: "123", version: 1 }), - createdAtMs: nowMs - 120_000, + createdAtMs: nowMs - 50_000, }); const dialcache = createShadowCache(redis, metrics); const getUser = dialcache.cached(async () => ({ id: "123", version: 2 }), { @@ -334,14 +334,14 @@ describe("DialCache Redis shadow validation", () => { expect(metrics.shadowEvents[0]?.outcome).toBe("mismatch"); expect(metrics.shadowAgeEvents).toHaveLength(1); - expect(metrics.shadowAgeEvents[0]?.seconds).toBe(120); + expect(metrics.shadowAgeEvents[0]?.seconds).toBe(50); expect(metrics.shadowAgeEvents[0]?.labels).toMatchObject({ useCase, outcome: "mismatch" }); } finally { nowSpy.mockRestore(); } }); - it("clamps a future-stamped frame to a zero value age instead of a negative one", async () => { + it("rejects a future-stamped frame as a logical miss instead of reporting a negative age", async () => { const nowMs = 1_700_000_000_000; const nowSpy = vi.spyOn(Date, "now").mockReturnValue(nowMs); try { @@ -356,17 +356,19 @@ describe("DialCache Redis shadow validation", () => { createdAtMs: nowMs + 60_000, }); const dialcache = createShadowCache(redis, metrics); - const getUser = dialcache.cached(async () => cachedValue, { + const source = vi.fn(async () => cachedValue); + const getUser = dialcache.cached(source, { ...trackedRemoteDefaults(useCase), cacheKey: () => "123", }); expect(await dialcache.enable(async () => await getUser())).toEqual(cachedValue); - await waitForShadowEvents(metrics, 1); + await nextImmediate(); - expect(metrics.shadowEvents[0]?.outcome).toBe("match"); - expect(metrics.shadowAgeEvents).toHaveLength(1); - expect(metrics.shadowAgeEvents[0]?.seconds).toBe(0); + expect(source).toHaveBeenCalledOnce(); + expect(metrics.shadowEvents).toHaveLength(0); + expect(metrics.shadowAgeEvents).toHaveLength(0); + expect(redis.setCalls).toBe(1); } finally { nowSpy.mockRestore(); } diff --git a/test/dialcache-stale-on-error.test.ts b/test/dialcache-stale-on-error.test.ts new file mode 100644 index 0000000..59706c4 --- /dev/null +++ b/test/dialcache-stale-on-error.test.ts @@ -0,0 +1,984 @@ +import { performance } from "node:perf_hooks"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { + CacheLayer, + DialCache, + DialCacheKey, + DialCacheKeyConfig, + type DecodedRedisFrame, + type DialCacheMetricsAdapter, + type RedisReadContext, + type RedisReadRequest, + type Serializer, +} from "../src/index.js"; +import { MARKER_ZSTD_UTF8 } from "../src/internal/compression.js"; +import { decodeFrame, encodeFrame, FakeRedis } from "./fake-redis.js"; + +const FRESH_TTL_SEC = 1; +const MAX_AGE_SEC = 10; + +class RecordingRedis extends FakeRedis { + readonly readRequests: RedisReadRequest[] = []; + readonly readContexts: Array = []; + + override async read( + request: RedisReadRequest, + context?: RedisReadContext, + ): Promise { + this.readRequests.push(request); + this.readContexts.push(context); + return await super.read(request); + } +} + +class HangingReadRedis extends FakeRedis { + readonly readRequests: RedisReadRequest[] = []; + readonly readContexts: Array = []; + + constructor(private readonly hangOnCall: number) { + super(); + } + + override async read( + request: RedisReadRequest, + context?: RedisReadContext, + ): Promise { + this.readRequests.push(request); + this.readContexts.push(context); + if (this.readRequests.length === this.hangOnCall) { + return await new Promise(() => undefined); + } + return await super.read(request); + } +} + +interface Deferred { + readonly promise: Promise; + resolve(value: T): void; + reject(error: unknown): void; +} + +function deferred(): Deferred { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +} + +function staleConfig(options: { readonly local?: boolean; readonly requestLocal?: boolean } = {}): DialCacheKeyConfig { + return new DialCacheKeyConfig({ + ttlSec: { + ...(options.local ? { [CacheLayer.LOCAL]: 60 } : {}), + [CacheLayer.REMOTE]: FRESH_TTL_SEC, + }, + ramp: { + ...(options.local ? { [CacheLayer.LOCAL]: 100 } : {}), + [CacheLayer.REMOTE]: 100, + }, + ...(options.requestLocal ? { requestLocal: true } : {}), + staleOnErrorMaxAgeSec: MAX_AGE_SEC, + }); +} + +function redisValueKey(useCase: string, id = "123", trackForInvalidation = false): string { + const key = new DialCacheKey({ keyType: "user_id", id, useCase, trackForInvalidation }); + return `${key.urn}:dialcache-frame-v1`; +} + +function watermarkKey(id = "123"): string { + return `{urn:user_id:${id}}#watermark`; +} + +function seedStale(redis: FakeRedis, useCase: string, value: unknown, trackForInvalidation = false): void { + redis.setRaw( + redisValueKey(useCase, "123", trackForInvalidation), + encodeFrame(value, Date.now() - 2_000), + MAX_AGE_SEC * 1_000, + ); +} + +function recordingMetrics(): { + readonly metrics: DialCacheMetricsAdapter; + readonly staleRecovery: ReturnType; + readonly shadowValidation: ReturnType; +} { + const staleRecovery = vi.fn(); + const shadowValidation = vi.fn(); + return { + staleRecovery, + shadowValidation, + metrics: { + request: vi.fn(), + miss: vi.fn(), + disabled: vi.fn(), + error: vi.fn(), + invalidation: vi.fn(), + shadowValidation, + staleRecovery, + observeGet: vi.fn(), + observeFallback: vi.fn(), + observeSerialization: vi.fn(), + observeSize: vi.fn(), + }, + }; +} + +function rejectionReason(result: PromiseSettledResult): unknown { + if (result.status !== "rejected") { + throw new Error("Expected rejection"); + } + return result.reason; +} + +describe("DialCache stale-on-error recovery", () => { + beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-02T12:00:00.000Z")); + const clockOriginMs = Date.now(); + vi.spyOn(performance, "now").mockImplementation(() => Date.now() - clockOriginMs); + }); + + afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); + }); + + it("rereads a logical miss at the maximum age and serves it without publication", async () => { + const useCase = "StaleRecoveryServed"; + const redis = new RecordingRedis(); + const staleValue = { id: "123", version: 1 }; + seedStale(redis, useCase, staleValue); + const ttlBefore = redis.ttlMs(redisValueKey(useCase)); + const sourceError = Object.freeze({ code: "SOURCE_UNAVAILABLE" }); + const source = vi.fn((): { readonly id: string; readonly version: number } => { + throw sourceError; + }); + const { metrics, staleRecovery } = recordingMetrics(); + const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 }, metrics }); + const getUser = dialcache.cached(source, { + keyType: "user_id", + useCase, + cacheKey: () => "123", + defaultConfig: staleConfig(), + }); + + await expect(dialcache.enable(async () => await getUser())).resolves.toEqual(staleValue); + + expect(source).toHaveBeenCalledOnce(); + expect(redis.readRequests.map(({ maxAgeMs }) => maxAgeMs)).toEqual([1_000, 10_000]); + expect(redis.setCalls).toBe(0); + expect(redis.ttlMs(redisValueKey(useCase))).toBe(ttlBefore); + expect(staleRecovery).toHaveBeenCalledOnce(); + expect(staleRecovery).toHaveBeenCalledWith({ + cacheNamespace: "urn", + useCase, + keyType: "user_id", + outcome: "served", + }); + }); + + it("returns a logically fresh Redis hit without calling the source or recovery", async () => { + const useCase = "StaleRecoveryFreshHit"; + const redis = new RecordingRedis(); + redis.setRaw( + redisValueKey(useCase), + encodeFrame({ id: "123", version: 1 }, Date.now()), + MAX_AGE_SEC * 1_000, + ); + const source = vi.fn(async () => ({ id: "123", version: 2 })); + const { metrics, staleRecovery } = recordingMetrics(); + const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 }, metrics }); + const getUser = dialcache.cached(source, { + keyType: "user_id", + useCase, + cacheKey: () => "123", + defaultConfig: staleConfig(), + }); + + await expect(dialcache.enable(async () => await getUser())).resolves.toEqual({ id: "123", version: 1 }); + + expect(source).not.toHaveBeenCalled(); + expect(redis.readRequests.map(({ maxAgeMs }) => maxAgeMs)).toEqual([1_000]); + expect(staleRecovery).not.toHaveBeenCalled(); + }); + + it("treats a frame at the exact fresh-age boundary as stale", async () => { + const useCase = "StaleRecoveryExactFreshBoundary"; + const redis = new RecordingRedis(); + const retained = { id: "123", version: 1 }; + redis.setRaw( + redisValueKey(useCase), + encodeFrame(retained, Date.now() - FRESH_TTL_SEC * 1_000), + MAX_AGE_SEC * 1_000, + ); + const source = vi.fn(async () => { + throw new Error("source unavailable"); + }); + const { metrics, staleRecovery } = recordingMetrics(); + const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 }, metrics }); + const getUser = dialcache.cached(source, { + keyType: "user_id", + useCase, + cacheKey: () => "123", + defaultConfig: staleConfig(), + }); + + await expect(dialcache.enable(async () => await getUser())).resolves.toEqual(retained); + + expect(source).toHaveBeenCalledOnce(); + expect(redis.readRequests.map(({ maxAgeMs }) => maxAgeMs)).toEqual([1_000, 10_000]); + expect(staleRecovery).toHaveBeenCalledWith(expect.objectContaining({ outcome: "served" })); + }); + + it("keeps feature-off writes at the fresh TTL and never performs a recovery read", async () => { + const useCase = "StaleRecoveryFeatureOff"; + const redis = new RecordingRedis(); + const sourceError = Object.freeze({ code: "SOURCE_UNAVAILABLE" }); + const source = vi.fn<() => Promise<{ readonly id: string; readonly version: number }>>() + .mockResolvedValueOnce({ id: "123", version: 1 }) + .mockRejectedValueOnce(sourceError); + const { metrics, staleRecovery } = recordingMetrics(); + const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 }, metrics }); + const getUser = dialcache.cached(source, { + keyType: "user_id", + useCase, + cacheKey: () => "123", + defaultConfig: new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: FRESH_TTL_SEC }, + ramp: { [CacheLayer.REMOTE]: 100 }, + }), + }); + + await expect(dialcache.enable(async () => await getUser())).resolves.toEqual({ id: "123", version: 1 }); + expect(redis.ttlMs(redisValueKey(useCase))).toBe(1_000); + await vi.advanceTimersByTimeAsync(1_000); + const [settled] = await Promise.allSettled([dialcache.enable(async () => await getUser())]); + + expect(rejectionReason(settled!)).toBe(sourceError); + expect(redis.readRequests.map(({ maxAgeMs }) => maxAgeMs)).toEqual([1_000, 1_000]); + expect(redis.setCalls).toBe(1); + expect(staleRecovery).not.toHaveBeenCalled(); + }); + + it("refreshes a retained logically stale frame without a recovery reread and writes with maximum retention", async () => { + const useCase = "StaleRecoverySourceSuccess"; + const redis = new RecordingRedis(); + seedStale(redis, useCase, { id: "123", version: 1 }); + const { metrics, staleRecovery } = recordingMetrics(); + const sourceValue = { id: "123", version: 2 }; + const source = vi.fn(async () => sourceValue); + const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 }, metrics }); + const getUser = dialcache.cached(source, { + keyType: "user_id", + useCase, + cacheKey: () => "123", + defaultConfig: staleConfig(), + }); + + await expect(dialcache.enable(async () => await getUser())).resolves.toBe(sourceValue); + + expect(source).toHaveBeenCalledOnce(); + expect(redis.readRequests.map(({ maxAgeMs }) => maxAgeMs)).toEqual([1_000]); + expect(redis.setCalls).toBe(1); + expect(redis.ttlMs(redisValueKey(useCase))).toBe(10_000); + const refreshedFrame = decodeFrame(redis.raw(redisValueKey(useCase))); + expect(refreshedFrame.createdAtMs).toBe(Date.now()); + expect(JSON.parse(refreshedFrame.payload as string)).toEqual(sourceValue); + expect(staleRecovery).not.toHaveBeenCalled(); + }); + + it("rejects with the original source error when a retained frame reaches the exact maximum age", async () => { + const useCase = "StaleRecoveryCrossesMaximumDuringSource"; + const redis = new RecordingRedis(); + const valueKey = redisValueKey(useCase); + redis.setRaw( + valueKey, + encodeFrame({ id: "123", version: 1 }, Date.now() - 9_000), + 20_000, + ); + const sourceError = Object.freeze({ code: "SOURCE_UNAVAILABLE" }); + const sourceStarted = deferred(); + const sourceGate = deferred<{ readonly id: string; readonly version: number }>(); + const { metrics, staleRecovery } = recordingMetrics(); + const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 }, metrics }); + const getUser = dialcache.cached(async () => { + sourceStarted.resolve(); + return await sourceGate.promise; + }, { + keyType: "user_id", + useCase, + cacheKey: () => "123", + defaultConfig: staleConfig(), + }); + + const result = Promise.allSettled([dialcache.enable(async () => await getUser())]); + await sourceStarted.promise; + await vi.advanceTimersByTimeAsync(1_000); + sourceGate.reject(sourceError); + const [settled] = await result; + + expect(rejectionReason(settled!)).toBe(sourceError); + expect(redis.readRequests.map(({ maxAgeMs }) => maxAgeMs)).toEqual([1_000, 10_000]); + expect(redis.ttlMs(valueKey)).toBe(19_000); + expect(redis.setCalls).toBe(0); + expect(staleRecovery).toHaveBeenCalledOnce(); + expect(staleRecovery).toHaveBeenCalledWith(expect.objectContaining({ outcome: "miss" })); + }); + + it("uses one runtime policy snapshot for the initial read and delayed recovery", async () => { + const useCase = "StaleRecoveryRuntimePolicySnapshot"; + const redis = new RecordingRedis(); + const staleValue = { id: "123", version: 1 }; + seedStale(redis, useCase, staleValue); + const sourceError = Object.freeze({ code: "SOURCE_UNAVAILABLE" }); + const sourceStarted = deferred(); + const sourceGate = deferred<{ readonly id: string; readonly version: number }>(); + let freshTtlSec = 1; + let maxAgeSec = 3; + let remoteReadTimeoutMs = 25; + const cacheConfigProvider = vi.fn(async () => new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: freshTtlSec }, + ramp: { [CacheLayer.REMOTE]: 100 }, + staleOnErrorMaxAgeSec: maxAgeSec, + remoteReadTimeoutMs, + })); + const dialcache = new DialCache({ + redis: { client: redis, readTimeoutMs: 1_000 }, + cacheConfigProvider, + }); + const getUser = dialcache.cached(async () => { + sourceStarted.resolve(); + return await sourceGate.promise; + }, { + keyType: "user_id", + useCase, + cacheKey: () => "123", + }); + + const result = Promise.allSettled([dialcache.enable(async () => await getUser())]); + await sourceStarted.promise; + freshTtlSec = 4; + maxAgeSec = 20; + remoteReadTimeoutMs = 75; + sourceGate.reject(sourceError); + const [settled] = await result; + + expect(settled).toEqual({ status: "fulfilled", value: staleValue }); + expect(cacheConfigProvider).toHaveBeenCalledOnce(); + expect(redis.readRequests.map(({ maxAgeMs }) => maxAgeMs)).toEqual([1_000, 3_000]); + expect(redis.readContexts.map((context) => context?.timeoutMs)).toEqual([25, 25]); + }); + + it("applies the current runtime fresh age to an existing retained frame", async () => { + const useCase = "StaleRecoveryRuntimeFreshAge"; + const redis = new RecordingRedis(); + const retainedValue = { id: "123", version: 1 }; + redis.setRaw( + redisValueKey(useCase), + encodeFrame(retainedValue, Date.now() - 3_000), + MAX_AGE_SEC * 1_000, + ); + const sourceError = Object.freeze({ code: "SOURCE_UNAVAILABLE" }); + const source = vi.fn(async () => { + throw sourceError; + }); + let freshTtlSec = 4; + const cacheConfigProvider = vi.fn(async () => new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: freshTtlSec }, + ramp: { [CacheLayer.REMOTE]: 100 }, + staleOnErrorMaxAgeSec: MAX_AGE_SEC, + })); + const dialcache = new DialCache({ + redis: { client: redis, readTimeoutMs: 1_000 }, + cacheConfigProvider, + }); + const getUser = dialcache.cached(source, { + keyType: "user_id", + useCase, + cacheKey: () => "123", + }); + + await expect(dialcache.enable(async () => await getUser())).resolves.toEqual(retainedValue); + expect(source).not.toHaveBeenCalled(); + + freshTtlSec = 2; + await expect(dialcache.enable(async () => await getUser())).resolves.toEqual(retainedValue); + expect(source).toHaveBeenCalledOnce(); + + freshTtlSec = 4; + await expect(dialcache.enable(async () => await getUser())).resolves.toEqual(retainedValue); + + expect(source).toHaveBeenCalledOnce(); + expect(cacheConfigProvider).toHaveBeenCalledTimes(3); + expect(redis.readRequests.map(({ maxAgeMs }) => maxAgeMs)).toEqual([ + 4_000, + 2_000, + 10_000, + 4_000, + ]); + }); + + it("retains a tracked watermark through the maximum age plus its existing margin", async () => { + const useCase = "StaleRecoveryTrackedRetention"; + const redis = new RecordingRedis(); + const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 } }); + const getUser = dialcache.cached(async () => ({ id: "123" }), { + keyType: "user_id", + useCase, + cacheKey: () => "123", + trackForInvalidation: true, + defaultConfig: staleConfig(), + }); + + await expect(dialcache.enable(async () => await getUser())).resolves.toEqual({ id: "123" }); + + expect(redis.ttlMs(redisValueKey(useCase, "123", true))).toBe(10_000); + expect(redis.ttlMs(watermarkKey())).toBe(70_000); + expect(redis.readRequests).toEqual([ + expect.objectContaining({ maxAgeMs: 1_000, watermarkKey: watermarkKey() }), + ]); + }); + + it.each([ + ["object", Object.freeze({ code: "SOURCE_OBJECT" })], + ["null", null], + ["undefined", undefined], + ] as const)("preserves an arbitrary %s rejection when recovery misses", async (_name, sourceError) => { + const useCase = `StaleRecoveryIdentity${_name}`; + const redis = new RecordingRedis(); + const { metrics, staleRecovery } = recordingMetrics(); + const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 }, metrics }); + const getUser = dialcache.cached(async () => { + throw sourceError; + }, { + keyType: "user_id", + useCase, + cacheKey: () => "123", + defaultConfig: staleConfig(), + }); + + const [settled] = await Promise.allSettled([dialcache.enable(async () => await getUser())]); + + expect(rejectionReason(settled!)).toBe(sourceError); + expect(redis.readRequests.map(({ maxAgeMs }) => maxAgeMs)).toEqual([1_000, 10_000]); + expect(staleRecovery).toHaveBeenCalledOnce(); + expect(staleRecovery).toHaveBeenCalledWith(expect.objectContaining({ outcome: "miss" })); + }); + + it("never attempts recovery after the initial Redis read fails", async () => { + const useCase = "StaleRecoveryInitialReadError"; + const redis = new RecordingRedis(); + redis.failGet = true; + const sourceError = Object.freeze({ code: "SOURCE_UNAVAILABLE" }); + const { metrics, staleRecovery } = recordingMetrics(); + const dialcache = new DialCache({ + redis: { client: redis, readTimeoutMs: 1_000 }, + metrics, + logger: { debug: vi.fn(), error: vi.fn(), warn: vi.fn() }, + }); + const getUser = dialcache.cached(async () => { + throw sourceError; + }, { + keyType: "user_id", + useCase, + cacheKey: () => "123", + defaultConfig: staleConfig(), + }); + + const [settled] = await Promise.allSettled([dialcache.enable(async () => await getUser())]); + + expect(rejectionReason(settled!)).toBe(sourceError); + expect(redis.readRequests).toHaveLength(1); + expect(staleRecovery).not.toHaveBeenCalled(); + }); + + it("preserves the source rejection when the recovery read fails", async () => { + const useCase = "StaleRecoveryReadError"; + const redis = new RecordingRedis(); + const sourceError = Object.freeze({ code: "SOURCE_UNAVAILABLE" }); + const { metrics, staleRecovery } = recordingMetrics(); + const dialcache = new DialCache({ + redis: { client: redis, readTimeoutMs: 1_000 }, + metrics, + logger: { debug: vi.fn(), error: vi.fn(), warn: vi.fn() }, + }); + const getUser = dialcache.cached(async () => { + redis.failGet = true; + throw sourceError; + }, { + keyType: "user_id", + useCase, + cacheKey: () => "123", + defaultConfig: staleConfig(), + }); + + const [settled] = await Promise.allSettled([dialcache.enable(async () => await getUser())]); + + expect(rejectionReason(settled!)).toBe(sourceError); + expect(redis.readRequests).toHaveLength(2); + expect(staleRecovery).toHaveBeenCalledOnce(); + expect(staleRecovery).toHaveBeenCalledWith(expect.objectContaining({ outcome: "read_error" })); + }); + + it("gives recovery an independent read deadline and preserves the source rejection on timeout", async () => { + const useCase = "StaleRecoveryReadTimeout"; + const redis = new HangingReadRedis(2); + const sourceError = Object.freeze({ code: "SOURCE_UNAVAILABLE" }); + const { metrics, staleRecovery } = recordingMetrics(); + const dialcache = new DialCache({ + redis: { client: redis, readTimeoutMs: 10 }, + metrics, + logger: { debug: vi.fn(), error: vi.fn(), warn: vi.fn() }, + }); + const getUser = dialcache.cached(async () => { + throw sourceError; + }, { + keyType: "user_id", + useCase, + cacheKey: () => "123", + defaultConfig: staleConfig(), + }); + + const result = Promise.allSettled([dialcache.enable(async () => await getUser())]); + await vi.advanceTimersByTimeAsync(0); + expect(redis.readRequests).toHaveLength(2); + expect(redis.readContexts[1]?.signal.aborted).toBe(false); + await vi.advanceTimersByTimeAsync(10); + const [settled] = await result; + + expect(rejectionReason(settled!)).toBe(sourceError); + expect(redis.readContexts[1]?.timeoutMs).toBe(10); + expect(redis.readContexts[1]?.signal.aborted).toBe(true); + expect(staleRecovery).toHaveBeenCalledOnce(); + expect(staleRecovery).toHaveBeenCalledWith(expect.objectContaining({ outcome: "read_timeout" })); + }); + + it("does not retry Redis when the initial read times out", async () => { + const useCase = "StaleRecoveryInitialReadTimeout"; + const redis = new HangingReadRedis(1); + const sourceError = Object.freeze({ code: "SOURCE_UNAVAILABLE" }); + const { metrics, staleRecovery } = recordingMetrics(); + const dialcache = new DialCache({ + redis: { client: redis, readTimeoutMs: 10 }, + metrics, + logger: { debug: vi.fn(), error: vi.fn(), warn: vi.fn() }, + }); + const getUser = dialcache.cached(async () => { + throw sourceError; + }, { + keyType: "user_id", + useCase, + cacheKey: () => "123", + defaultConfig: staleConfig(), + }); + + const result = Promise.allSettled([dialcache.enable(async () => await getUser())]); + await vi.advanceTimersByTimeAsync(10); + const [settled] = await result; + + expect(rejectionReason(settled!)).toBe(sourceError); + expect(redis.readRequests).toHaveLength(1); + expect(redis.readContexts[0]?.signal.aborted).toBe(true); + expect(staleRecovery).not.toHaveBeenCalled(); + }); + + it("serves stale after the fallback deadline and ignores the late source result", async () => { + const useCase = "StaleRecoveryFallbackTimeout"; + const redis = new RecordingRedis(); + seedStale(redis, useCase, { id: "123", version: 1 }); + const sourceGate = deferred<{ readonly id: string; readonly version: number }>(); + const sourceStarted = deferred(); + const { metrics, staleRecovery } = recordingMetrics(); + const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 100 }, metrics }); + const getUser = dialcache.cached(async () => { + sourceStarted.resolve(); + return await sourceGate.promise; + }, { + keyType: "user_id", + useCase, + cacheKey: () => "123", + fallbackTimeoutMs: 10, + defaultConfig: staleConfig(), + }); + + const result = dialcache.enable(async () => await getUser()); + await sourceStarted.promise; + await vi.advanceTimersByTimeAsync(10); + + await expect(result).resolves.toEqual({ id: "123", version: 1 }); + expect(redis.readRequests).toHaveLength(2); + expect(redis.setCalls).toBe(0); + expect(staleRecovery).toHaveBeenCalledOnce(); + expect(staleRecovery).toHaveBeenCalledWith(expect.objectContaining({ outcome: "served" })); + + sourceGate.resolve({ id: "123", version: 2 }); + await vi.advanceTimersByTimeAsync(0); + expect(redis.setCalls).toBe(0); + }); + + it("classifies recovery deserialization failure and never retries a normal deserialization miss", async () => { + const recoveryUseCase = "StaleRecoveryDeserializeError"; + const normalUseCase = "StaleRecoveryInitialDeserializeError"; + const redis = new RecordingRedis(); + seedStale(redis, recoveryUseCase, { id: "123" }); + redis.setRaw( + redisValueKey(normalUseCase), + encodeFrame({ id: "123" }, Date.now()), + MAX_AGE_SEC * 1_000, + ); + const sourceError = Object.freeze({ code: "SOURCE_UNAVAILABLE" }); + const serializer: Serializer<{ readonly id: string }> = { + dump: vi.fn(async (value) => JSON.stringify(value)), + load: vi.fn(async () => { + throw new Error("cannot decode"); + }), + }; + const { metrics, staleRecovery } = recordingMetrics(); + const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 }, metrics }); + const recover = dialcache.cached(async (): Promise<{ readonly id: string }> => { + throw sourceError; + }, { + keyType: "user_id", + useCase: recoveryUseCase, + cacheKey: () => "123", + defaultConfig: staleConfig(), + serializer, + }); + const initialFailure = dialcache.cached(async (): Promise<{ readonly id: string }> => { + throw sourceError; + }, { + keyType: "user_id", + useCase: normalUseCase, + cacheKey: () => "123", + defaultConfig: staleConfig(), + serializer, + }); + + const [recoverySettled] = await Promise.allSettled([dialcache.enable(async () => await recover())]); + expect(rejectionReason(recoverySettled!)).toBe(sourceError); + expect(staleRecovery).toHaveBeenCalledOnce(); + expect(staleRecovery).toHaveBeenCalledWith(expect.objectContaining({ + useCase: recoveryUseCase, + outcome: "deserialization_error", + })); + + const readsBeforeInitialFailure = redis.readRequests.length; + const [initialSettled] = await Promise.allSettled([ + dialcache.enable(async () => await initialFailure()), + ]); + expect(rejectionReason(initialSettled!)).toBe(sourceError); + expect(redis.readRequests).toHaveLength(readsBeforeInitialFailure + 1); + expect(staleRecovery).toHaveBeenCalledTimes(1); + expect(staleRecovery).not.toHaveBeenCalledWith(expect.objectContaining({ useCase: normalUseCase })); + }); + + it("rechecks tracked invalidation after the source attempt and blocks recovery", async () => { + const useCase = "StaleRecoveryInvalidatedDuringSource"; + const redis = new RecordingRedis(); + seedStale(redis, useCase, { id: "123", version: 1 }, true); + redis.setRaw(watermarkKey(), "0", MAX_AGE_SEC * 1_000 + 60_000); + const sourceError = Object.freeze({ code: "SOURCE_UNAVAILABLE" }); + const sourceStarted = deferred(); + const sourceGate = deferred<{ readonly id: string; readonly version: number }>(); + const { metrics, staleRecovery } = recordingMetrics(); + const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 }, metrics }); + const getUser = dialcache.cached(async () => { + sourceStarted.resolve(); + return await sourceGate.promise; + }, { + keyType: "user_id", + useCase, + cacheKey: () => "123", + trackForInvalidation: true, + defaultConfig: staleConfig(), + }); + + const result = Promise.allSettled([dialcache.enable(async () => await getUser())]); + await sourceStarted.promise; + await dialcache.invalidateRemote("user_id", "123"); + sourceGate.reject(sourceError); + const [settled] = await result; + + expect(rejectionReason(settled!)).toBe(sourceError); + expect(redis.readRequests).toHaveLength(2); + expect(redis.readRequests.every(({ watermarkKey: key }) => key === watermarkKey())).toBe(true); + expect(staleRecovery).toHaveBeenCalledOnce(); + expect(staleRecovery).toHaveBeenCalledWith(expect.objectContaining({ outcome: "miss" })); + }); + + it("recovers cached undefined without starting shadow validation", async () => { + const useCase = "StaleRecoveryUndefinedNoShadow"; + const redis = new RecordingRedis(); + redis.setRaw( + redisValueKey(useCase, "123", true), + encodeFrame("__dialcache_json_undefined_v1__", Date.now() - 2_000), + MAX_AGE_SEC * 1_000, + ); + redis.setRaw(watermarkKey(), "0", MAX_AGE_SEC * 1_000 + 60_000); + const { metrics, staleRecovery, shadowValidation } = recordingMetrics(); + const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 }, metrics }); + const getOptional = dialcache.cached(async (): Promise => { + throw new Error("source unavailable"); + }, { + keyType: "user_id", + useCase, + cacheKey: () => "123", + trackForInvalidation: true, + defaultConfig: new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: FRESH_TTL_SEC }, + ramp: { [CacheLayer.REMOTE]: 100 }, + staleOnErrorMaxAgeSec: MAX_AGE_SEC, + shadow: { ramp: 100 }, + }), + }); + + await expect(dialcache.enable(async () => await getOptional())).resolves.toBeUndefined(); + await vi.advanceTimersByTimeAsync(0); + + expect(redis.readRequests).toHaveLength(2); + expect(redis.setCalls).toBe(0); + expect(staleRecovery).toHaveBeenCalledWith(expect.objectContaining({ outcome: "served" })); + expect(shadowValidation).not.toHaveBeenCalled(); + }); + + it("applies a lowered runtime recovery maximum to an existing retained frame immediately", async () => { + const useCase = "StaleRecoveryLoweredRuntimeMaximum"; + const redis = new RecordingRedis(); + redis.setRaw( + redisValueKey(useCase), + encodeFrame({ id: "123", version: 1 }, Date.now() - 5_000), + 10_000, + ); + let maxAgeSec = 10; + const sourceError = Object.freeze({ code: "SOURCE_UNAVAILABLE" }); + const dialcache = new DialCache({ + redis: { client: redis, readTimeoutMs: 1_000 }, + cacheConfigProvider: async () => new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: FRESH_TTL_SEC }, + ramp: { [CacheLayer.REMOTE]: 100 }, + staleOnErrorMaxAgeSec: maxAgeSec, + }), + }); + const getUser = dialcache.cached(async (): Promise<{ readonly id: string; readonly version: number }> => { + throw sourceError; + }, { + keyType: "user_id", + useCase, + cacheKey: () => "123", + }); + + await expect(dialcache.enable(async () => await getUser())).resolves.toEqual({ id: "123", version: 1 }); + maxAgeSec = 3; + const [settled] = await Promise.allSettled([dialcache.enable(async () => await getUser())]); + + expect(rejectionReason(settled!)).toBe(sourceError); + expect(redis.readRequests.map(({ maxAgeMs }) => maxAgeMs)).toEqual([ + 1_000, + 10_000, + 1_000, + 3_000, + ]); + }); + + it("does not resurrect or extend a frame after raising the runtime recovery maximum", async () => { + const useCase = "StaleRecoveryRaisedRuntimeMaximum"; + const redis = new RecordingRedis(); + let maxAgeSec = 3; + const sourceError = Object.freeze({ code: "SOURCE_UNAVAILABLE" }); + let sourceCalls = 0; + const source = vi.fn(async (): Promise<{ readonly id: string; readonly version: number }> => { + if (++sourceCalls === 1) { + return { id: "123", version: 1 }; + } + throw sourceError; + }); + const dialcache = new DialCache({ + redis: { client: redis, readTimeoutMs: 1_000 }, + cacheConfigProvider: async () => new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: FRESH_TTL_SEC }, + ramp: { [CacheLayer.REMOTE]: 100 }, + staleOnErrorMaxAgeSec: maxAgeSec, + }), + }); + const getUser = dialcache.cached(source, { + keyType: "user_id", + useCase, + cacheKey: () => "123", + }); + + await expect(dialcache.enable(async () => await getUser())).resolves.toEqual({ id: "123", version: 1 }); + expect(redis.ttlMs(redisValueKey(useCase))).toBe(3_000); + await vi.advanceTimersByTimeAsync(3_000); + maxAgeSec = 10; + const [settled] = await Promise.allSettled([dialcache.enable(async () => await getUser())]); + + expect(rejectionReason(settled!)).toBe(sourceError); + expect(redis.readRequests.map(({ maxAgeMs }) => maxAgeMs)).toEqual([1_000, 1_000, 10_000]); + expect(redis.setCalls).toBe(1); + }); + + it("coalesces recovery and does not populate process-local cache", async () => { + const useCase = "StaleRecoveryProcessCoalescing"; + const redis = new RecordingRedis(); + seedStale(redis, useCase, { id: "123", version: 1 }); + const sourceError = Object.freeze({ code: "SOURCE_UNAVAILABLE" }); + const source = vi.fn(async () => { + throw sourceError; + }); + const { metrics, staleRecovery } = recordingMetrics(); + const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 }, metrics }); + const localCache = (dialcache as unknown as { + readonly localCache: { put: (...args: unknown[]) => Promise }; + }).localCache; + const localPut = vi.spyOn(localCache, "put"); + const getUser = dialcache.cached(source, { + keyType: "user_id", + useCase, + cacheKey: () => "123", + defaultConfig: staleConfig({ local: true }), + }); + + const values = await dialcache.enable(async () => await Promise.all([getUser(), getUser(), getUser()])); + + expect(source).toHaveBeenCalledOnce(); + expect(redis.readRequests).toHaveLength(2); + expect(values[1]).toBe(values[0]); + expect(values[2]).toBe(values[0]); + expect(localPut).not.toHaveBeenCalled(); + expect(staleRecovery).toHaveBeenCalledTimes(1); + + await expect(dialcache.enable(async () => await getUser())).resolves.toEqual(values[0]); + expect(source).toHaveBeenCalledTimes(2); + expect(redis.readRequests).toHaveLength(4); + expect(localPut).not.toHaveBeenCalled(); + }); + + it("memoizes a recovered reference only within the active request-local scope", async () => { + const useCase = "StaleRecoveryRequestLocal"; + const redis = new RecordingRedis(); + seedStale(redis, useCase, { id: "123", version: 1 }); + const source = vi.fn(async () => { + throw new Error("source unavailable"); + }); + const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 } }); + const getUser = dialcache.cached(source, { + keyType: "user_id", + useCase, + cacheKey: () => "123", + defaultConfig: staleConfig({ requestLocal: true }), + }); + + const [first, second] = await dialcache.enable(async () => { + const firstValue = await getUser(); + const secondValue = await getUser(); + return [firstValue, secondValue] as const; + }); + + expect(second).toBe(first); + expect(source).toHaveBeenCalledOnce(); + expect(redis.readRequests).toHaveLength(2); + + await expect(dialcache.enable(async () => await getUser())).resolves.toEqual(first); + expect(source).toHaveBeenCalledTimes(2); + expect(redis.readRequests).toHaveLength(4); + }); + + it("decompresses a retained value during stale recovery", async () => { + const useCase = "StaleRecoveryCompressed"; + const redis = new RecordingRedis(); + const retained = { id: "123", blob: "compressible stale payload ".repeat(1_024) }; + const sourceError = Object.freeze({ code: "SOURCE_UNAVAILABLE" }); + let available = true; + const source = vi.fn(async () => { + if (available) { + return retained; + } + throw sourceError; + }); + const { metrics, staleRecovery } = recordingMetrics(); + const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 }, metrics }); + const getUser = dialcache.cached(source, { + keyType: "user_id", + useCase, + cacheKey: () => "123", + defaultConfig: staleConfig(), + }); + + await expect(dialcache.enable(async () => await getUser())).resolves.toBe(retained); + const stored = decodeFrame(redis.raw(redisValueKey(useCase))).payload; + expect(Buffer.isBuffer(stored) && stored[0]).toBe(MARKER_ZSTD_UTF8); + + available = false; + await vi.advanceTimersByTimeAsync(2_000); + await expect(dialcache.enable(async () => await getUser())).resolves.toEqual(retained); + + expect(redis.readRequests.map(({ maxAgeMs }) => maxAgeMs)).toEqual([1_000, 1_000, 10_000]); + expect(staleRecovery).toHaveBeenCalledWith(expect.objectContaining({ outcome: "served" })); + }); + + it("contains a corrupt compression envelope and preserves the source rejection", async () => { + const useCase = "StaleRecoveryCorruptCompression"; + const redis = new RecordingRedis(); + redis.setRaw( + redisValueKey(useCase), + encodeFrame( + Buffer.concat([Buffer.from([MARKER_ZSTD_UTF8]), Buffer.from("not a zstd frame")]), + Date.now() - 2_000, + 1, + ), + MAX_AGE_SEC * 1_000, + ); + const sourceError = Object.freeze({ code: "SOURCE_UNAVAILABLE" }); + const { metrics, staleRecovery } = recordingMetrics(); + const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 }, metrics }); + const getUser = dialcache.cached(async (): Promise<{ readonly id: string }> => { + throw sourceError; + }, { + keyType: "user_id", + useCase, + cacheKey: () => "123", + defaultConfig: staleConfig(), + }); + + const [settled] = await Promise.allSettled([dialcache.enable(async () => await getUser())]); + + expect(rejectionReason(settled!)).toBe(sourceError); + expect(staleRecovery).toHaveBeenCalledWith(expect.objectContaining({ outcome: "deserialization_error" })); + expect(redis.readRequests.map(({ maxAgeMs }) => maxAgeMs)).toEqual([1_000, 10_000]); + }); + + it("runs independent stale recovery chains when coalescing is disabled", async () => { + const useCase = "StaleRecoveryCoalescingDisabled"; + const redis = new RecordingRedis(); + const retained = { id: "123", version: 1 }; + seedStale(redis, useCase, retained); + const source = vi.fn(async () => { + throw new Error("source unavailable"); + }); + const { metrics, staleRecovery } = recordingMetrics(); + const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 }, metrics }); + const getUser = dialcache.cached(source, { + keyType: "user_id", + useCase, + cacheKey: () => "123", + defaultConfig: new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: FRESH_TTL_SEC }, + ramp: { [CacheLayer.REMOTE]: 100 }, + staleOnErrorMaxAgeSec: MAX_AGE_SEC, + coalesce: false, + }), + }); + + const values = await dialcache.enable(async () => await Promise.all([getUser(), getUser()])); + + expect(values).toEqual([retained, retained]); + expect(source).toHaveBeenCalledTimes(2); + expect(redis.readRequests.map(({ maxAgeMs }) => maxAgeMs)).toEqual([1_000, 1_000, 10_000, 10_000]); + expect(staleRecovery).toHaveBeenCalledTimes(2); + }); +}); diff --git a/test/fake-redis.ts b/test/fake-redis.ts index 4633b99..93f1ed7 100644 --- a/test/fake-redis.ts +++ b/test/fake-redis.ts @@ -12,6 +12,7 @@ const FRAME_VERSION = 1; const ENCODING_OFFSET = 9; const PAYLOAD_OFFSET = 10; const WATERMARK_TTL_MARGIN_MS = 60_000; +const MAX_SUPPORTED_DURATION_MS = 365 * 24 * 60 * 60 * 1_000; interface StoredValue { value: Buffer; @@ -19,6 +20,7 @@ interface StoredValue { } export class FakeRedis implements DialCacheRedisClient { + readonly enforcesMaxAge = true as const; readonly values = new Map(); getCalls = 0; mGetCalls = 0; @@ -28,7 +30,8 @@ export class FakeRedis implements DialCacheRedisClient { failWatermarkGet = false; getGate: Promise | null = null; - async read({ valueKey, watermarkKey }: RedisReadRequest): Promise { + async read({ valueKey, watermarkKey, maxAgeMs }: RedisReadRequest): Promise { + assertValidMaxAgeMs(maxAgeMs); if (watermarkKey === undefined) { this.getCalls += 1; } else { @@ -36,7 +39,7 @@ export class FakeRedis implements DialCacheRedisClient { } await this.waitForRead(); this.throwIfReadFails(watermarkKey !== undefined); - return this.readPayload(valueKey, watermarkKey ?? null); + return this.readPayload(valueKey, watermarkKey ?? null, maxAgeMs); } async write({ @@ -123,13 +126,21 @@ export class FakeRedis implements DialCacheRedisClient { } } - private readPayload(valueKey: string, watermarkKey: string | null): DecodedRedisFrame | null { + private readPayload( + valueKey: string, + watermarkKey: string | null, + maxAgeMs: number, + ): DecodedRedisFrame | null { const raw = this.readRaw(valueKey); if (raw === null || raw.length < PAYLOAD_OFFSET || raw[0] !== FRAME_VERSION) { return null; } const createdAtMs = Number(readTimestamp(raw)); + const ageMs = Date.now() - createdAtMs; + if (ageMs < 0 || ageMs >= maxAgeMs) { + return null; + } if (watermarkKey !== null) { let watermark: number | null; try { @@ -207,6 +218,17 @@ export class FakeRedis implements DialCacheRedisClient { } } +function assertValidMaxAgeMs(maxAgeMs: unknown): asserts maxAgeMs is number { + if ( + typeof maxAgeMs !== "number" + || !Number.isSafeInteger(maxAgeMs) + || maxAgeMs <= 0 + || maxAgeMs > MAX_SUPPORTED_DURATION_MS + ) { + throw new RangeError("Invalid DialCache Redis maxAgeMs"); + } +} + export function encodeFrame(value: unknown, createdAtMs = Date.now(), encoding = 0): Buffer { const timestamp = Buffer.alloc(8); timestamp.writeBigUInt64BE(BigInt(createdAtMs)); diff --git a/test/node-redis.test.ts b/test/node-redis.test.ts index 258e3aa..6176380 100644 --- a/test/node-redis.test.ts +++ b/test/node-redis.test.ts @@ -8,7 +8,10 @@ import { DialCacheRedisProtocolError, } from "../src/index.js"; import { createNodeRedisDialCacheClient, dialcacheRedisScripts } from "../src/node-redis.js"; -import { INVALIDATE_CACHE_SCRIPT } from "../src/redis-protocol.js"; +import { + INVALIDATE_CACHE_SCRIPT, + WRITE_UNTRACKED_STAMP_SCRIPT, +} from "../src/redis-protocol.js"; const INVALID_WRITE_REPLIES: readonly unknown[] = [ -1, @@ -29,7 +32,9 @@ interface FakeReplies { readonly get?: unknown; readonly mGet?: unknown; readonly set?: unknown; + readonly time?: unknown; readonly eval?: unknown; + readonly untrackedStamp?: unknown; readonly stamp?: unknown; readonly invalidate?: unknown; } @@ -43,11 +48,22 @@ function fakeClient(replies: FakeReplies = {}) { if (args[0] === "SET") { return Object.hasOwn(replies, "set") ? replies.set : "OK"; } + if (args[0] === "TIME") { + return Object.hasOwn(replies, "time") + ? replies.time + : [Buffer.from("1"), Buffer.from("0")]; + } if (args[0] === "EVAL") { return Object.hasOwn(replies, "eval") ? replies.eval : 1; } - return Object.hasOwn(replies, "mGet") ? replies.mGet : [null, null]; + if (args[0] === "MGET") { + return Object.hasOwn(replies, "mGet") ? replies.mGet : [null, null]; + } + return Object.hasOwn(replies, "get") ? replies.get : null; }), + dialcacheWriteUntrackedStamp: vi.fn( + async () => Object.hasOwn(replies, "untrackedStamp") ? replies.untrackedStamp : 1, + ), dialcacheWriteTrackedStamp: vi.fn(async () => Object.hasOwn(replies, "stamp") ? replies.stamp : 1), dialcacheInvalidate: vi.fn(async () => Object.hasOwn(replies, "invalidate") ? replies.invalidate : 1), }; @@ -86,9 +102,23 @@ describe("node-redis adapter", () => { it("provides the expected arguments for every bundled mutation script", () => { const nonce = Buffer.from("01234567"); expect(Object.keys(dialcacheRedisScripts)).toEqual([ + "dialcacheWriteUntrackedStamp", "dialcacheWriteTrackedStamp", "dialcacheInvalidate", ]); + expect( + dialcacheRedisScripts.dialcacheWriteUntrackedStamp.transformArguments( + "plain:value", + nonce, + ), + ).toEqual(["plain:value", nonce]); + expect(dialcacheRedisScripts.dialcacheWriteUntrackedStamp.SCRIPT) + .toBe(WRITE_UNTRACKED_STAMP_SCRIPT); + expect(dialcacheRedisScripts.dialcacheWriteUntrackedStamp.NUMBER_OF_KEYS).toBe(1); + expect(WRITE_UNTRACKED_STAMP_SCRIPT).toContain('redis.call("TIME")'); + expect(WRITE_UNTRACKED_STAMP_SCRIPT).toContain('redis.call("SETRANGE", KEYS[1]'); + expect(WRITE_UNTRACKED_STAMP_SCRIPT).not.toContain("ARGV[2]"); + expect(WRITE_UNTRACKED_STAMP_SCRIPT).not.toContain('"PX"'); expect( dialcacheRedisScripts.dialcacheWriteTrackedStamp.transformArguments( "tracked:{id}:value", @@ -114,6 +144,7 @@ describe("node-redis adapter", () => { () => createNodeRedisDialCacheClient({ get: vi.fn(), sendCommand: vi.fn(), + dialcacheWriteUntrackedStamp: vi.fn(), dialcacheWriteTrackedStamp: vi.fn(), } as never), ).toThrow(TypeError); @@ -121,6 +152,15 @@ describe("node-redis adapter", () => { () => createNodeRedisDialCacheClient({ get: vi.fn(), sendCommand: vi.fn(), + dialcacheWriteUntrackedStamp: vi.fn(), + dialcacheInvalidate: vi.fn(), + } as never), + ).toThrow(TypeError); + expect( + () => createNodeRedisDialCacheClient({ + get: vi.fn(), + sendCommand: vi.fn(), + dialcacheWriteTrackedStamp: vi.fn(), dialcacheInvalidate: vi.fn(), } as never), ).toThrow(TypeError); @@ -136,12 +176,17 @@ describe("node-redis adapter", () => { }); const adapter = createNodeRedisDialCacheClient(client as never); - await expect(adapter.read({ valueKey: "plain:value" })).resolves.toEqual({ + expect(adapter.enforcesMaxAge).toBe(true); + await expect(adapter.read({ valueKey: "plain:value", maxAgeMs: 10_000 })).resolves.toEqual({ payload: "plain", createdAtMs: 1, }); await expect( - adapter.read({ valueKey: "tracked:{id}:value", watermarkKey: "tracked:{id}:watermark" }), + adapter.read({ + valueKey: "tracked:{id}:value", + watermarkKey: "tracked:{id}:watermark", + maxAgeMs: 10_000, + }), ).resolves.toEqual({ payload: Buffer.from([0, 0xff]), createdAtMs: 2 }); await expect( adapter.write({ valueKey: "plain:value", cacheTtlMs: 1_000, value: "plain" }), @@ -159,16 +204,15 @@ describe("node-redis adapter", () => { ).resolves.toBeUndefined(); }); - it("writes untracked frames with one native SET", async () => { + it("pairs an untracked placeholder SET with its server-time stamp", async () => { const client = fakeClient(); const adapter = createNodeRedisDialCacheClient(client as never); - const before = Date.now(); await expect( adapter.write({ valueKey: "plain:value", cacheTtlMs: 1_000, value: "plain" }), ).resolves.toBe(true); - const after = Date.now(); expect(client.dialcacheWriteTrackedStamp).not.toHaveBeenCalled(); + expect(client.dialcacheWriteUntrackedStamp).toHaveBeenCalledTimes(1); expect(client.sendCommand).toHaveBeenCalledTimes(1); const [args, options] = client.sendCommand.mock.calls[0] as [Array, unknown]; expect(args[0]).toBe("SET"); @@ -176,12 +220,13 @@ describe("node-redis adapter", () => { expect(args[3]).toBe("PX"); expect(args[4]).toBe("1000"); const frame = args[2] as Buffer; - expect(frame[0]).toBe(1); + expect(frame[0]).toBe(0); expect(frame[9]).toBe(0); expect(frame.subarray(10).toString("utf8")).toBe("plain"); - const createdAtMs = Number(frame.readBigUInt64BE(1)); - expect(createdAtMs).toBeGreaterThanOrEqual(before); - expect(createdAtMs).toBeLessThanOrEqual(after); + expect(client.dialcacheWriteUntrackedStamp).toHaveBeenCalledWith( + "plain:value", + frame.subarray(1, 9), + ); expect(options).toMatchObject({ returnBuffers: true }); }); @@ -237,6 +282,19 @@ describe("node-redis adapter", () => { await expect(write).rejects.toBeInstanceOf(DialCacheRedisPlaceholderLostError); }); + it("fails an untracked write whose placeholder was lost before the stamp", async () => { + const adapter = createNodeRedisDialCacheClient(fakeClient({ untrackedStamp: 2 }) as never); + const write = adapter.write({ + valueKey: "plain:value", + cacheTtlMs: 1_000, + value: "plain", + }); + await expect(write).rejects.toThrow( + "DialCache untracked write lost its placeholder before the stamp", + ); + await expect(write).rejects.toBeInstanceOf(DialCacheRedisPlaceholderLostError); + }); + it("issues the stamp before the placeholder SET settles", async () => { const client = fakeClient(); let resolveSet: ((value: string) => void) | undefined; @@ -323,6 +381,7 @@ describe("node-redis adapter", () => { ).rejects.toThrow(RangeError); } expect(client.sendCommand).not.toHaveBeenCalled(); + expect(client.dialcacheWriteUntrackedStamp).not.toHaveBeenCalled(); expect(client.dialcacheWriteTrackedStamp).not.toHaveBeenCalled(); await adapter.write({ @@ -355,6 +414,17 @@ describe("node-redis adapter", () => { })).rejects.toBe(failure); expect(client.dialcacheWriteTrackedStamp).toHaveBeenCalledTimes(1); + const untrackedFailure = new Error("ERR untracked SET failed"); + const untrackedClient = fakeClient(); + untrackedClient.sendCommand.mockRejectedValueOnce(untrackedFailure); + const untrackedAdapter = createNodeRedisDialCacheClient(untrackedClient as never); + await expect(untrackedAdapter.write({ + valueKey: "plain:value", + cacheTtlMs: 1_000, + value: "plain", + })).rejects.toBe(untrackedFailure); + expect(untrackedClient.dialcacheWriteUntrackedStamp).toHaveBeenCalledTimes(1); + const stampFailure = new Error("ERR invalid DialCache watermark"); const stampClient = fakeClient(); stampClient.dialcacheWriteTrackedStamp.mockRejectedValueOnce(stampFailure); @@ -379,6 +449,18 @@ describe("node-redis adapter", () => { })), "Invalid DialCache Redis SET reply; expected OK", ); + + const untrackedCombinedClient = fakeClient({ set: "QUEUED" }); + untrackedCombinedClient.dialcacheWriteUntrackedStamp.mockRejectedValueOnce(new Error("ERR stamp")); + const untrackedCombinedAdapter = createNodeRedisDialCacheClient(untrackedCombinedClient as never); + await expectProtocolError( + Promise.resolve(untrackedCombinedAdapter.write({ + valueKey: "plain:value", + cacheTtlMs: 1_000, + value: "plain", + })), + "Invalid DialCache Redis SET reply; expected OK", + ); }); it("passes the cooperative read signal through node-redis command options", async () => { @@ -387,15 +469,24 @@ describe("node-redis adapter", () => { const controller = new AbortController(); const context = { timeoutMs: 25, signal: controller.signal } as const; - await adapter.read({ valueKey: "plain:value" }, context); + await adapter.read({ valueKey: "plain:value", maxAgeMs: 10_000 }, context); await adapter.read( - { valueKey: "tracked:{id}:value", watermarkKey: "tracked:{id}:watermark" }, + { + valueKey: "tracked:{id}:value", + watermarkKey: "tracked:{id}:watermark", + maxAgeMs: 10_000, + }, context, ); - expect(client.get).toHaveBeenCalledWith( + expect(client.get).not.toHaveBeenCalled(); + expect(client.sendCommand).toHaveBeenCalledWith( + ["GET", "plain:value"], + expect.objectContaining({ returnBuffers: true, signal: controller.signal }), + ); + expect(client.sendCommand).toHaveBeenCalledWith( + ["TIME"], expect.objectContaining({ returnBuffers: true, signal: controller.signal }), - "plain:value", ); expect(client.sendCommand).toHaveBeenCalledWith( ["MGET", "tracked:{id}:value", "tracked:{id}:watermark"], @@ -403,7 +494,89 @@ describe("node-redis adapter", () => { ); }); - it("forces tracked Cluster MGET reads to the primary", async () => { + it("enforces the exact maximum age using the paired Redis TIME", async () => { + const withinBoundary = createNodeRedisDialCacheClient(fakeClient({ + get: encodeFrame("plain", { createdAtMs: 1_000 }), + time: [Buffer.from("1"), Buffer.from("999999")], + }) as never); + await expect(withinBoundary.read({ + valueKey: "plain:value", + maxAgeMs: 1_000, + })).resolves.toEqual({ payload: "plain", createdAtMs: 1_000 }); + + const atBoundary = createNodeRedisDialCacheClient(fakeClient({ + get: encodeFrame("plain", { createdAtMs: 1_000 }), + time: [Buffer.from("2"), Buffer.from("0")], + }) as never); + await expect(atBoundary.read({ + valueKey: "plain:value", + maxAgeMs: 1_000, + })).resolves.toBeNull(); + + const futureTimestamp = createNodeRedisDialCacheClient(fakeClient({ + get: encodeFrame("plain", { createdAtMs: 1_001 }), + time: [Buffer.from("1"), Buffer.from("0")], + }) as never); + await expect(futureTimestamp.read({ + valueKey: "plain:value", + maxAgeMs: 1_000, + })).resolves.toBeNull(); + }); + + it("rejects invalid maximum ages before dispatch, including on a missing key", async () => { + const client = fakeClient(); + const adapter = createNodeRedisDialCacheClient(client as never); + + for (const maxAgeMs of [ + 0, + -1, + 0.5, + Number.NaN, + Number.POSITIVE_INFINITY, + 31_536_000_001, + "100" as unknown as number, + ]) { + await expect(adapter.read({ valueKey: "missing:value", maxAgeMs })).rejects.toEqual( + new RangeError( + "DialCache Redis maxAgeMs must be a positive safe integer no greater than 31536000000", + ), + ); + } + expect(client.sendCommand).not.toHaveBeenCalled(); + expect(client.get).not.toHaveBeenCalled(); + }); + + it("enqueues each native read before TIME and validates every paired TIME reply", async () => { + const order: string[] = []; + const client = fakeClient(); + client.sendCommand.mockImplementation(async (...callArgs: unknown[]) => { + const args = (Array.isArray(callArgs[0]) ? callArgs[0] : callArgs[2]) as Array; + order.push(String(args[0])); + if (args[0] === "TIME") { + return [Buffer.from("1"), Buffer.from("0")]; + } + return args[0] === "MGET" ? [null, null] : null; + }); + const adapter = createNodeRedisDialCacheClient(client as never); + + await adapter.read({ valueKey: "plain:value", maxAgeMs: 1_000 }); + await adapter.read({ + valueKey: "tracked:{id}:value", + watermarkKey: "tracked:{id}:watermark", + maxAgeMs: 1_000, + }); + expect(order).toEqual(["GET", "TIME", "MGET", "TIME"]); + + await expect( + createNodeRedisDialCacheClient(fakeClient({ time: [Buffer.from("1")] }) as never) + .read({ valueKey: "missing:value", maxAgeMs: 1_000 }), + ).rejects.toMatchObject({ + name: "DialCacheRedisPayloadError", + message: "Invalid DialCache Redis TIME reply; expected two unsigned decimal bulk strings", + }); + }); + + it("routes native reads and their TIME commands to the same Cluster primaries", async () => { const client = fakeCluster({ mGet: [encodeFrame("tracked", { createdAtMs: 2 }), Buffer.from("1")], }); @@ -411,9 +584,17 @@ describe("node-redis adapter", () => { const controller = new AbortController(); await expect(adapter.read( - { valueKey: "tracked:{id}:value", watermarkKey: "tracked:{id}:watermark" }, + { + valueKey: "tracked:{id}:value", + watermarkKey: "tracked:{id}:watermark", + maxAgeMs: 10_000, + }, { timeoutMs: 25, signal: controller.signal }, )).resolves.toEqual({ payload: "tracked", createdAtMs: 2 }); + await expect(adapter.read({ + valueKey: "plain:value", + maxAgeMs: 10_000, + })).resolves.toBeNull(); expect(client.sendCommand).toHaveBeenCalledWith( "tracked:{id}:value", @@ -421,6 +602,24 @@ describe("node-redis adapter", () => { ["MGET", "tracked:{id}:value", "tracked:{id}:watermark"], expect.objectContaining({ returnBuffers: true, signal: controller.signal }), ); + expect(client.sendCommand).toHaveBeenCalledWith( + "plain:value", + false, + ["GET", "plain:value"], + expect.objectContaining({ returnBuffers: true }), + ); + expect(client.sendCommand).toHaveBeenCalledWith( + "plain:value", + false, + ["TIME"], + expect.objectContaining({ returnBuffers: true }), + ); + expect(client.sendCommand).toHaveBeenCalledWith( + "tracked:{id}:value", + false, + ["TIME"], + expect.objectContaining({ returnBuffers: true, signal: controller.signal }), + ); }); it("does not mistake unrelated standalone metadata for the Cluster topology marker", async () => { @@ -435,18 +634,23 @@ describe("node-redis adapter", () => { await expect(adapter.read({ valueKey: "tracked:{id}:value", watermarkKey: "tracked:{id}:watermark", + maxAgeMs: 10_000, })).resolves.toEqual({ payload: "tracked", createdAtMs: 2 }); expect(client.sendCommand).toHaveBeenCalledWith( ["MGET", "tracked:{id}:value", "tracked:{id}:watermark"], expect.objectContaining({ returnBuffers: true }), ); + expect(client.sendCommand).toHaveBeenCalledWith( + ["TIME"], + expect.objectContaining({ returnBuffers: true }), + ); }); it("rejects malformed native read reply shapes", async () => { await expect( createNodeRedisDialCacheClient(fakeClient({ get: "not-bytes" }) as never) - .read({ valueKey: "plain:value" }), + .read({ valueKey: "plain:value", maxAgeMs: 10_000 }), ).rejects.toMatchObject({ name: "DialCacheRedisPayloadError", message: "Invalid DialCache Redis read reply; expected a bulk string or null", @@ -460,7 +664,11 @@ describe("node-redis adapter", () => { for (const reply of malformedMGetEnvelopes) { await expect( createNodeRedisDialCacheClient(fakeClient({ mGet: reply }) as never) - .read({ valueKey: "tracked:{id}:value", watermarkKey: "tracked:{id}:watermark" }), + .read({ + valueKey: "tracked:{id}:value", + watermarkKey: "tracked:{id}:watermark", + maxAgeMs: 10_000, + }), ).rejects.toMatchObject({ name: "DialCacheRedisPayloadError", message: "Invalid DialCache Redis tracked read reply; expected an array with two entries", @@ -470,7 +678,11 @@ describe("node-redis adapter", () => { for (const reply of [["not-bytes", null], [null, 0]]) { await expect( createNodeRedisDialCacheClient(fakeClient({ mGet: reply }) as never) - .read({ valueKey: "tracked:{id}:value", watermarkKey: "tracked:{id}:watermark" }), + .read({ + valueKey: "tracked:{id}:value", + watermarkKey: "tracked:{id}:watermark", + maxAgeMs: 10_000, + }), ).rejects.toMatchObject({ name: "DialCacheRedisPayloadError", message: "Invalid DialCache Redis read reply; expected a bulk string or null", @@ -480,8 +692,21 @@ describe("node-redis adapter", () => { it("rejects every out-of-domain reply returned by a node-redis client", async () => { const writeMessage = "Invalid DialCache Redis write reply; expected integer 0, 1, or 2"; + const untrackedWriteMessage = "Invalid DialCache Redis untracked write reply; expected integer 1 or 2"; const invalidationMessage = "Invalid DialCache Redis invalidate reply; expected integer 1"; + for (const reply of [0, ...INVALID_WRITE_REPLIES]) { + const untracked = createNodeRedisDialCacheClient(fakeClient({ untrackedStamp: reply }) as never); + await expectProtocolError( + Promise.resolve(untracked.write({ + valueKey: "plain:value", + cacheTtlMs: 1_000, + value: "plain", + })), + untrackedWriteMessage, + ); + } + for (const reply of INVALID_WRITE_REPLIES) { const tracked = createNodeRedisDialCacheClient(fakeClient({ stamp: reply }) as never); await expectProtocolError( @@ -649,11 +874,18 @@ describe("node-redis adapter", () => { }); it("validates replies at the public node-redis script transform boundary", () => { + expect(dialcacheRedisScripts.dialcacheWriteUntrackedStamp.transformReply(1)).toBe(1); + expect(dialcacheRedisScripts.dialcacheWriteUntrackedStamp.transformReply(2)).toBe(2); expect(dialcacheRedisScripts.dialcacheWriteTrackedStamp.transformReply(0)).toBe(0); expect(dialcacheRedisScripts.dialcacheWriteTrackedStamp.transformReply(1)).toBe(1); expect(dialcacheRedisScripts.dialcacheWriteTrackedStamp.transformReply(2)).toBe(2); expect(dialcacheRedisScripts.dialcacheInvalidate.transformReply(1)).toBe(1); + for (const reply of [0, ...INVALID_WRITE_REPLIES]) { + expect(() => dialcacheRedisScripts.dialcacheWriteUntrackedStamp.transformReply(reply as number)).toThrow( + DialCacheRedisProtocolError, + ); + } for (const reply of INVALID_WRITE_REPLIES) { expect(() => dialcacheRedisScripts.dialcacheWriteTrackedStamp.transformReply(reply as number)).toThrow( DialCacheRedisProtocolError, diff --git a/test/prometheus.test.ts b/test/prometheus.test.ts index bcfebbc..c6a20a2 100644 --- a/test/prometheus.test.ts +++ b/test/prometheus.test.ts @@ -16,6 +16,7 @@ import { type DisabledReason, type MetricErrorKind, type ShadowValidationOutcome, + type StaleRecoveryOutcome, } from "../src/index.js"; import { PrometheusDialCacheMetrics, createPrometheusDialCacheMetrics } from "../src/prometheus.js"; import { FakeRedis } from "./fake-redis.js"; @@ -82,6 +83,13 @@ const SHADOW_VALIDATION_OUTCOMES: Readonly timeout: true, dropped: true, }; +const STALE_RECOVERY_OUTCOMES: Readonly> = { + served: true, + miss: true, + read_error: true, + read_timeout: true, + deserialization_error: true, +}; interface IncompatibleCollectorCase { readonly schemaPart: string; @@ -182,6 +190,12 @@ describe("Prometheus metrics adapter", () => { keyType: labels.keyType, outcome: "match", }); + metrics.staleRecovery({ + cacheNamespace: labels.cacheNamespace, + useCase: labels.useCase, + keyType: labels.keyType, + outcome: "served", + }); metrics.observeShadowValueAge( { cacheNamespace: labels.cacheNamespace, @@ -252,6 +266,10 @@ describe("Prometheus metrics adapter", () => { VALUE_AGE_BUCKETS, ), histogramSchema("schema_dialcache_size_histogram", ["cache_namespace", "use_case", "key_type", "layer"], SIZE_BUCKETS), + counterSchema( + "schema_dialcache_stale_recovery_counter", + ["cache_namespace", "use_case", "key_type", "outcome"], + ), histogramSchema( "schema_dialcache_stored_size_histogram", ["cache_namespace", "use_case", "key_type", "layer"], @@ -383,6 +401,39 @@ describe("Prometheus metrics adapter", () => { ); }); + it("exports every bounded stale-recovery outcome without adding cache identity or layer labels", async () => { + const registry = new Registry(); + const metrics = new PrometheusDialCacheMetrics({ registry, prefix: "stale_" }); + const labels = { + cacheNamespace: "users", + useCase: "PrometheusStaleRecovery", + keyType: "user_id", + } as const; + const outcomes = Object.keys(STALE_RECOVERY_OUTCOMES) as StaleRecoveryOutcome[]; + + for (const outcome of outcomes) { + metrics.staleRecovery({ ...labels, outcome }); + } + + for (const outcome of outcomes) { + await expect( + sumMetric(registry, "stale_dialcache_stale_recovery_counter", { + cache_namespace: labels.cacheNamespace, + use_case: labels.useCase, + key_type: labels.keyType, + outcome, + }), + ).resolves.toBe(1); + } + + const family = ((await registry.getMetricsAsJSON()) as unknown as MetricFamily[]).find( + ({ name }) => name === "stale_dialcache_stale_recovery_counter", + ); + expect(family?.values.map(({ labels: emitted }) => Object.keys(emitted))).toEqual( + outcomes.map(() => ["cache_namespace", "use_case", "key_type", "outcome"]), + ); + }); + it("exports every bounded compression outcome without rewriting labels", async () => { const registry = new Registry(); const metrics = new PrometheusDialCacheMetrics({ registry, prefix: "compression_" }); diff --git a/test/redis-cluster.integration.test.ts b/test/redis-cluster.integration.test.ts index bdffb16..fa0712a 100644 --- a/test/redis-cluster.integration.test.ts +++ b/test/redis-cluster.integration.test.ts @@ -17,6 +17,7 @@ const remoteOnly = new DialCacheKeyConfig({ ttlSec: { [CacheLayer.REMOTE]: 60 }, ramp: { [CacheLayer.REMOTE]: 100 }, }); +const PROTOCOL_READ_MAX_AGE_MS = 60_000; const createTestCluster = (options: RedisClusterOptions) => createCluster({ @@ -237,6 +238,7 @@ describe("DialCache Redis protocol on Redis Cluster", () => { scriptClient.read({ valueKey: "{slot-a}:value", watermarkKey: "{slot-b}:watermark", + maxAgeMs: PROTOCOL_READ_MAX_AGE_MS, }), ).rejects.toThrow(/CROSSSLOT/); await expect( @@ -258,7 +260,7 @@ describe("DialCache Redis protocol on Redis Cluster", () => { const payload = Buffer.from(Array.from({ length: 256 }, (_, index) => index)); expect(await scriptClient.write({ valueKey, cacheTtlMs: 60_000, value: payload })).toBe(true); - const untrackedRead = await scriptClient.read({ valueKey }); + const untrackedRead = await scriptClient.read({ valueKey, maxAgeMs: PROTOCOL_READ_MAX_AGE_MS }); expect(untrackedRead?.payload).toEqual(payload); expect(untrackedRead?.createdAtMs).toBeGreaterThan(0); @@ -278,7 +280,11 @@ describe("DialCache Redis protocol on Redis Cluster", () => { value: trackedPayload, }), ).toBe(true); - const trackedRead = await scriptClient.read({ valueKey: trackedValueKey, watermarkKey }); + const trackedRead = await scriptClient.read({ + valueKey: trackedValueKey, + watermarkKey, + maxAgeMs: PROTOCOL_READ_MAX_AGE_MS, + }); expect(trackedRead?.payload).toEqual(trackedPayload); expect(trackedRead?.createdAtMs).toBeGreaterThan(0); }); @@ -294,21 +300,24 @@ describe("DialCache Redis protocol on Redis Cluster", () => { expect( await adapter.write({ valueKey, watermarkKey, cacheTtlMs: 60_000, value: "glide" }), ).toBe(true); - expect((await adapter.read({ valueKey, watermarkKey }))?.payload).toBe("glide"); + expect((await adapter.read({ valueKey, watermarkKey, maxAgeMs: PROTOCOL_READ_MAX_AGE_MS }))?.payload).toBe("glide"); await adapter.invalidate({ watermarkKey, futureBufferMs: 0 }); // The follow-up write's stamp is fenced unless server time passes the // zero-buffer watermark; the read-null below holds at any margin. await new Promise((resolve) => setTimeout(resolve, 25)); - expect(await adapter.read({ valueKey, watermarkKey })).toBeNull(); + expect(await adapter.read({ valueKey, watermarkKey, maxAgeMs: PROTOCOL_READ_MAX_AGE_MS })).toBeNull(); expect( await adapter.write({ valueKey, watermarkKey, cacheTtlMs: 60_000, value: "glide-2" }), ).toBe(true); - expect((await adapter.read({ valueKey, watermarkKey }))?.payload).toBe("glide-2"); + expect((await adapter.read({ valueKey, watermarkKey, maxAgeMs: PROTOCOL_READ_MAX_AGE_MS }))?.payload).toBe("glide-2"); const untrackedKey = "glide-cluster:{item:untracked}:value"; expect(await adapter.write({ valueKey: untrackedKey, cacheTtlMs: 60_000, value: "plain" })).toBe(true); - expect((await adapter.read({ valueKey: untrackedKey }))?.payload).toBe("plain"); + expect((await adapter.read({ + valueKey: untrackedKey, + maxAgeMs: PROTOCOL_READ_MAX_AGE_MS, + }))?.payload).toBe("plain"); await expect(adapter.write({ valueKey: "{glide-a}:value", @@ -339,10 +348,10 @@ describe("DialCache Redis protocol on Redis Cluster", () => { expect( await adapter.write({ valueKey, watermarkKey, cacheTtlMs: 60_000, value: "recovered" }), ).toBe(true); - expect((await adapter.read({ valueKey, watermarkKey }))?.payload).toBe("recovered"); + expect((await adapter.read({ valueKey, watermarkKey, maxAgeMs: PROTOCOL_READ_MAX_AGE_MS }))?.payload).toBe("recovered"); await flushAllMasters(); await expect(adapter.invalidate({ watermarkKey, futureBufferMs: 0 })).resolves.toBeUndefined(); - expect(await adapter.read({ valueKey, watermarkKey })).toBeNull(); + expect(await adapter.read({ valueKey, watermarkKey, maxAgeMs: PROTOCOL_READ_MAX_AGE_MS })).toBeNull(); }); }); diff --git a/test/redis-payload.test.ts b/test/redis-payload.test.ts index 3eb3d12..1602313 100644 --- a/test/redis-payload.test.ts +++ b/test/redis-payload.test.ts @@ -1,8 +1,11 @@ import { + assertValidRedisMaxAgeMs, + decodeRedisServerTime, decodeRedisFrame, decodeTrackedRedisFrame, encodeRedisFrame, encodeTrackedRedisPlaceholder, + isRedisFrameWithinMaxAge, } from "../src/redis-protocol.js"; import { DialCacheRedisPayloadEncodingError, @@ -26,6 +29,71 @@ function encodeFrame( } describe("Redis frame decoding", () => { + it("decodes Redis TIME exactly to millisecond precision", () => { + expect(decodeRedisServerTime([Buffer.from("0"), Buffer.from("0")])).toBe(0); + expect(decodeRedisServerTime([Buffer.from("1723456789"), Buffer.from("123999")])) + .toBe(1_723_456_789_123); + expect(decodeRedisServerTime([Buffer.from("1"), Buffer.from("999999")])).toBe(1_999); + }); + + it("rejects malformed, out-of-range, and unsafe Redis TIME replies", () => { + const malformed: readonly unknown[] = [ + null, + [], + [Buffer.from("1")], + [Buffer.from("1"), Buffer.from("0"), Buffer.from("0")], + ["1", Buffer.from("0")], + [Buffer.from("1"), "0"], + [Buffer.from(""), Buffer.from("0")], + [Buffer.from("-1"), Buffer.from("0")], + [Buffer.from("1.0"), Buffer.from("0")], + [Buffer.from("1"), Buffer.from("1000000")], + [Buffer.from("9007199254741"), Buffer.from("0")], + ]; + + for (const raw of malformed) { + expect(() => decodeRedisServerTime(raw)).toThrow(DialCacheRedisPayloadError); + expect(() => decodeRedisServerTime(raw)).toThrow( + "Invalid DialCache Redis TIME reply; expected two unsigned decimal bulk strings", + ); + } + }); + + it("uses a strict maximum-age boundary against Redis server time", () => { + const frame = decodeRedisFrame(encodeFrame("cached", 0, 1_000)); + if (frame === null) { + throw new Error("Expected a decoded frame"); + } + + expect(isRedisFrameWithinMaxAge(frame, 1_099, 100)).toBe(true); + expect(isRedisFrameWithinMaxAge(frame, 1_100, 100)).toBe(false); + expect(isRedisFrameWithinMaxAge(frame, 1_101, 100)).toBe(false); + // A valid GET/MGET-then-TIME pair cannot observe a future server-stamped + // frame, so reject legacy client-clock and corrupt future stamps. + expect(isRedisFrameWithinMaxAge(frame, 999, 100)).toBe(false); + }); + + it("rejects invalid maximum ages even before a frame can be served", () => { + expect(() => assertValidRedisMaxAgeMs(1)).not.toThrow(); + expect(() => assertValidRedisMaxAgeMs(31_536_000_000)).not.toThrow(); + + for (const maxAgeMs of [ + 0, + -1, + 0.5, + Number.NaN, + Number.POSITIVE_INFINITY, + 31_536_000_001, + "100" as unknown as number, + ]) { + expect(() => assertValidRedisMaxAgeMs(maxAgeMs)).toThrow( + new RangeError( + "DialCache Redis maxAgeMs must be a positive safe integer no greater than 31536000000", + ), + ); + } + }); + it("decodes UTF-8 and binary payloads without copying binary data", () => { expect(decodeRedisFrame(encodeFrame("cached"))).toEqual({ payload: "cached", createdAtMs: 1_000 }); @@ -42,10 +110,15 @@ describe("Redis frame decoding", () => { expect(payload.byteLength).toBe(frame.byteLength - 10); }); - it("treats missing, short, and unsupported frames as misses", () => { + it("treats missing, short, unsupported, and unsafe-timestamp frames as misses", () => { expect(decodeRedisFrame(null)).toBeNull(); expect(decodeRedisFrame(Buffer.alloc(9))).toBeNull(); expect(decodeRedisFrame(encodeFrame("cached", 0, 1_000, 2))).toBeNull(); + + const unsafeTimestamp = encodeFrame("cached"); + unsafeTimestamp.writeBigUInt64BE(BigInt(Number.MAX_SAFE_INTEGER) + 1n, 1); + expect(decodeRedisFrame(unsafeTimestamp)).toBeNull(); + expect(decodeTrackedRedisFrame(unsafeTimestamp, Buffer.from("0"))).toBeNull(); }); it("rejects unsupported payload encodings after validating the frame", () => { diff --git a/test/redis-real.integration.test.ts b/test/redis-real.integration.test.ts index 645f7d7..8aef56a 100644 --- a/test/redis-real.integration.test.ts +++ b/test/redis-real.integration.test.ts @@ -33,6 +33,7 @@ const adapterKinds = [ ] as const; type AdapterKind = (typeof adapterKinds)[number]["kind"]; const MAX_SUPPORTED_DURATION_MS = 31_536_000_000; +const PROTOCOL_READ_MAX_AGE_MS = 60_000; const WATERMARK_TTL_MARGIN_MS = 60_000; interface Deferred { @@ -227,6 +228,147 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { expect(inlineCalls).toBe(1); }); + it("enforces Redis-server logical age for untracked and tracked reads", async () => { + if (client === undefined || admin === undefined) { + throw new Error("Redis test clients did not start"); + } + const serverNowMs = (await admin.time()).getTime(); + const untrackedFreshKey = "read-age:{item:untracked-fresh}:value"; + const untrackedExpiredKey = "read-age:{item:untracked-expired}:value"; + const trackedFreshKey = "read-age:{item:tracked-fresh}:value"; + const trackedFreshWatermarkKey = "read-age:{item:tracked-fresh}:watermark"; + const trackedExpiredKey = "read-age:{item:tracked-expired}:value"; + const trackedExpiredWatermarkKey = "read-age:{item:tracked-expired}:watermark"; + + await admin.set(untrackedFreshKey, encodeFrame("untracked-fresh", 0, serverNowMs), { PX: 60_000 }); + await admin.set(untrackedExpiredKey, encodeFrame("untracked-expired", 0, serverNowMs - 5_000), { + PX: 60_000, + }); + await admin.set(trackedFreshKey, encodeFrame("tracked-fresh", 0, serverNowMs), { PX: 60_000 }); + await admin.set(trackedFreshWatermarkKey, "0", { PX: 60_000 }); + await admin.set(trackedExpiredKey, encodeFrame("tracked-expired", 0, serverNowMs - 5_000), { + PX: 60_000, + }); + await admin.set(trackedExpiredWatermarkKey, "0", { PX: 60_000 }); + + expect((await client.adapter.read({ + valueKey: untrackedFreshKey, + maxAgeMs: PROTOCOL_READ_MAX_AGE_MS, + }))?.payload).toBe("untracked-fresh"); + expect(await client.adapter.read({ valueKey: untrackedExpiredKey, maxAgeMs: 1_000 })).toBeNull(); + expect((await client.adapter.read({ + valueKey: trackedFreshKey, + watermarkKey: trackedFreshWatermarkKey, + maxAgeMs: PROTOCOL_READ_MAX_AGE_MS, + }))?.payload).toBe("tracked-fresh"); + expect(await client.adapter.read({ + valueKey: trackedExpiredKey, + watermarkKey: trackedExpiredWatermarkKey, + maxAgeMs: 1_000, + })).toBeNull(); + }); + + it("stamps untracked writes with Redis server time even when the application clock is skewed", async () => { + if (client === undefined || admin === undefined) { + throw new Error("Redis test clients did not start"); + } + const valueKey = "server-stamp:{item:untracked}:value"; + const serverBeforeMs = (await admin.time()).getTime(); + const nowSpy = vi.spyOn(Date, "now").mockReturnValue(1); + try { + await expect(client.adapter.write({ + valueKey, + cacheTtlMs: 60_000, + value: "server-stamped", + })).resolves.toBe(true); + } finally { + nowSpy.mockRestore(); + } + const serverAfterMs = (await admin.time()).getTime(); + const stored = await admin.get(commandOptions({ returnBuffers: true }), valueKey); + expect(stored).not.toBeNull(); + const createdAtMs = Number(stored?.readBigUInt64BE(1)); + + expect(createdAtMs).toBeGreaterThanOrEqual(serverBeforeMs); + expect(createdAtMs).toBeLessThanOrEqual(serverAfterMs); + expect((await client.adapter.read({ valueKey, maxAgeMs: PROTOCOL_READ_MAX_AGE_MS }))?.payload).toBe( + "server-stamped", + ); + }); + + it.each([false, true])( + "retains a logically stale value and recovers it after source rejection (tracked=%s)", + async (trackForInvalidation) => { + if (client === undefined || admin === undefined) { + throw new Error("Redis test clients did not start"); + } + const namespace = `real-stale-${kind}-${trackForInvalidation ? "tracked" : "untracked"}`; + const useCase = "RealStaleOnError"; + const id = "123"; + const key = new DialCacheKey({ namespace, keyType: "item_id", id, useCase, trackForInvalidation }); + const valueKey = `${key.urn}:dialcache-frame-v1`; + const sourceValue = { id, version: 1 }; + const sourceError = Object.freeze({ code: "SOURCE_UNAVAILABLE" }); + let sourceCalls = 0; + const source = vi.fn(async (): Promise => { + if (++sourceCalls === 1) { + return sourceValue; + } + throw sourceError; + }); + const staleRecovery = vi.fn(); + const metrics = { + request: vi.fn(), + miss: vi.fn(), + disabled: vi.fn(), + error: vi.fn(), + invalidation: vi.fn(), + staleRecovery, + observeGet: vi.fn(), + observeFallback: vi.fn(), + observeSerialization: vi.fn(), + observeSize: vi.fn(), + } satisfies DialCacheMetricsAdapter; + const dialcache = new DialCache({ + namespace, + redis: { client: client.adapter, readTimeoutMs: 10_000 }, + metrics, + }); + const getItem = dialcache.cached(source, { + keyType: "item_id", + useCase, + cacheKey: () => id, + trackForInvalidation, + defaultConfig: new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 1 }, + ramp: { [CacheLayer.REMOTE]: 100 }, + staleOnErrorMaxAgeSec: 60, + }), + }); + + await expect(dialcache.enable(async () => await getItem())).resolves.toBe(sourceValue); + expect(await admin.pTTL(valueKey)).toBeGreaterThan(55_000); + + const redisNowMs = (await admin.time()).getTime(); + await admin.set(valueKey, encodeFrame(JSON.stringify(sourceValue), 0, redisNowMs - 2_000), { + PX: 60_000, + }); + const ttlBeforeRecovery = await admin.pTTL(valueKey); + + await expect(dialcache.enable(async () => await getItem())).resolves.toEqual(sourceValue); + + expect(source).toHaveBeenCalledTimes(2); + expect(staleRecovery).toHaveBeenCalledOnce(); + expect(staleRecovery).toHaveBeenCalledWith({ + cacheNamespace: namespace, + useCase, + keyType: "item_id", + outcome: "served", + }); + expect(await admin.pTTL(valueKey)).toBeLessThanOrEqual(ttlBeforeRecovery); + }, + ); + it("compresses values above the threshold and stores small values byte-identical", async () => { if (client === undefined || admin === undefined) { throw new Error("Redis test clients did not start"); @@ -375,7 +517,7 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { const valueKey = `binary-raw:{item:${index}}:value`; expect(await scriptClient.write({ valueKey, cacheTtlMs: 60_000, value: payload })).toBe(true); - const roundTrip = await scriptClient.read({ valueKey }); + const roundTrip = await scriptClient.read({ valueKey, maxAgeMs: PROTOCOL_READ_MAX_AGE_MS }); const stored = await admin.get(commandOptions({ returnBuffers: true }), valueKey); expect(Buffer.isBuffer(roundTrip?.payload)).toBe(true); @@ -399,7 +541,11 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { value: trackedPayload, }), ).toBe(true); - const trackedRead = await scriptClient.read({ valueKey: trackedValueKey, watermarkKey }); + const trackedRead = await scriptClient.read({ + valueKey: trackedValueKey, + watermarkKey, + maxAgeMs: PROTOCOL_READ_MAX_AGE_MS, + }); expect(trackedRead?.payload).toEqual(trackedPayload); expect(trackedRead?.createdAtMs).toBeGreaterThan(0); }); @@ -799,6 +945,7 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { }); expect((await client.adapter.read({ valueKey, + maxAgeMs: PROTOCOL_READ_MAX_AGE_MS, ...(tracked ? { watermarkKey } : {}), }))?.payload).toBe(JSON.stringify(sourceValue)); expect(await admin.pTTL(valueKey)).toBeGreaterThan(55_000); @@ -932,7 +1079,7 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { await admin.scriptFlush(); expect(await scriptClient.write({ valueKey, cacheTtlMs: 60_000, value: "untracked" })).toBe(true); - expect((await scriptClient.read({ valueKey }))?.payload).toBe("untracked"); + expect((await scriptClient.read({ valueKey, maxAgeMs: PROTOCOL_READ_MAX_AGE_MS }))?.payload).toBe("untracked"); const trackedValueKey = "script-recovery:{item:tracked}:value"; const watermarkKey = "script-recovery:{item:tracked}:watermark"; @@ -945,7 +1092,11 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { value: "tracked", }), ).toBe(true); - expect((await scriptClient.read({ valueKey: trackedValueKey, watermarkKey }))?.payload).toBe("tracked"); + expect((await scriptClient.read({ + valueKey: trackedValueKey, + watermarkKey, + maxAgeMs: PROTOCOL_READ_MAX_AGE_MS, + }))?.payload).toBe("tracked"); // The recovered write must cache the stamp under sha1(source) — the // digest node-redis registers and the GLIDE batch dispatches — so later // writes take the single-round-trip path. (The unit suites pin each @@ -960,7 +1111,11 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { futureBufferMs: 0, }), ).resolves.toBeUndefined(); - expect(await scriptClient.read({ valueKey: trackedValueKey, watermarkKey })).toBeNull(); + expect(await scriptClient.read({ + valueKey: trackedValueKey, + watermarkKey, + maxAgeMs: PROTOCOL_READ_MAX_AGE_MS, + })).toBeNull(); }); it("treats every invalid read frame and watermark state as a miss", async () => { @@ -970,29 +1125,34 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { const scriptClient = client.adapter; const valueKey = "read-paths:{item:read}:value"; const watermarkKey = "read-paths:{item:read}:watermark"; + const createdAtMs = (await admin.time()).getTime(); - expect(await scriptClient.read({ valueKey })).toBeNull(); + expect(await scriptClient.read({ valueKey, maxAgeMs: PROTOCOL_READ_MAX_AGE_MS })).toBeNull(); await admin.set(valueKey, Buffer.alloc(9)); - expect(await scriptClient.read({ valueKey })).toBeNull(); + expect(await scriptClient.read({ valueKey, maxAgeMs: PROTOCOL_READ_MAX_AGE_MS })).toBeNull(); await admin.set(valueKey, encodeFrame("wrong-version", 0, 1_000, 2)); - expect(await scriptClient.read({ valueKey })).toBeNull(); + expect(await scriptClient.read({ valueKey, maxAgeMs: PROTOCOL_READ_MAX_AGE_MS })).toBeNull(); - await admin.set(valueKey, encodeFrame("tracked", 0, 1_000)); - expect(await scriptClient.read({ valueKey, watermarkKey })).toBeNull(); + await admin.set(valueKey, encodeFrame("tracked", 0, createdAtMs)); + expect(await scriptClient.read({ valueKey, watermarkKey, maxAgeMs: PROTOCOL_READ_MAX_AGE_MS })).toBeNull(); await admin.set(watermarkKey, "not-a-watermark"); - expect(await scriptClient.read({ valueKey, watermarkKey })).toBeNull(); + expect(await scriptClient.read({ valueKey, watermarkKey, maxAgeMs: PROTOCOL_READ_MAX_AGE_MS })).toBeNull(); await admin.set(watermarkKey, "9".repeat(400)); - expect(await scriptClient.read({ valueKey, watermarkKey })).toBeNull(); + expect(await scriptClient.read({ valueKey, watermarkKey, maxAgeMs: PROTOCOL_READ_MAX_AGE_MS })).toBeNull(); - await admin.set(watermarkKey, "1000"); - expect(await scriptClient.read({ valueKey, watermarkKey })).toBeNull(); + await admin.set(watermarkKey, String(createdAtMs)); + expect(await scriptClient.read({ valueKey, watermarkKey, maxAgeMs: PROTOCOL_READ_MAX_AGE_MS })).toBeNull(); - await admin.set(watermarkKey, "999.5"); - expect((await scriptClient.read({ valueKey, watermarkKey }))?.payload).toBe("tracked"); + await admin.set(watermarkKey, String(createdAtMs - 0.5)); + expect((await scriptClient.read({ + valueKey, + watermarkKey, + maxAgeMs: PROTOCOL_READ_MAX_AGE_MS, + }))?.payload).toBe("tracked"); }); it("records a stale tracked frame as a remote miss without a read error", async () => { @@ -1065,13 +1225,21 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { await admin.hSet(valueKey, "field", "value"); await admin.set(watermarkKey, "0"); - await expect(scriptClient.read({ valueKey })).rejects.toThrow(/WRONGTYPE/); - await expect(scriptClient.read({ valueKey, watermarkKey })).resolves.toBeNull(); + await expect(scriptClient.read({ valueKey, maxAgeMs: PROTOCOL_READ_MAX_AGE_MS })).rejects.toThrow(/WRONGTYPE/); + await expect(scriptClient.read({ + valueKey, + watermarkKey, + maxAgeMs: PROTOCOL_READ_MAX_AGE_MS, + })).resolves.toBeNull(); await admin.del([valueKey, watermarkKey]); await admin.set(valueKey, encodeFrame("cached", 0, 1_000)); await admin.hSet(watermarkKey, "field", "value"); - await expect(scriptClient.read({ valueKey, watermarkKey })).resolves.toBeNull(); + await expect(scriptClient.read({ + valueKey, + watermarkKey, + maxAgeMs: PROTOCOL_READ_MAX_AGE_MS, + })).resolves.toBeNull(); const namespace = "wrong-type-repair"; const repairValueKey = `{${namespace}:item_id:repair}#WrongTypeRepair:dialcache-frame-v1`; @@ -1176,7 +1344,11 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { // so the original frame is replaced by an unreadable version-0 placeholder. const stored = await admin.get(commandOptions({ returnBuffers: true }), valueKey); expect(stored?.[0]).toBe(0); - await expect(client.adapter.read({ valueKey, watermarkKey })).resolves.toBeNull(); + await expect(client.adapter.read({ + valueKey, + watermarkKey, + maxAgeMs: PROTOCOL_READ_MAX_AGE_MS, + })).resolves.toBeNull(); }); it("rejects invalid raw script arguments before mutating Redis", async () => { @@ -1395,7 +1567,7 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { })).rejects.toThrow("invalid DialCache watermark"); // The paired SET lands before the stamp validates the watermark, so the // tracked path serves nothing and the placeholder stays unpromoted. - expect(await scriptClient.read({ valueKey, watermarkKey })).toBeNull(); + expect(await scriptClient.read({ valueKey, watermarkKey, maxAgeMs: PROTOCOL_READ_MAX_AGE_MS })).toBeNull(); const stored = await admin.get(commandOptions({ returnBuffers: true }), valueKey); expect(stored?.[0]).toBe(0); await admin.del(valueKey); @@ -1549,7 +1721,11 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { expect(wrote).toBe(true); expect(await admin.get(watermarkKey)).toBe("1.75"); expect(await admin.pTTL(watermarkKey)).toBeGreaterThanOrEqual(61_000); - expect((await scriptClient.read({ valueKey, watermarkKey }))?.payload).toBe("cached"); + expect((await scriptClient.read({ + valueKey, + watermarkKey, + maxAgeMs: PROTOCOL_READ_MAX_AGE_MS, + }))?.payload).toBe("cached"); }); it("does not rewrite sufficient or persistent watermarks on tracked writes", async () => { @@ -1611,22 +1787,26 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { expect(ttlAfterWrite).toBeGreaterThanOrEqual(61_000); await scriptClient.invalidate({ watermarkKey, futureBufferMs: 100 }); - expect(await scriptClient.read({ valueKey, watermarkKey })).toBeNull(); + expect(await scriptClient.read({ valueKey, watermarkKey, maxAgeMs: PROTOCOL_READ_MAX_AGE_MS })).toBeNull(); const watermarkBeforeBlockedWrite = await admin.get(watermarkKey); const watermarkTtlBeforeBlockedWrite = await admin.pTTL(watermarkKey); expect(await scriptClient.write({ ...writeRequest, value: "blocked" })).toBe(false); - expect(await scriptClient.read({ valueKey })).toBeNull(); + expect(await scriptClient.read({ valueKey, maxAgeMs: PROTOCOL_READ_MAX_AGE_MS })).toBeNull(); expect(await admin.get(watermarkKey)).toBe(watermarkBeforeBlockedWrite); const watermarkTtlAfterBlockedWrite = await admin.pTTL(watermarkKey); expect(watermarkTtlAfterBlockedWrite).toBeGreaterThan(watermarkTtlBeforeBlockedWrite - 1_000); expect(watermarkTtlAfterBlockedWrite).toBeLessThanOrEqual(watermarkTtlBeforeBlockedWrite); const ttlBeforeRead = await admin.pTTL(watermarkKey); - await scriptClient.read({ valueKey, watermarkKey }); + await scriptClient.read({ valueKey, watermarkKey, maxAgeMs: PROTOCOL_READ_MAX_AGE_MS }); expect(await admin.pTTL(watermarkKey)).toBeLessThanOrEqual(ttlBeforeRead); await new Promise((resolve) => setTimeout(resolve, 110)); expect(await scriptClient.write({ ...writeRequest, value: "fresh" })).toBe(true); - expect((await scriptClient.read({ valueKey, watermarkKey }))?.payload).toBe("fresh"); + expect((await scriptClient.read({ + valueKey, + watermarkKey, + maxAgeMs: PROTOCOL_READ_MAX_AGE_MS, + }))?.payload).toBe("fresh"); }); it("documents that losing a watermark removes its publication fence", async () => { @@ -1650,7 +1830,11 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { expect(await scriptClient.write(staleWrite)).toBe(true); expect(await admin.get(watermarkKey)).toBe("0"); - expect((await scriptClient.read({ valueKey, watermarkKey }))?.payload).toBe("stale"); + expect((await scriptClient.read({ + valueKey, + watermarkKey, + maxAgeMs: PROTOCOL_READ_MAX_AGE_MS, + }))?.payload).toBe("stale"); }); it("never serves an unstamped placeholder and refuses foreign stamps", async () => { @@ -1663,18 +1847,22 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { await admin.set(valueKey, frame, { PX: 60_000 }); await admin.set(watermarkKey, "0", { PX: 120_000 }); - expect(await client.adapter.read({ valueKey, watermarkKey })).toBeNull(); - expect(await client.adapter.read({ valueKey })).toBeNull(); + expect(await client.adapter.read({ valueKey, watermarkKey, maxAgeMs: PROTOCOL_READ_MAX_AGE_MS })).toBeNull(); + expect(await client.adapter.read({ valueKey, maxAgeMs: PROTOCOL_READ_MAX_AGE_MS })).toBeNull(); // A stamp carrying a different write's nonce must not promote this // placeholder: a leftover from a failed write stays unreadable even // after later invalidations pass. expect(await client.raw.stamp(valueKey, watermarkKey, 2_000, Buffer.alloc(8, 0xab))).toBe(2); - expect(await client.adapter.read({ valueKey, watermarkKey })).toBeNull(); + expect(await client.adapter.read({ valueKey, watermarkKey, maxAgeMs: PROTOCOL_READ_MAX_AGE_MS })).toBeNull(); // Only the paired nonce promotes it to a served, server-stamped frame. expect(await client.raw.stamp(valueKey, watermarkKey, 2_000, nonce)).toBe(1); - expect((await client.adapter.read({ valueKey, watermarkKey }))?.payload).toBe("pending"); + expect((await client.adapter.read({ + valueKey, + watermarkKey, + maxAgeMs: PROTOCOL_READ_MAX_AGE_MS, + }))?.payload).toBe("pending"); const stored = await admin.get(commandOptions({ returnBuffers: true }), valueKey); expect(stored?.[0]).toBe(1); expect(stored?.readBigUInt64BE(1) ?? 0n).toBeGreaterThan(0n); @@ -1695,7 +1883,7 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { const stored = await admin.get(commandOptions({ returnBuffers: true }), valueKey); expect(stored?.readBigUInt64BE(1)).toBe(1_000n); - expect(await client.adapter.read({ valueKey, watermarkKey })).toBeNull(); + expect(await client.adapter.read({ valueKey, watermarkKey, maxAgeMs: PROTOCOL_READ_MAX_AGE_MS })).toBeNull(); }); it("does not create a value key when stamping after a lost SET", async () => { @@ -1723,10 +1911,16 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { const binary = Buffer.from([0, 0xff, 0xc3, 0x28, 0x80]); await nodeRedis.write({ valueKey: "interop:node-to-glide", cacheTtlMs: 60_000, value: binary }); - expect((await valkeyGlide.read({ valueKey: "interop:node-to-glide" }))?.payload).toEqual(binary); + expect((await valkeyGlide.read({ + valueKey: "interop:node-to-glide", + maxAgeMs: PROTOCOL_READ_MAX_AGE_MS, + }))?.payload).toEqual(binary); await valkeyGlide.write({ valueKey: "interop:glide-to-node", cacheTtlMs: 60_000, value: "hello" }); - expect((await nodeRedis.read({ valueKey: "interop:glide-to-node" }))?.payload).toBe("hello"); + expect((await nodeRedis.read({ + valueKey: "interop:glide-to-node", + maxAgeMs: PROTOCOL_READ_MAX_AGE_MS, + }))?.payload).toBe("hello"); const nodeTrackedValueKey = "interop:{node-tracked}:value"; const nodeTrackedWatermarkKey = "interop:{node-tracked}:watermark"; @@ -1739,6 +1933,7 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { expect((await valkeyGlide.read({ valueKey: nodeTrackedValueKey, watermarkKey: nodeTrackedWatermarkKey, + maxAgeMs: PROTOCOL_READ_MAX_AGE_MS, }))?.payload).toEqual(binary); const glideTrackedValueKey = "interop:{glide-tracked}:value"; @@ -1752,6 +1947,7 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { expect((await nodeRedis.read({ valueKey: glideTrackedValueKey, watermarkKey: glideTrackedWatermarkKey, + maxAgeMs: PROTOCOL_READ_MAX_AGE_MS, }))?.payload).toBe("tracked"); }); }); diff --git a/test/valkey-glide.test.ts b/test/valkey-glide.test.ts index cb92929..9b584a4 100644 --- a/test/valkey-glide.test.ts +++ b/test/valkey-glide.test.ts @@ -8,7 +8,11 @@ import { DialCacheRedisPlaceholderLostError, DialCacheRedisProtocolError, } from "../src/redis-client.js"; -import { INVALIDATE_CACHE_SCRIPT, WRITE_TRACKED_STAMP_SCRIPT } from "../src/redis-protocol.js"; +import { + INVALIDATE_CACHE_SCRIPT, + WRITE_TRACKED_STAMP_SCRIPT, + WRITE_UNTRACKED_STAMP_SCRIPT, +} from "../src/redis-protocol.js"; import { createValkeyGlideDialCacheClient } from "../src/valkey-glide.js"; const INVALID_WRITE_REPLIES: readonly unknown[] = [ @@ -24,6 +28,7 @@ const INVALID_WRITE_REPLIES: readonly unknown[] = [ null, undefined, ]; +const INVALID_UNTRACKED_WRITE_REPLIES: readonly unknown[] = [0, ...INVALID_WRITE_REPLIES]; const INVALID_INVALIDATION_REPLIES: readonly unknown[] = [0, 2, ...INVALID_WRITE_REPLIES]; const decoderBytes = Symbol("bytes"); @@ -36,6 +41,11 @@ class MockBatch { readonly commands: Array> = []; readonly mget = vi.fn((keys: Array) => { this.keys = keys; + this.commands.push(["MGET", ...keys]); + return this; + }); + readonly get = vi.fn((key: string | Buffer) => { + this.commands.push(["GET", key]); return this; }); readonly customCommand = vi.fn((args: Array) => { @@ -128,6 +138,13 @@ function redisFrame( return frame; } +function redisTime(nowMs: number): [Buffer, Buffer] { + return [ + Buffer.from(String(Math.floor(nowMs / 1_000))), + Buffer.from(String((nowMs % 1_000) * 1_000)), + ]; +} + async function expectProtocolError(operation: Promise, message: string): Promise { let rejection: unknown; try { @@ -145,70 +162,135 @@ describe("Valkey GLIDE adapter", () => { clusterBatchInstances.length = 0; }); - it("uses GET and a non-atomic primary MGET batch that preserves caller WATCH state", async () => { + it("orders native reads before TIME in non-atomic primary batches", async () => { const client = fakeClient( - redisFrame("plain"), - [[redisFrame(Buffer.from([0, 0xff])), Buffer.from("0")]], - null, + [redisFrame("plain"), redisTime(1_500)], + [[redisFrame(Buffer.from([0, 0xff])), Buffer.from("0")], redisTime(1_500)], + [null, redisTime(1_500)], ); const adapter = createValkeyGlideDialCacheClient(client, mockGlide); - await expect(adapter.read({ valueKey: "plain:value" })).resolves.toEqual({ + expect(adapter.enforcesMaxAge).toBe(true); + await expect(adapter.read({ valueKey: "plain:value", maxAgeMs: 1_000 })).resolves.toEqual({ payload: "plain", createdAtMs: 1_000, }); await expect( - adapter.read({ valueKey: "tracked:{id}:value", watermarkKey: "tracked:{id}:watermark" }), + adapter.read({ + valueKey: "tracked:{id}:value", + watermarkKey: "tracked:{id}:watermark", + maxAgeMs: 1_000, + }), ).resolves.toEqual({ payload: Buffer.from([0, 0xff]), createdAtMs: 1_000 }); - await expect(adapter.read({ valueKey: "missing:value" })).resolves.toBeNull(); - - expect(client.get).toHaveBeenNthCalledWith( - 1, - "plain:value", - { decoder: decoderBytes }, - ); - expect(client.get).toHaveBeenNthCalledWith( - 2, - "missing:value", - { decoder: decoderBytes }, - ); - expect(batchInstances).toHaveLength(1); - expect(batchInstances[0]?.isAtomic).toBe(false); - expect(batchInstances[0]?.mget).toHaveBeenCalledWith([ - "tracked:{id}:value", - "tracked:{id}:watermark", + await expect( + adapter.read({ valueKey: "missing:value", maxAgeMs: 1_000 }), + ).resolves.toBeNull(); + + expect(batchInstances).toHaveLength(3); + expect(batchInstances.every(({ isAtomic }) => !isAtomic)).toBe(true); + expect(batchInstances[0]?.commands).toEqual([ + ["GET", "plain:value"], + ["TIME"], ]); - expect(client.exec).toHaveBeenCalledWith( - batchInstances[0], - true, - { decoder: decoderBytes }, - ); + expect(batchInstances[1]?.commands).toEqual([ + ["MGET", "tracked:{id}:value", "tracked:{id}:watermark"], + ["TIME"], + ]); + expect(batchInstances[2]?.commands).toEqual([ + ["GET", "missing:value"], + ["TIME"], + ]); + for (const batch of batchInstances) { + expect(client.exec).toHaveBeenCalledWith(batch, true, { decoder: decoderBytes }); + } + expect(client.get).not.toHaveBeenCalled(); expect(client.customCommand).not.toHaveBeenCalled(); }); - it("routes tracked cluster MGET directly to the slot primary", async () => { - const client = fakeClusterClient([ - redisFrame("tracked-cluster"), - Buffer.from("0"), - ]); + it("enforces the strict max-age boundary against the ordered Redis TIME", async () => { + const frame = redisFrame("cached", { createdAtMs: 1_000 }); + const client = fakeClient( + [frame, redisTime(2_999)], + [frame, redisTime(3_000)], + ); + const adapter = createValkeyGlideDialCacheClient(client, mockGlide); + + await expect( + adapter.read({ valueKey: "fresh:value", maxAgeMs: 2_000 }), + ).resolves.toEqual({ payload: "cached", createdAtMs: 1_000 }); + await expect( + adapter.read({ valueKey: "expired:value", maxAgeMs: 2_000 }), + ).resolves.toBeNull(); + }); + + it("rejects invalid maximum ages before dispatching a batch", async () => { + const client = fakeClient(); const adapter = createValkeyGlideDialCacheClient(client, mockGlide); + for (const maxAgeMs of [ + 0, + -1, + 0.5, + Number.NaN, + Number.POSITIVE_INFINITY, + 31_536_000_001, + "100" as unknown as number, + ]) { + await expect(adapter.read({ valueKey: "plain:value", maxAgeMs })).rejects.toThrow( + "DialCache Redis maxAgeMs must be a positive safe integer no greater than 31536000000", + ); + } + + expect(client.exec).not.toHaveBeenCalled(); + expect(batchInstances).toHaveLength(0); + }); + + it("routes GET/MGET and their following TIME to the same slot primary on cluster", async () => { + const client = fakeClusterClient( + [redisFrame("plain-cluster"), redisTime(1_500)], + [[redisFrame("tracked-cluster"), Buffer.from("0")], redisTime(1_500)], + ); + const adapter = createValkeyGlideDialCacheClient(client, mockGlide); + + await expect( + adapter.read({ valueKey: "plain:{id}:value", maxAgeMs: 1_000 }), + ).resolves.toEqual({ payload: "plain-cluster", createdAtMs: 1_000 }); await expect( adapter.read({ - valueKey: "cluster:{id}:value", - watermarkKey: "cluster:{id}:watermark", + valueKey: "tracked:{id}:value", + watermarkKey: "tracked:{id}:watermark", + maxAgeMs: 1_000, }), ).resolves.toEqual({ payload: "tracked-cluster", createdAtMs: 1_000 }); - expect(client.customCommand).toHaveBeenCalledWith( - ["MGET", "cluster:{id}:value", "cluster:{id}:watermark"], + expect(clusterBatchInstances).toHaveLength(2); + expect(clusterBatchInstances[0]?.commands).toEqual([ + ["GET", "plain:{id}:value"], + ["TIME"], + ]); + expect(clusterBatchInstances[1]?.commands).toEqual([ + ["MGET", "tracked:{id}:value", "tracked:{id}:watermark"], + ["TIME"], + ]); + expect(client.exec).toHaveBeenNthCalledWith( + 1, + clusterBatchInstances[0], + true, { decoder: decoderBytes, - route: { type: "primarySlotKey", key: "cluster:{id}:value" }, + route: { type: "primarySlotKey", key: "plain:{id}:value" }, }, ); - expect(client.exec).not.toHaveBeenCalled(); - expect(batchInstances).toHaveLength(0); + expect(client.exec).toHaveBeenNthCalledWith( + 2, + clusterBatchInstances[1], + true, + { + decoder: decoderBytes, + route: { type: "primarySlotKey", key: "tracked:{id}:value" }, + }, + ); + expect(client.customCommand).not.toHaveBeenCalled(); }); it("rejects forwarding wrappers instead of silently treating them as standalone", () => { @@ -275,35 +357,35 @@ describe("Valkey GLIDE adapter", () => { }); it("preserves GLIDE invocation options when given a core read context", async () => { - const client = fakeClient(null); + const client = fakeClient([null, redisTime(1_500)]); const adapter = createValkeyGlideDialCacheClient(client, mockGlide); const controller = new AbortController(); await adapter.read( - { valueKey: "plain:value" }, + { valueKey: "plain:value", maxAgeMs: 1_000 }, { timeoutMs: 25, signal: controller.signal }, ); - expect(client.get).toHaveBeenCalledWith( - "plain:value", + expect(client.exec).toHaveBeenCalledWith( + batchInstances[0], + true, { decoder: decoderBytes }, ); + expect(batchInstances[0]?.commands).toEqual([["GET", "plain:value"], ["TIME"]]); }); - it("writes untracked SETs directly and tracked pairs through a batch", async () => { + it("writes untracked and tracked placeholders through ordered non-atomic batches", async () => { const binary = Buffer.from([0, 0xff, 0x80]); const client = fakeClient( - Buffer.from("OK"), + [Buffer.from("OK"), 1], [Buffer.from("OK"), 0], 1, ); const adapter = createValkeyGlideDialCacheClient(client, mockGlide); - const before = Date.now(); await expect( adapter.write({ valueKey: "plain:value", cacheTtlMs: 1_000, value: "hello" }), ).resolves.toBe(true); - const after = Date.now(); await expect( adapter.write({ valueKey: "tracked:{id}:value", @@ -316,23 +398,34 @@ describe("Valkey GLIDE adapter", () => { adapter.invalidate({ watermarkKey: "tracked:{id}:watermark", futureBufferMs: 100 }), ).resolves.toBeUndefined(); - const [untrackedSet, untrackedOptions] = client.customCommand.mock.calls[0] - ?? [[], undefined]; - expect(untrackedSet[0]).toBe("SET"); - expect(untrackedSet[1]).toBe("plain:value"); - expect(untrackedSet[3]).toBe("PX"); - expect(untrackedSet[4]).toBe("1000"); - const untrackedFrame = untrackedSet[2] as Buffer; - expect(untrackedFrame[0]).toBe(1); + expect(batchInstances).toHaveLength(2); + const [untrackedBatch, trackedBatch] = batchInstances; + expect(untrackedBatch?.isAtomic).toBe(false); + expect(untrackedBatch?.commands).toHaveLength(2); + const [untrackedSet, untrackedStamp] = untrackedBatch?.commands ?? []; + expect(untrackedSet?.[0]).toBe("SET"); + expect(untrackedSet?.[1]).toBe("plain:value"); + expect(untrackedSet?.[3]).toBe("PX"); + expect(untrackedSet?.[4]).toBe("1000"); + const untrackedFrame = untrackedSet?.[2] as Buffer; + expect(untrackedFrame[0]).toBe(0); expect(untrackedFrame[9]).toBe(0); expect(untrackedFrame.subarray(10).toString("utf8")).toBe("hello"); - const createdAtMs = Number(untrackedFrame.readBigUInt64BE(1)); - expect(createdAtMs).toBeGreaterThanOrEqual(before); - expect(createdAtMs).toBeLessThanOrEqual(after); - expect(untrackedOptions).toEqual({ decoder: decoderBytes }); + const untrackedNonce = untrackedFrame.subarray(1, 9); + expect(untrackedStamp).toEqual([ + "EVALSHA", + createHash("sha1").update(WRITE_UNTRACKED_STAMP_SCRIPT).digest("hex"), + "1", + "plain:value", + untrackedNonce, + ]); + expect(client.exec).toHaveBeenNthCalledWith( + 1, + untrackedBatch, + false, + { decoder: decoderBytes }, + ); - expect(batchInstances).toHaveLength(1); - const trackedBatch = batchInstances[0]; expect(trackedBatch?.isAtomic).toBe(false); expect(trackedBatch?.commands).toHaveLength(2); const [trackedSet, stamp] = trackedBatch?.commands ?? []; @@ -354,13 +447,18 @@ describe("Valkey GLIDE adapter", () => { "2000", nonce, ]); - expect(client.exec).toHaveBeenCalledTimes(1); - expect(client.exec).toHaveBeenCalledWith(trackedBatch, false, { decoder: decoderBytes }); + expect(client.exec).toHaveBeenCalledTimes(2); + expect(client.exec).toHaveBeenNthCalledWith( + 2, + trackedBatch, + false, + { decoder: decoderBytes }, + ); - // Call 1 is the untracked SET; invalidation dispatches by its source SHA1. - expect(client.customCommand).toHaveBeenCalledTimes(2); + // Writes use batches; invalidation alone dispatches directly by source SHA1. + expect(client.customCommand).toHaveBeenCalledTimes(1); expect(client.customCommand).toHaveBeenNthCalledWith( - 2, + 1, [ "EVALSHA", createHash("sha1").update(INVALIDATE_CACHE_SCRIPT).digest("hex"), @@ -388,8 +486,41 @@ describe("Valkey GLIDE adapter", () => { expect(client.customCommand).not.toHaveBeenCalled(); }); + it("fails an untracked write whose placeholder was lost before the stamp", async () => { + const client = fakeClient([Buffer.from("OK"), 2]); + const adapter = createValkeyGlideDialCacheClient(client, mockGlide); + + const write = adapter.write({ + valueKey: "plain:value", + cacheTtlMs: 1_000, + value: "plain", + }); + await expect(write).rejects.toThrow("DialCache untracked write lost its placeholder before the stamp"); + await expect(write).rejects.toBeInstanceOf(DialCacheRedisPlaceholderLostError); + expect(client.customCommand).not.toHaveBeenCalled(); + }); + + it("falls back to the untracked stamp source only after NOSCRIPT", async () => { + const noscript = new Error("NOSCRIPT No matching script. Please use EVAL."); + const client = fakeClient([Buffer.from("OK"), noscript], 1); + const adapter = createValkeyGlideDialCacheClient(client, mockGlide); + + await expect(adapter.write({ + valueKey: "plain:value", + cacheTtlMs: 2_000, + value: "plain", + })).resolves.toBe(true); + + const frame = batchInstances[0]?.commands[0]?.[2] as Buffer; + expect(client.customCommand).toHaveBeenCalledOnce(); + expect(client.customCommand).toHaveBeenCalledWith( + ["EVAL", WRITE_UNTRACKED_STAMP_SCRIPT, "1", "plain:value", frame.subarray(1, 9)], + { decoder: decoderBytes }, + ); + }); + it("routes cluster writes and invalidations to the slot primary", async () => { - const client = fakeClusterClient("OK", ["OK", 1], 1); + const client = fakeClusterClient(["OK", 1], ["OK", 1], 1); const adapter = createValkeyGlideDialCacheClient(client, mockGlide); await expect( @@ -407,17 +538,16 @@ describe("Valkey GLIDE adapter", () => { adapter.invalidate({ watermarkKey: "tracked:{id}:watermark", futureBufferMs: 25 }), ).resolves.toBeUndefined(); - const [, untrackedOptions] = client.customCommand.mock.calls[0] ?? [[], undefined]; - expect(untrackedOptions).toEqual({ + expect(clusterBatchInstances).toHaveLength(2); + expect(client.exec).toHaveBeenNthCalledWith(1, clusterBatchInstances[0], false, { decoder: decoderBytes, route: { type: "primarySlotKey", key: "plain:value" }, }); - expect(clusterBatchInstances).toHaveLength(1); - expect(client.exec).toHaveBeenCalledWith(clusterBatchInstances[0], false, { + expect(client.exec).toHaveBeenNthCalledWith(2, clusterBatchInstances[1], false, { decoder: decoderBytes, route: { type: "primarySlotKey", key: "tracked:{id}:value" }, }); - const [, invalidateOptions] = client.customCommand.mock.calls[1] ?? [[], undefined]; + const [, invalidateOptions] = client.customCommand.mock.calls[0] ?? [[], undefined]; expect(invalidateOptions).toEqual({ decoder: decoderBytes, route: { type: "primarySlotKey", key: "tracked:{id}:watermark" }, @@ -527,24 +657,26 @@ describe("Valkey GLIDE adapter", () => { it("surfaces batched SET and stamp command errors", async () => { const setFailure = new Error("OOM command not allowed when used memory > 'maxmemory'."); - const setClient = fakeClient([setFailure, 1]); + const setClient = fakeClient([ + setFailure, + new Error("NOSCRIPT No matching script. Please use EVAL."), + ]); const setAdapter = createValkeyGlideDialCacheClient(setClient, mockGlide); await expect(setAdapter.write({ - valueKey: "tracked:{id}:value", - watermarkKey: "tracked:{id}:watermark", + valueKey: "plain:value", cacheTtlMs: 1_000, - value: "tracked", + value: "plain", })).rejects.toBe(setFailure); + // The failed SET wins even over NOSCRIPT, so no stamp recovery is dispatched. expect(setClient.customCommand).not.toHaveBeenCalled(); - const stampFailure = new Error("ERR invalid DialCache watermark"); + const stampFailure = new Error("ERR invalid DialCache stamp nonce"); const stampClient = fakeClient([Buffer.from("OK"), stampFailure]); const stampAdapter = createValkeyGlideDialCacheClient(stampClient, mockGlide); await expect(stampAdapter.write({ - valueKey: "tracked:{id}:value", - watermarkKey: "tracked:{id}:watermark", + valueKey: "plain:value", cacheTtlMs: 1_000, - value: "tracked", + value: "plain", })).rejects.toBe(stampFailure); expect(stampClient.customCommand).not.toHaveBeenCalled(); }); @@ -559,7 +691,7 @@ describe("Valkey GLIDE adapter", () => { value: "tracked", })).rejects.toBeInstanceOf(DialCacheRedisPayloadError); - const setReplyClient = fakeClient("QUEUED"); + const setReplyClient = fakeClient(["QUEUED", 1]); const setReplyAdapter = createValkeyGlideDialCacheClient(setReplyClient, mockGlide); await expectProtocolError( Promise.resolve(setReplyAdapter.write({ valueKey: "plain:value", cacheTtlMs: 1_000, value: "plain" })), @@ -582,24 +714,36 @@ describe("Valkey GLIDE adapter", () => { it("rejects malformed native read and mutation script replies", async () => { const client = fakeClient( - "not-bytes", - redisFrame("invalid", { encoding: 2 }), + ["not-bytes", redisTime(1_500)], + [redisFrame("invalid", { encoding: 2 }), redisTime(1_500)], "not-a-batch-reply", - [[redisFrame("missing-watermark")]], + [[redisFrame("missing-watermark")], redisTime(1_500)], [Buffer.from("OK"), "not-an-integer"], null, ); const adapter = createValkeyGlideDialCacheClient(client, mockGlide); - await expect(adapter.read({ valueKey: "wrong-type" })).rejects.toBeInstanceOf(DialCacheRedisPayloadError); - await expect(adapter.read({ valueKey: "wrong-encoding" })).rejects.toBeInstanceOf( + await expect( + adapter.read({ valueKey: "wrong-type", maxAgeMs: 1_000 }), + ).rejects.toBeInstanceOf(DialCacheRedisPayloadError); + await expect( + adapter.read({ valueKey: "wrong-encoding", maxAgeMs: 1_000 }), + ).rejects.toBeInstanceOf( DialCacheRedisPayloadEncodingError, ); await expect( - adapter.read({ valueKey: "bad:{id}:value", watermarkKey: "bad:{id}:watermark" }), + adapter.read({ + valueKey: "bad:{id}:value", + watermarkKey: "bad:{id}:watermark", + maxAgeMs: 1_000, + }), ).rejects.toBeInstanceOf(DialCacheRedisPayloadError); await expect( - adapter.read({ valueKey: "bad-pair:{id}:value", watermarkKey: "bad-pair:{id}:watermark" }), + adapter.read({ + valueKey: "bad-pair:{id}:value", + watermarkKey: "bad-pair:{id}:watermark", + maxAgeMs: 1_000, + }), ).rejects.toBeInstanceOf(DialCacheRedisPayloadError); await expectProtocolError( Promise.resolve(adapter.write({ @@ -618,10 +762,38 @@ describe("Valkey GLIDE adapter", () => { ); }); + it("rejects a malformed TIME result even when the paired value is a miss", async () => { + const client = fakeClient([null, [Buffer.from("1")]]); + const adapter = createValkeyGlideDialCacheClient(client, mockGlide); + + await expect( + adapter.read({ valueKey: "missing:value", maxAgeMs: 1_000 }), + ).rejects.toMatchObject({ + name: "DialCacheRedisPayloadError", + message: "Invalid DialCache Redis TIME reply; expected two unsigned decimal bulk strings", + }); + }); + it("rejects every out-of-domain write and invalidation reply", async () => { const writeMessage = "Invalid DialCache Redis write reply; expected integer 0, 1, or 2"; + const untrackedWriteMessage = "Invalid DialCache Redis untracked write reply; expected integer 1 or 2"; const invalidationMessage = "Invalid DialCache Redis invalidate reply; expected integer 1"; + for (const reply of INVALID_UNTRACKED_WRITE_REPLIES) { + const untracked = createValkeyGlideDialCacheClient( + fakeClient([Buffer.from("OK"), reply]), + mockGlide, + ); + await expectProtocolError( + Promise.resolve(untracked.write({ + valueKey: "plain:value", + cacheTtlMs: 1_000, + value: "plain", + })), + untrackedWriteMessage, + ); + } + for (const reply of INVALID_WRITE_REPLIES) { const tracked = createValkeyGlideDialCacheClient( fakeClient([Buffer.from("OK"), reply]), @@ -749,7 +921,7 @@ describe("Valkey GLIDE adapter", () => { Decoder: { Bytes: Symbol("other-bytes") }, }; const client = fakeClient( - [[redisFrame("tracked"), Buffer.from("0")]], + [[redisFrame("tracked"), Buffer.from("0")], redisTime(1_500)], [Buffer.from("OK"), 1], 1, ); @@ -758,6 +930,7 @@ describe("Valkey GLIDE adapter", () => { await adapter.read({ valueKey: "module:{instance}:value", watermarkKey: "module:{instance}:watermark", + maxAgeMs: 1_000, }); await adapter.write({ valueKey: "module:{instance}:value",