From 419ae64e30a26e9ff5e9d76375a1138d09784619 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Tue, 25 Aug 2026 07:23:03 +0200 Subject: [PATCH 1/2] perf(runtime): index captured closure reuse --- crates/perry-runtime/src/closure/alloc.rs | 279 ++++++++++++++++++---- 1 file changed, 235 insertions(+), 44 deletions(-) diff --git a/crates/perry-runtime/src/closure/alloc.rs b/crates/perry-runtime/src/closure/alloc.rs index 70c85b989a..f0c5b3da2b 100644 --- a/crates/perry-runtime/src/closure/alloc.rs +++ b/crates/perry-runtime/src/closure/alloc.rs @@ -12,20 +12,7 @@ crate::perry_thread_local! { static SINGLETON_CLOSURES: RefCell> = RefCell::new(crate::fast_hash::new_ptr_hash_map()); - /// Per-`func_ptr` single-slot cache for closures with captures. - /// Each value is `(last_captures, last_closure)` — when the same - /// closure literal is created again with the SAME capture bits, - /// we return the cached closure; otherwise we allocate a fresh - /// one and replace the slot. - /// - /// One entry per closure literal (bounded by the number of - /// `Expr::Closure` sites in the program), not per - /// `(func_ptr, capture-tuple)` pair — this prevents a closure - /// whose captures vary per call (e.g. - /// `getOrCompute(map, key, () => new Foo(sortedTypes))` capturing - /// a fresh array per call) from filling the cache and crowding - /// out closures with stable captures. - /// Per-`func_ptr` small-LRU cache. Each entry holds up to + /// Per-`func_ptr` small-LRU cache. Each value holds up to /// `MAX_CAPTURED_CLOSURE_SLOTS` (captures-bits, ClosureHeader) /// pairs. Multiple slots are critical for the parallel-instance /// async-await pattern (e.g. `Promise.all` of N async closures @@ -34,10 +21,207 @@ crate::perry_thread_local! { /// `PtrHasher`-keyed for the same reason as the other registries /// here — on `promise_all_chains` this is hit on every closure /// alloc (150 k/run). - static SINGLETON_CAPTURED_CLOSURES: RefCell, *mut ClosureHeader)>>> = + static SINGLETON_CAPTURED_CLOSURES: RefCell> = RefCell::new(crate::fast_hash::new_ptr_hash_map()); } +#[derive(Clone)] +struct CapturedClosureEntry { + /// Non-semantic prefilter for `captures`. Hash collisions always fall + /// through to the exact bitwise tuple comparison below. + fingerprint: u64, + captures: Vec, + closure: *mut ClosureHeader, + last_used: u64, +} + +/// Direct-mapped index into the exact entry vector. A collision only falls +/// back to the bounded scan; neither the fingerprint nor this hint is ever +/// trusted without exact capture-bit equality. +const CAPTURED_HINT_SLOTS: usize = 128; + +struct CapturedClosureCache { + entries: Vec, + hint_fingerprints: [u64; CAPTURED_HINT_SLOTS], + hint_indices_plus_one: [u8; CAPTURED_HINT_SLOTS], + clock: u64, +} + +impl CapturedClosureCache { + fn new() -> Self { + Self { + entries: Vec::new(), + hint_fingerprints: [0; CAPTURED_HINT_SLOTS], + hint_indices_plus_one: [0; CAPTURED_HINT_SLOTS], + clock: 0, + } + } + + #[inline] + fn hint_slot(fingerprint: u64) -> usize { + // Fold high bits before masking: capture pointers are aligned and FNV's + // low bits alone otherwise make avoidable direct-map collisions. + (fingerprint ^ (fingerprint >> 32)) as usize & (CAPTURED_HINT_SLOTS - 1) + } + + #[inline] + fn touch(&mut self, index: usize) -> *mut ClosureHeader { + self.clock = self.clock.wrapping_add(1); + self.entries[index].last_used = self.clock; + self.entries[index].closure + } + + fn lookup(&mut self, fingerprint: u64, captures: &[u64]) -> Option<*mut ClosureHeader> { + let hint_slot = Self::hint_slot(fingerprint); + let hinted = self.hint_indices_plus_one[hint_slot]; + if hinted != 0 && self.hint_fingerprints[hint_slot] == fingerprint { + let index = (hinted - 1) as usize; + if self + .entries + .get(index) + .is_some_and(|entry| entry.captures.as_slice() == captures) + { + return Some(self.touch(index)); + } + } + + let index = self.entries.iter().position(|entry| { + entry.fingerprint == fingerprint && entry.captures.as_slice() == captures + })?; + self.hint_fingerprints[hint_slot] = fingerprint; + self.hint_indices_plus_one[hint_slot] = (index + 1) as u8; + Some(self.touch(index)) + } + + fn insert(&mut self, fingerprint: u64, captures: Vec, closure: *mut ClosureHeader) { + self.clock = self.clock.wrapping_add(1); + let entry = CapturedClosureEntry { + fingerprint, + captures, + closure, + last_used: self.clock, + }; + let index = if self.entries.len() < MAX_CAPTURED_CLOSURE_SLOTS { + let index = self.entries.len(); + self.entries.push(entry); + index + } else { + let (index, _) = self + .entries + .iter() + .enumerate() + .min_by_key(|(_, entry)| entry.last_used) + .expect("full captured-closure cache must have an LRU entry"); + self.entries[index] = entry; + index + }; + let hint_slot = Self::hint_slot(fingerprint); + self.hint_fingerprints[hint_slot] = fingerprint; + self.hint_indices_plus_one[hint_slot] = (index + 1) as u8; + } + + fn clear_hints(&mut self) { + self.hint_indices_plus_one.fill(0); + } +} + +#[cfg(test)] +mod captured_closure_cache_tests { + use super::*; + + fn fake_closure(id: usize) -> *mut ClosureHeader { + // The cache treats these as opaque values. No test dereferences them. + (0x1000 + id * std::mem::align_of::()) as *mut ClosureHeader + } + + #[test] + fn direct_hint_collision_falls_back_to_exact_capture_match() { + let first = [0u64]; + let first_fingerprint = capture_fingerprint(&first); + let first_slot = CapturedClosureCache::hint_slot(first_fingerprint); + let second_word = (1..10_000u64) + .find(|&word| { + let fingerprint = capture_fingerprint(&[word]); + fingerprint != first_fingerprint + && CapturedClosureCache::hint_slot(fingerprint) == first_slot + }) + .expect("the bounded search must find a direct-hint collision"); + let second = [second_word]; + let second_fingerprint = capture_fingerprint(&second); + + let mut cache = CapturedClosureCache::new(); + cache.insert(first_fingerprint, first.to_vec(), fake_closure(1)); + cache.insert(second_fingerprint, second.to_vec(), fake_closure(2)); + + // Inserting `second` displaced `first` from their shared hint slot. + // Both lookups must still find their exact tuple via fallback, and + // each fallback must repair the hint for the next lookup. + assert_eq!( + cache.lookup(first_fingerprint, &first), + Some(fake_closure(1)) + ); + assert_eq!( + cache.lookup(first_fingerprint, &first), + Some(fake_closure(1)) + ); + assert_eq!( + cache.lookup(second_fingerprint, &second), + Some(fake_closure(2)) + ); + assert_eq!(cache.lookup(first_fingerprint, &second), None); + } + + #[test] + fn full_cache_evicts_least_recently_used_entry() { + let mut cache = CapturedClosureCache::new(); + for word in 0..MAX_CAPTURED_CLOSURE_SLOTS as u64 { + let captures = vec![word]; + cache.insert( + capture_fingerprint(&captures), + captures, + fake_closure(word as usize + 1), + ); + } + + let retained = [0u64]; + assert_eq!( + cache.lookup(capture_fingerprint(&retained), &retained), + Some(fake_closure(1)) + ); + + let replacement = [MAX_CAPTURED_CLOSURE_SLOTS as u64]; + cache.insert( + capture_fingerprint(&replacement), + replacement.to_vec(), + fake_closure(MAX_CAPTURED_CLOSURE_SLOTS + 1), + ); + + let evicted = [1u64]; + assert_eq!(cache.lookup(capture_fingerprint(&evicted), &evicted), None); + assert_eq!( + cache.lookup(capture_fingerprint(&retained), &retained), + Some(fake_closure(1)) + ); + assert_eq!( + cache.lookup(capture_fingerprint(&replacement), &replacement), + Some(fake_closure(MAX_CAPTURED_CLOSURE_SLOTS + 1)) + ); + } +} + +#[inline] +fn capture_fingerprint(captures: &[u64]) -> u64 { + // FNV-1a over fixed-width capture words. The cache never trusts this as an + // identity: it only avoids calling slice equality (and its outlined + // `memcmp`) for entries that cannot possibly match. + let mut hash = 0xcbf2_9ce4_8422_2325u64 ^ captures.len() as u64; + for &word in captures { + hash ^= word; + hash = hash.wrapping_mul(0x0100_0000_01b3); + } + hash +} + /// Header for heap-allocated closures #[repr(C)] pub struct ClosureHeader { @@ -222,13 +406,21 @@ pub fn scan_singleton_closure_roots_mut(visitor: &mut crate::gc::RuntimeRootVisi }); SINGLETON_CAPTURED_CLOSURES.with(|s| { let mut captured = s.borrow_mut(); - for slots in captured.values_mut() { - for (capture_key, closure) in slots.iter_mut() { - visitor.visit_raw_mut_ptr_slot(closure); - for word in capture_key.iter_mut() { + for cache in captured.values_mut() { + for entry in cache.entries.iter_mut() { + visitor.visit_raw_mut_ptr_slot(&mut entry.closure); + for word in entry.captures.iter_mut() { visitor.visit_heap_word_u64_slot(word); } + // A copying collection may have rewritten pointer-bearing + // capture words. Keep the non-semantic prefilter synchronized + // with the exact tuple that remains authoritative. + entry.fingerprint = capture_fingerprint(&entry.captures); } + // Fingerprints and exact tuples were just rewritten in place. A + // hint is disposable acceleration state; clearing avoids an + // address-derived pre-GC fingerprint/index ever being consulted. + cache.clear_hints(); } }); } @@ -256,8 +448,8 @@ pub(crate) fn test_seed_captured_singleton_closure_cache( SINGLETON_CAPTURED_CLOSURES.with(|s| { s.borrow_mut() .entry(func_ptr as usize) - .or_insert_with(Vec::new) - .insert(0, (capture_key, closure)); + .or_insert_with(CapturedClosureCache::new) + .insert(capture_fingerprint(&capture_key), capture_key, closure); }); } @@ -275,7 +467,13 @@ pub(crate) fn test_captured_singleton_closure_cache_entries( SINGLETON_CAPTURED_CLOSURES.with(|s| { s.borrow() .get(&(func_ptr as usize)) - .cloned() + .map(|cache| { + cache + .entries + .iter() + .map(|entry| (entry.captures.clone(), entry.closure)) + .collect() + }) .unwrap_or_default() }) } @@ -331,6 +529,7 @@ pub extern "C" fn js_closure_alloc_with_captures_singleton( } else { unsafe { std::slice::from_raw_parts(captures_ptr, n) } }; + let fingerprint = capture_fingerprint(captures_slice); // Adaptive bypass: if this func_ptr has missed the cache N times in // a row, skip the cache entirely. Async-step closures (`__step` / @@ -362,25 +561,15 @@ pub extern "C" fn js_closure_alloc_with_captures_singleton( return allocated; } - // Fast path: scan the per-`func_ptr` slot list looking for a - // matching capture-tuple. We touch only the cached `Vec` (small, - // bounded by MAX_CAPTURED_CLOSURE_SLOTS). The match check is - // bit-equality of u64 capture slots — same as a plain primitive - // value comparison. Move the matched entry to the front to keep - // recency information for the LRU eviction policy below. + // Fast path: use the tuple fingerprint's direct-mapped hint, then exact + // bit-equality of every capture slot. Hint collisions fall back to the + // bounded entry scan and repair the hint. Entries carry a monotonic + // last-use timestamp, so lookups no longer memmove the Vec yet full-cache + // eviction preserves the same least-recently-used policy. if let Some(cached) = SINGLETON_CAPTURED_CLOSURES.with(|s| { let mut s = s.borrow_mut(); - if let Some(slots) = s.get_mut(&(func_ptr as usize)) { - for i in 0..slots.len() { - if slots[i].0.as_slice() == captures_slice { - let entry = slots.remove(i); - let ptr = entry.1; - slots.insert(0, entry); - return Some(ptr); - } - } - } - None + s.get_mut(&(func_ptr as usize)) + .and_then(|cache| cache.lookup(fingerprint, captures_slice)) }) { crate::promise::bump(&CLOSURE_CAP_SINGLETON_HIT); // Cache hit — reset the streak so a workload that briefly @@ -419,11 +608,13 @@ pub extern "C" fn js_closure_alloc_with_captures_singleton( } SINGLETON_CAPTURED_CLOSURES.with(|s| { let mut s = s.borrow_mut(); - let slots = s.entry(func_ptr as usize).or_insert_with(Vec::new); - slots.insert(0, (rewritten_captures, allocated)); - if slots.len() > MAX_CAPTURED_CLOSURE_SLOTS { - slots.truncate(MAX_CAPTURED_CLOSURE_SLOTS); - } + s.entry(func_ptr as usize) + .or_insert_with(CapturedClosureCache::new) + .insert( + capture_fingerprint(&rewritten_captures), + rewritten_captures, + allocated, + ); }); // Bump the miss-streak counter; flip to disabled sentinel when we // hit the threshold. From dd90a6566acf546172a490e064590bbb6f891645 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Tue, 25 Aug 2026 07:26:39 +0200 Subject: [PATCH 2/2] docs(changelog): note captured closure hints --- changelog.d/8792-captured-closure-cache-hints.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/8792-captured-closure-cache-hints.md diff --git a/changelog.d/8792-captured-closure-cache-hints.md b/changelog.d/8792-captured-closure-cache-hints.md new file mode 100644 index 0000000000..31c8dcb383 --- /dev/null +++ b/changelog.d/8792-captured-closure-cache-hints.md @@ -0,0 +1 @@ +Captured-closure singleton reuse now fingerprints each exact capture tuple and maintains a direct-mapped, collision-safe hint into its bounded LRU. Hits no longer move vector entries, hash or hint collisions still require bit-exact capture equality, and copying GC recomputes fingerprints while discarding stale hints after rewriting pointer-bearing captures.