From 99ab728031af95a3db3590c17b2e4baafb85221e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Tue, 25 Aug 2026 23:53:16 +0200 Subject: [PATCH] perf: reuse guarded ECS entity indices --- .../8839-guarded-ecs-entity-index-cache.md | 6 + crates/perry-codegen/src/block.rs | 89 ++++- crates/perry-codegen/src/expr/mod.rs | 29 ++ .../src/stmt/stable_packed_loop.rs | 361 +++++++++++++++++- ...issue_8773_closure_capture_packed_loops.rs | 25 ++ 5 files changed, 489 insertions(+), 21 deletions(-) create mode 100644 changelog.d/8839-guarded-ecs-entity-index-cache.md diff --git a/changelog.d/8839-guarded-ecs-entity-index-cache.md b/changelog.d/8839-guarded-ecs-entity-index-cache.md new file mode 100644 index 0000000000..3b01d6ba94 --- /dev/null +++ b/changelog.d/8839-guarded-ecs-entity-index-cache.md @@ -0,0 +1,6 @@ +Nested stable-packed ECS loops now reuse an admitted receiver proof on call-free paths and cache +repeated reads of the same entity index. Semantic calls invalidate both facts before execution; the +next indexed access reloads the rooted receiver and revalidates it, with an exact generic read on +failure rather than replaying prior iteration effects. This preserves getters, proxies, exceptions, +mutation, and moving-GC behavior while making the unchanged Wolf `simple_iter` kernel 8.82% faster +in an 11-pair controlled cohort (11/11 wins and 30/30 semantic-oracle passes). diff --git a/crates/perry-codegen/src/block.rs b/crates/perry-codegen/src/block.rs index be7ad0c5f7..be659c1264 100644 --- a/crates/perry-codegen/src/block.rs +++ b/crates/perry-codegen/src/block.rs @@ -92,6 +92,18 @@ pub struct RegCounter { /// callee is UB, so the registry, not the emitting code, is the single /// source of truth. `None` for functions built outside a module (tests). preserve_none_fns: RefCell>>>>, + /// Compiler-private validity bits for nested stable-packed loop proofs. + /// + /// A guarded inner receiver may keep its raw address across call-free + /// direct-load/store arms. Every actually executed runtime/indirect call + /// that can collect or run semantic heap work dirties the active proofs + /// before control can enter the callee; the next indexed read then reloads + /// its GC root and revalidates. Root bookkeeping and write barriers are + /// proof-preserving: they neither collect nor change JS-visible receiver + /// state. Keeping this at the call-emission choke point makes invalidation + /// path-sensitive: cold IC misses dirty the proof, while their unexecuted + /// hot siblings do not impose a revalidation on every read. + stable_packed_revalidation_slots: RefCell>, } impl RegCounter { @@ -101,9 +113,29 @@ impl RegCounter { eh_unwind_labels: RefCell::new(Vec::new()), shadow_slot_allocas: RefCell::new(HashSet::new()), preserve_none_fns: RefCell::new(None), + stable_packed_revalidation_slots: RefCell::new(Vec::new()), } } + pub(crate) fn push_stable_packed_revalidation_slot(&self, slot: String) { + self.stable_packed_revalidation_slots + .borrow_mut() + .push(slot); + } + + pub(crate) fn pop_stable_packed_revalidation_slot(&self, expected: &str) { + let actual = self + .stable_packed_revalidation_slots + .borrow_mut() + .pop() + .expect("stable-packed revalidation slot stack underflow"); + debug_assert_eq!(actual, expected); + } + + fn stable_packed_revalidation_slots(&self) -> Vec { + self.stable_packed_revalidation_slots.borrow().clone() + } + /// Install the module's `preserve_nonecc` symbol registry (#8175). Called /// once per function by `LlModule::define_function`; the shared cell means /// registration order does not matter — reads happen at call-emission and @@ -278,6 +310,26 @@ impl LlBlock { format!("%r{}", self.counter.next()) } + /// Invalidate every nested packed receiver whose live raw address may be + /// observed after this call. Intrinsics cannot enter Perry or user code. + /// Shadow-stack operations and write barriers are also safe: both families + /// are noncollecting GC bookkeeping and cannot mutate the guarded object's + /// JS-visible shape, prototype, length, or indexed values. Every other + /// direct call stays conservative, including unknown GC-leaf helpers that + /// may perform a semantic write without collecting. + fn dirty_stable_packed_revalidations_before_call(&mut self, direct_callee: Option<&str>) { + if direct_callee.is_some_and(|callee| { + callee.starts_with("llvm.") + || callee.starts_with("js_shadow_") + || callee.starts_with("js_write_barrier") + }) { + return; + } + for slot in self.counter.stable_packed_revalidation_slots() { + self.store(crate::types::I1, "1", &slot); + } + } + pub fn next_reg(&self) -> String { self.reg() } @@ -1245,6 +1297,7 @@ impl LlBlock { args: &[(LlvmType, &str)], gc_leaf: bool, ) -> String { + self.dirty_stable_packed_revalidations_before_call(Some(func_name)); // #835 + #846: record this emission against the FFI provenance // registry. The driver consults the registry after all per-module // codegen finishes to auto-link the providing crate. @@ -1285,6 +1338,7 @@ impl LlBlock { } pub fn call_void(&mut self, func_name: &str, args: &[(LlvmType, &str)]) { + self.dirty_stable_packed_revalidations_before_call(Some(func_name)); // #835 + #846: same registry hook as `call` — see comment there. crate::ext_registry::record_ffi_call(func_name); self.counter @@ -1353,6 +1407,7 @@ impl LlBlock { args: &[(LlvmType, &str)], gc_leaf: bool, ) -> String { + self.dirty_stable_packed_revalidations_before_call(None); let r = self.reg(); // Indirect targets (closures, method pointers) can always throw. if let Some(lpad) = self.counter.current_eh_unwind_label() { @@ -1492,7 +1547,7 @@ fn format_args(args: &[(LlvmType, &str)]) -> String { #[cfg(test)] mod tests { use super::*; - use crate::types::{DOUBLE, I64}; + use crate::types::{DOUBLE, I64, PTR}; use std::thread; fn fresh() -> LlBlock { @@ -1611,6 +1666,38 @@ mod tests { .contains("call double @js_nanbox_string(i64 %handle)")); } + #[test] + fn active_stable_packed_proofs_are_dirtied_only_by_executed_non_intrinsic_calls() { + let mut b = fresh(); + b.counter + .push_stable_packed_revalidation_slot("%proof_dirty".to_string()); + b.call(DOUBLE, "llvm.fabs.f64", &[(DOUBLE, "%value")]); + b.call_void("js_shadow_slot_bind", &[(I64, "0"), (PTR, "%root")]); + b.call_void("js_write_barrier_root_nanbox", &[(I64, "%bits")]); + b.call(DOUBLE, "js_dyn_index_get", &[(DOUBLE, "%object")]); + b.call_indirect(DOUBLE, "%callback", &[(DOUBLE, "%value")]); + b.counter + .pop_stable_packed_revalidation_slot("%proof_dirty"); + b.call(DOUBLE, "js_dyn_index_get", &[(DOUBLE, "%object")]); + + let ir = b.to_ir(); + assert_eq!( + ir.matches("store i1 1, ptr %proof_dirty").count(), + 2, + "{ir}" + ); + assert!( + ir.find("call double @llvm.fabs.f64") < ir.find("store i1 1, ptr %proof_dirty"), + "proof-preserving calls must not dirty the proof: {ir}" + ); + let first_dirty = ir.find("store i1 1, ptr %proof_dirty").unwrap(); + assert!( + ir.find("@js_shadow_slot_bind").unwrap() < first_dirty + && ir.find("@js_write_barrier_root_nanbox").unwrap() < first_dirty, + "GC bookkeeping must preserve the proof: {ir}" + ); + } + #[test] fn direct_gc_leaf_call_places_the_callsite_attribute_after_arguments() { let mut b = fresh(); diff --git a/crates/perry-codegen/src/expr/mod.rs b/crates/perry-codegen/src/expr/mod.rs index 2a78e17793..1b16fbcb8d 100644 --- a/crates/perry-codegen/src/expr/mod.rs +++ b/crates/perry-codegen/src/expr/mod.rs @@ -1656,6 +1656,22 @@ pub(crate) struct StablePackedNumericAccess { pub object_spill_base: String, } +#[derive(Clone, Debug)] +pub(crate) struct StablePackedReadCache { + /// The cache is keyed by the scalar loop counter rather than assumed to + /// expire on the back edge. This remains correct through `continue` edges + /// and lets LLVM promote all three slots without relying on block layout. + pub valid_slot: String, + pub counter_slot: String, + /// A boxed JS value. It is not a GC root: any call that could move a + /// pointer dirties the associated proof before entering the callee, and a + /// dirty cache is never loaded. + pub value_slot: String, + /// Compile-time source-order state. The first lowered occurrence only + /// populates the slots; later occurrences emit a runtime hit/miss test. + pub has_producer: bool, +} + #[derive(Clone, Debug)] pub(crate) struct StablePackedLoopFact { pub counter_local_id: u32, @@ -1678,6 +1694,19 @@ pub(crate) struct StablePackedLoopFact { /// use, after those temporaries, so none of their runtime loads can leave a /// stale raw address. pub revalidate_before_indexed_read: bool, + /// Path-sensitive validity bit for a nested-derived raw receiver. Calls + /// set it before entering the callee; a successful exact revalidation + /// clears it. LLVM promotes the compiler-private alloca to SSA, so the + /// clean hot arm is one branch and no runtime call. + pub revalidation_dirty_slot: Option, + /// Non-root cache paired with `revalidation_dirty_slot`. It is read only + /// on the clean arm; a call dirties the proof before a moving collection, + /// and successful revalidation refreshes this word before clearing it. + pub revalidation_live_raw_slot: Option, + /// One exact `array[counter]` result shared by repeated occurrences in the + /// same source iteration. A hit additionally requires a clean revalidation + /// proof, so observable calls force an exact reread at the next occurrence. + pub repeated_read_cache: Option, pub live_receiver_handle: Option, /// Admission scanned the complete indexed range and proved every value is /// an untagged IEEE Number. This is requested only when the indexed value diff --git a/crates/perry-codegen/src/stmt/stable_packed_loop.rs b/crates/perry-codegen/src/stmt/stable_packed_loop.rs index a41e5f46d2..41cfaa47a2 100644 --- a/crates/perry-codegen/src/stmt/stable_packed_loop.rs +++ b/crates/perry-codegen/src/stmt/stable_packed_loop.rs @@ -8,7 +8,7 @@ use anyhow::Result; use perry_hir::{CompareOp, Expr, Stmt, UpdateOp}; -use crate::expr::{FnCtx, StablePackedLoopFact, StablePackedNumericAccess}; +use crate::expr::{FnCtx, StablePackedLoopFact, StablePackedNumericAccess, StablePackedReadCache}; use crate::native_value::{BoundsState, BufferAccessMode, LoweredValue, MaterializationReason}; use crate::types::{DOUBLE, I1, I32, I64, PTR}; @@ -27,6 +27,7 @@ struct Candidate { capture_uses_box: bool, nested_derived: bool, nested_requires_access_revalidation: bool, + cache_repeated_index_reads: bool, } fn target_below_numeric_operator( @@ -111,6 +112,150 @@ fn stmt_flags(stmt: &Stmt, array_id: u32, counter_id: u32) -> (bool, bool) { (target, call) } +/// Count exact `receiver[counter]` reads without descending into closures. +fn target_read_count(expr: &Expr, array_id: u32, counter_id: u32) -> usize { + if matches!( + expr, + Expr::IndexGet { object, index } + if matches!(object.as_ref(), Expr::LocalGet(id) if *id == array_id) + && matches!(index.as_ref(), Expr::LocalGet(id) if *id == counter_id) + ) { + return 1; + } + if matches!(expr, Expr::Closure { .. }) { + return 0; + } + let mut count = 0; + perry_hir::walker::walk_expr_children(expr, &mut |child| { + count += target_read_count(child, array_id, counter_id); + }); + count +} + +/// Count exact target reads in the straight-line statements admitted here. +fn body_target_read_count(body: &[Stmt], array_id: u32, counter_id: u32) -> usize { + body.iter() + .map(|stmt| match stmt { + Stmt::Let { + init: Some(expr), .. + } + | Stmt::Expr(expr) + | Stmt::Throw(expr) + | Stmt::Return(Some(expr)) => target_read_count(expr, array_id, counter_id), + _ => 0, + }) + .sum() +} + +/// A repeated element value can survive calls only through the dirty-bit +/// protocol below. Direct writes need a separate alias argument. A statically +/// proven TypedArray store is brand-disjoint from the admitted +/// Array/Array-subclass receiver. An erased IndexSet has the same property on +/// its sole no-call arm; every other brand crosses a dirtying runtime call. +/// Property writes, statically Array stores, and in-place Array operations +/// disable value caching. +fn indexed_store_direct_arm_is_brand_disjoint(ctx: &FnCtx<'_>, object: &Expr) -> bool { + matches!( + crate::type_analysis::static_type_of(ctx, object), + None | Some(perry_hir::types::Type::Any) | Some(perry_hir::types::Type::Unknown) + ) || crate::type_analysis::is_typed_array_expr(ctx, object) +} + +/// Whether `expr` has a direct mutation arm that can alias the cached Array. +fn expr_blocks_repeated_read_cache(ctx: &FnCtx<'_>, expr: &Expr) -> bool { + if matches!( + expr, + Expr::PropertySet { .. } + | Expr::PropertyUpdate { .. } + | Expr::SuperPropertySet { .. } + | Expr::ObjectSuperPropertySet { .. } + | Expr::ObjectAssign { .. } + | Expr::ObjectDefineProperty(..) + | Expr::ObjectDefineProperties(..) + | Expr::ObjectSetPrototypeOf(..) + | Expr::ArrayPush { .. } + | Expr::ArrayPushSpread { .. } + | Expr::ArrayPop(..) + | Expr::ArrayShift(..) + | Expr::ArrayUnshift { .. } + | Expr::ArraySplice { .. } + | Expr::ArraySort { .. } + | Expr::ArrayReverseValue { .. } + | Expr::ArrayCopyWithin { .. } + | Expr::ArrayCopyWithinValue { .. } + ) { + return true; + } + if let Expr::IndexSet { object, .. } = expr { + // An erased IndexSet's only no-call direct arm is the guarded + // TypedArray store; every other brand reaches `js_dyn_index_set`, + // which dirties the proof before mutating. This is exactly Wolf's + // unannotated component-column shape. A statically Array-typed store, + // on the other hand, can directly mutate an alias of the source. + if !indexed_store_direct_arm_is_brand_disjoint(ctx, object) { + return true; + } + } + if let Expr::PutValueSet { + target, + key, + receiver, + .. + } = expr + { + // Source assignments reach HIR as PutValueSet. The codegen's narrow + // same-receiver, non-string-key route immediately delegates to the + // IndexSet arm described above. Match only the side-effect-free local + // identity form here; every explicit-receiver or computed-base form + // remains conservatively blocked. + let same_local = matches!( + (target.as_ref(), receiver.as_ref()), + (Expr::LocalGet(target_id), Expr::LocalGet(receiver_id)) + if target_id == receiver_id + ); + let static_string_or_symbol = matches!( + key.as_ref(), + Expr::String(_) | Expr::WtfString(_) | Expr::SymbolFor(_) + ) || crate::type_analysis::is_string_expr(ctx, key); + if !same_local + || static_string_or_symbol + || !indexed_store_direct_arm_is_brand_disjoint(ctx, target) + { + return true; + } + } + if let Expr::IndexUpdate { object, .. } = expr { + // The update lowering has more direct receiver arms than IndexSet, so + // require a static TypedArray brand rather than admitting `any`. + if !crate::type_analysis::is_typed_array_expr(ctx, object) { + return true; + } + } + if matches!(expr, Expr::Closure { .. }) { + return false; + } + let mut blocked = false; + perry_hir::walker::walk_expr_children(expr, &mut |child| { + if !blocked && expr_blocks_repeated_read_cache(ctx, child) { + blocked = true; + } + }); + blocked +} + +/// Whether any admitted body statement can directly invalidate the cache. +fn body_blocks_repeated_read_cache(ctx: &FnCtx<'_>, body: &[Stmt]) -> bool { + body.iter().any(|stmt| match stmt { + Stmt::Let { + init: Some(expr), .. + } + | Stmt::Expr(expr) + | Stmt::Throw(expr) + | Stmt::Return(Some(expr)) => expr_blocks_repeated_read_cache(ctx, expr), + _ => false, + }) +} + /// The direct read must be in the first straight-line statement and before any /// explicit user call. Later statements may allocate or invoke callbacks: the /// next iteration reloads the root and validates before using it again. @@ -274,6 +419,9 @@ fn match_candidate( let nested_derived = derived_parent.is_some(); let nested_requires_access_revalidation = derived_parent .is_some_and(|fact| fact.revalidate_each_iteration || fact.revalidate_before_indexed_read); + let cache_repeated_index_reads = nested_requires_access_revalidation + && body_target_read_count(body, array_id, counter_id) > 1 + && !body_blocks_repeated_read_cache(ctx, body); let storage_is_available = capture_index.is_some() || (ctx.locals.contains_key(&array_id) && !ctx.boxed_vars.contains(&array_id)) || (!ctx.locals.contains_key(&array_id) && ctx.module_globals.contains_key(&array_id)); @@ -346,6 +494,7 @@ fn match_candidate( capture_uses_box: capture_index.is_some() && ctx.boxed_vars.contains(&array_id), nested_derived, nested_requires_access_revalidation, + cache_repeated_index_reads, }) } @@ -516,6 +665,9 @@ fn record_artifacts(ctx: &mut FnCtx<'_>, candidate: &Candidate, receiver: &str) .push("nested_read_miss=generic_read_without_iteration_replay".to_string()); } } + if candidate.cache_repeated_index_reads { + selected_facts.push("same_counter_read_cache=call_invalidated".to_string()); + } ctx.record_lowered_value_with_access_mode_and_facts( "StablePackedArraylikeLoop", Some(array_id), @@ -580,10 +732,30 @@ pub(crate) fn try_lower_index_get( // so it must dominate both successors. let counter_slot = ctx.i32_counter_slots.get(counter_id)?.clone(); let idx_i32 = ctx.block().load(I32, &counter_slot); + let repeated_read_cache = begin_repeated_read_cache(ctx, &fact, &idx_i32); let mut per_read_fallback = None; + let mut per_read_live_raw = None; + let mut per_read_numeric_access = None; if fact.revalidate_before_indexed_read { + let dirty_slot = fact.revalidation_dirty_slot.as_ref()?; + let live_raw_slot = fact.revalidation_live_raw_slot.as_ref()?; let receiver_slot = ctx.locals.get(array_id)?.clone(); let receiver = ctx.block().load(DOUBLE, &receiver_slot); + let dirty = ctx.block().load(I1, dirty_slot); + let validate_idx = ctx.new_block("stable_packed.indexed_read.proof_dirty"); + let clean_idx = ctx.new_block("stable_packed.indexed_read.proof_clean"); + let live_merge_idx = ctx.new_block("stable_packed.indexed_read.live_merge"); + let validate_label = ctx.block_label(validate_idx); + let clean_label = ctx.block_label(clean_idx); + let live_merge_label = ctx.block_label(live_merge_idx); + ctx.block().cond_br(&dirty, &validate_label, &clean_label); + + ctx.current_block = clean_idx; + let clean_raw = ctx.block().load(I64, live_raw_slot); + let clean_end = ctx.block().label.clone(); + ctx.block().br(&live_merge_label); + + ctx.current_block = validate_idx; let live_raw = ctx.block().call( I64, "js_packed_arraylike_loop_revalidate_live", @@ -614,18 +786,20 @@ pub(crate) fn try_lower_index_get( let fallback_label = ctx.block_label(fallback_idx); ctx.block().cond_br(&pass, &continue_label, &fallback_label); ctx.current_block = continue_idx; - let numeric_access = fact + ctx.block().store(I64, &live_raw, live_raw_slot); + ctx.block().store(I1, "0", dirty_slot); + let validated_end = ctx.block().label.clone(); + ctx.block().br(&live_merge_label); + + ctx.current_block = live_merge_idx; + let merged_live_raw = ctx.block().phi( + I64, + &[(&clean_raw, &clean_end), (&live_raw, &validated_end)], + ); + per_read_numeric_access = fact .numeric_elements - .then(|| build_numeric_access(ctx, &fact.descriptor, &live_raw)); - let active = ctx - .stable_packed_loop_facts - .iter_mut() - .rev() - .find(|active| { - active.array_local_id == *array_id && active.counter_local_id == *counter_id - })?; - active.live_receiver_handle = Some(live_raw); - active.numeric_access = numeric_access; + .then(|| build_numeric_access(ctx, &fact.descriptor, &merged_live_raw)); + per_read_live_raw = Some(merged_live_raw); per_read_fallback = Some((fallback_idx, read_merge_idx, fallback_label, receiver)); } let fact = ctx @@ -634,9 +808,9 @@ pub(crate) fn try_lower_index_get( .rev() .find(|fact| fact.array_local_id == *array_id && fact.counter_local_id == *counter_id)? .clone(); - let raw = fact.live_receiver_handle?; + let raw = per_read_live_raw.or(fact.live_receiver_handle)?; let idx_i64 = ctx.block().zext(I32, &idx_i32, I64); - if let Some(access) = fact.numeric_access { + if let Some(access) = per_read_numeric_access.or(fact.numeric_access) { let byte_offset = ctx.block().shl(I64, &idx_i64, "3"); let plain_addr = ctx.block().add(I64, &access.plain_base, &byte_offset); let inline_addr = ctx @@ -656,11 +830,12 @@ pub(crate) fn try_lower_index_get( .select(I1, &access.is_plain, I64, &plain_addr, &object_addr); let element_ptr = ctx.block().inttoptr(I64, &element_addr); let direct = ctx.block().load(DOUBLE, &element_ptr); - return Some(finish_revalidated_read( + let resolved = finish_revalidated_read(ctx, direct, idx_i32.clone(), per_read_fallback); + return Some(finish_repeated_read_cache( ctx, - direct, + resolved, idx_i32, - per_read_fallback, + repeated_read_cache, )); } let kind = descriptor_word(ctx, &fact.descriptor, 0); @@ -784,15 +959,120 @@ pub(crate) fn try_lower_index_get( (&spill_value, &spill_end), ], ); - Some(finish_revalidated_read( + let resolved = finish_revalidated_read(ctx, direct, idx_i32.clone(), per_read_fallback); + Some(finish_repeated_read_cache( ctx, - direct, + resolved, idx_i32, - per_read_fallback, + repeated_read_cache, )) } type PerReadFallback = (usize, usize, String, String); +enum RepeatedReadCacheMiss { + Populate(StablePackedReadCache), + Lookup { + cache: StablePackedReadCache, + cached: String, + hit_end: String, + merge_idx: usize, + }, +} + +/// Enter the miss arm of a same-counter value cache. A cached value is read +/// only when no semantic call has executed since it was produced; this is what +/// makes an unrooted boxed pointer safe under the moving collector as well as +/// preserving getters, proxies, and mutation between source occurrences. +fn begin_repeated_read_cache( + ctx: &mut FnCtx<'_>, + fact: &StablePackedLoopFact, + idx_i32: &str, +) -> Option { + let mut cache = fact.repeated_read_cache.clone()?; + let active = ctx + .stable_packed_loop_facts + .iter_mut() + .rev() + .find(|active| { + active.array_local_id == fact.array_local_id + && active.counter_local_id == fact.counter_local_id + })?; + if !active + .repeated_read_cache + .as_ref() + .is_some_and(|active_cache| active_cache.has_producer) + { + active + .repeated_read_cache + .as_mut() + .expect("active repeated-read cache") + .has_producer = true; + cache.has_producer = true; + return Some(RepeatedReadCacheMiss::Populate(cache)); + } + let dirty_slot = fact.revalidation_dirty_slot.as_ref()?; + let valid = ctx.block().load(I1, &cache.valid_slot); + let cached_counter = ctx.block().load(I32, &cache.counter_slot); + let same_counter = ctx.block().icmp_eq(I32, &cached_counter, idx_i32); + let dirty = ctx.block().load(I1, dirty_slot); + let clean = ctx.block().icmp_eq(I1, &dirty, "0"); + let valid_and_same = ctx.block().and(I1, &valid, &same_counter); + let hit = ctx.block().and(I1, &valid_and_same, &clean); + let hit_idx = ctx.new_block("stable_packed.indexed_read.cache_hit"); + let miss_idx = ctx.new_block("stable_packed.indexed_read.cache_miss"); + let merge_idx = ctx.new_block("stable_packed.indexed_read.cache_merge"); + let hit_label = ctx.block_label(hit_idx); + let miss_label = ctx.block_label(miss_idx); + let merge_label = ctx.block_label(merge_idx); + ctx.block().cond_br(&hit, &hit_label, &miss_label); + + ctx.current_block = hit_idx; + let cached = ctx.block().load(DOUBLE, &cache.value_slot); + let hit_end = ctx.block().label.clone(); + ctx.block().br(&merge_label); + + ctx.current_block = miss_idx; + Some(RepeatedReadCacheMiss::Lookup { + cache, + cached, + hit_end, + merge_idx, + }) +} + +/// Publish a miss value, then merge it with any previously emitted hit arm. +fn finish_repeated_read_cache( + ctx: &mut FnCtx<'_>, + resolved: String, + idx_i32: String, + cache_miss: Option, +) -> String { + let Some(cache_miss) = cache_miss else { + return resolved; + }; + let (cache, hit) = match cache_miss { + RepeatedReadCacheMiss::Populate(cache) => (cache, None), + RepeatedReadCacheMiss::Lookup { + cache, + cached, + hit_end, + merge_idx, + } => (cache, Some((cached, hit_end, merge_idx))), + }; + ctx.block().store(I32, &idx_i32, &cache.counter_slot); + ctx.block().store(DOUBLE, &resolved, &cache.value_slot); + ctx.block().store(I1, "1", &cache.valid_slot); + let Some((cached, hit_end, merge_idx)) = hit else { + return resolved; + }; + let miss_end = ctx.block().label.clone(); + let merge_label = ctx.block_label(merge_idx); + ctx.block().br(&merge_label); + + ctx.current_block = merge_idx; + ctx.block() + .phi(DOUBLE, &[(&cached, &hit_end), (&resolved, &miss_end)]) +} /// Complete a nested-derived indexed read. The direct arm has already /// consumed the live raw address with no intervening safepoint. On a guard or @@ -975,6 +1255,39 @@ pub(super) fn lower( } else { None }; + let revalidation_dirty_slot = candidate + .nested_requires_access_revalidation + .then(|| ctx.func.alloca_entry(I1)); + let revalidation_live_raw_slot = candidate + .nested_requires_access_revalidation + .then(|| ctx.func.alloca_entry(I64)); + let repeated_read_cache = candidate + .cache_repeated_index_reads + .then(|| StablePackedReadCache { + valid_slot: ctx.func.alloca_entry(I1), + counter_slot: ctx.func.alloca_entry(I32), + value_slot: ctx.func.alloca_entry(DOUBLE), + has_producer: false, + }); + if let Some(cache) = repeated_read_cache.as_ref() { + ctx.block().store(I1, "0", &cache.valid_slot); + } + if let Some(slot) = revalidation_dirty_slot.as_ref() { + // The admitting guard and the post-guard receiver reload establish a + // clean proof. Calls emitted after this point dirty it at their actual + // control-flow location via LlBlock's call choke points. + ctx.block().store(I1, "0", slot); + ctx.block().store( + I64, + &fast_raw, + revalidation_live_raw_slot + .as_ref() + .expect("nested revalidation raw slot"), + ); + ctx.func + .reg_counter() + .push_stable_packed_revalidation_slot(slot.clone()); + } ctx.stable_packed_loop_facts.push(StablePackedLoopFact { counter_local_id: candidate.counter_id, array_local_id: candidate.array_id, @@ -985,6 +1298,9 @@ pub(super) fn lower( live_length_bound: matches!(candidate.bound, LoopBound::LiveLength), revalidate_each_iteration: candidate.capture_index.is_some(), revalidate_before_indexed_read: candidate.nested_requires_access_revalidation, + revalidation_dirty_slot: revalidation_dirty_slot.clone(), + revalidation_live_raw_slot, + repeated_read_cache, live_receiver_handle: Some(fast_raw), numeric_elements: candidate.numeric_elements, numeric_access, @@ -1000,6 +1316,11 @@ pub(super) fn lower( Some((candidate.counter_id, bound_i32)), )?; ctx.stable_packed_loop_facts.pop(); + if let Some(slot) = revalidation_dirty_slot.as_ref() { + ctx.func + .reg_counter() + .pop_stable_packed_revalidation_slot(slot); + } if !ctx.block().is_terminated() { // A call-free clone cannot grow or shrink its receiver, so exhausting // the admitted bound is also the exact live-length loop exit. diff --git a/crates/perry/tests/issue_8773_closure_capture_packed_loops.rs b/crates/perry/tests/issue_8773_closure_capture_packed_loops.rs index 919a0024f2..f570e5a28b 100644 --- a/crates/perry/tests/issue_8773_closure_capture_packed_loops.rs +++ b/crates/perry/tests/issue_8773_closure_capture_packed_loops.rs @@ -183,6 +183,29 @@ console.log(checksum); ); assert!(ir.contains("stable_packed.iteration.capture_valid")); assert!(ir.contains("call i64 @js_packed_arraylike_loop_guard_live(")); + let clean_read_blocks = named_blocks(&ir, &["stable_packed.indexed_read.proof_clean"]); + let dirty_read_blocks = named_blocks(&ir, &["stable_packed.indexed_read.proof_dirty"]); + assert!( + !clean_read_blocks.is_empty() + && !clean_read_blocks.contains("js_packed_arraylike_loop_revalidate_live"), + "the clean nested-read path must retain its proof without a runtime call\n{clean_read_blocks}" + ); + assert!( + dirty_read_blocks.contains("js_packed_arraylike_loop_revalidate_live"), + "a path dirtied by a preceding call must retain exact revalidation\n{dirty_read_blocks}" + ); + let cache_hit_blocks = named_blocks(&ir, &["stable_packed.indexed_read.cache_hit"]); + assert!( + !cache_hit_blocks.is_empty() + && !cache_hit_blocks.contains("js_packed_arraylike_loop_revalidate_live") + && !cache_hit_blocks.contains("js_packed_arraylike_index_get"), + "a same-counter cache hit must be a call-free exact-value load\n{cache_hit_blocks}" + ); + assert!( + ir.contains("packed_index.generic_fallback") + && ir.contains("packed_index.revalidated_merge"), + "a failed nested proof must branch to the exact-source generic read and rejoin" + ); let fast_blocks = named_blocks(&ir, &["stable_packed", "for.stable_packed_fast"]); assert!( @@ -203,6 +226,8 @@ console.log(checksum); "candidate_storage=closure_capture_slot", "revalidation=each_iteration_capture_reload", "candidate_origin=guarded_outer_index_read", + "nested_read_miss=generic_read_without_iteration_replay", + "same_counter_read_cache=call_invalidated", "guard_identity=stable_packed_arraylike:", "fallback_identity=stable_packed_arraylike:", ] {