diff --git a/crates/perry-codegen/src/expr/closure.rs b/crates/perry-codegen/src/expr/closure.rs index 1fcfb52a90..f53c588aef 100644 --- a/crates/perry-codegen/src/expr/closure.rs +++ b/crates/perry-codegen/src/expr/closure.rs @@ -340,6 +340,16 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { None }; + // Bulk-init admission: a fresh closure whose captures are all plain + // bits. Box-cell captures keep the per-slot setter path — their + // `set_closure_box_capture` bookkeeping has no bulk twin. + let bulk_fresh_init = !no_capture_singleton + && !captured_singleton + && total_caps > 0 + && !captured_value_bits.is_empty() + && auto_captures + .iter() + .all(|cap_id| is_plain_async_step || !ctx.boxed_vars.contains(cap_id)); let closure_handle = if no_capture_singleton { let blk = ctx.block(); blk.call(I64, "js_closure_alloc_singleton", &[(PTR, &func_ref)]) @@ -374,6 +384,32 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { "js_closure_alloc_with_captures_singleton", &[(PTR, &func_ref), (I32, &cap_count), (PTR, &buf)], ) + } else if bulk_fresh_init { + // Fresh (identity-carrying) closure with plain-bits captures: + // ONE runtime call does allocation + slots + layout instead of + // `js_closure_alloc` plus a `js_closure_set_capture_bits` per + // capture (each re-resolving the header, forwarding, kind + // dispatch and the barrier's page classification). The reserved + // `this` / `new.target` slots are pre-filled with the pointer-free + // sentinel the per-slot path relied on `js_closure_alloc` writing; + // the post-create patch below fills them exactly as before. + let buf = ctx.func.alloca_entry_array(I64, total_caps); + { + let blk = ctx.block(); + for i in 0..total_caps { + let slot = blk.gep(I64, &buf, &[(I64, &format!("{}", i))]); + match captured_value_bits.get(i) { + Some(v_bits) => blk.store(I64, v_bits, &slot), + None => blk.store(I64, crate::nanbox::TAG_UNDEFINED_I64, &slot), + } + } + } + let blk = ctx.block(); + blk.call( + I64, + "js_closure_alloc_init", + &[(PTR, &func_ref), (I32, &cap_count), (PTR, &buf)], + ) } else { let blk = ctx.block(); blk.call( @@ -410,6 +446,10 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { let blk = ctx.block(); for (idx, val_bits) in captured_value_bits.iter().enumerate() { let track_box_capture = boxed_capture_slots[idx] && !is_plain_async_step; + if bulk_fresh_init { + // Every slot was written by `js_closure_alloc_init`. + continue; + } if !captured_singleton || track_box_capture { let idx_str = idx.to_string(); let setter = if track_box_capture { diff --git a/crates/perry-codegen/src/runtime_decls/strings.rs b/crates/perry-codegen/src/runtime_decls/strings.rs index d91c0fbae0..c3ef46e745 100644 --- a/crates/perry-codegen/src/runtime_decls/strings.rs +++ b/crates/perry-codegen/src/runtime_decls/strings.rs @@ -195,6 +195,7 @@ pub fn declare_phase_b_strings(module: &mut LlModule) { // (see crates/perry-runtime/src/closure.rs); the call site cap in // lower_call.rs matches. module.declare_function("js_closure_alloc", I64, &[PTR, I32]); + module.declare_function("js_closure_alloc_init", I64, &[PTR, I32, PTR]); // Singleton-cached variant for non-capturing closures and FuncRef // wrappers — same `func_ptr` returns the same cached ClosureHeader, // skipping per-evaluation closure allocation on the hot loop. See diff --git a/crates/perry-runtime/src/closure/alloc.rs b/crates/perry-runtime/src/closure/alloc.rs index ca21e81741..102e212b95 100644 --- a/crates/perry-runtime/src/closure/alloc.rs +++ b/crates/perry-runtime/src/closure/alloc.rs @@ -266,6 +266,98 @@ pub fn closure_alloc_storage(actual_count: usize) -> *mut u8 { } } +/// [`closure_alloc_storage`] with the no-collect contract: `Some` came out of +/// the already-open nursery block (nothing moved, no trigger check); `None` +/// means the caller must take the collecting path. +#[inline(always)] +fn closure_alloc_storage_no_collect(actual_count: usize) -> Option<*mut u8> { + let payload = closure_payload_size(actual_count); + if crate::gc::GC_HEADER_SIZE + payload > crate::gc::LARGE_OBJECT_THRESHOLD_BYTES { + return None; + } + let raw = crate::arena::arena_alloc_gc_no_collect( + payload, + std::mem::align_of::(), + crate::gc::GC_TYPE_CLOSURE, + ); + (!raw.is_null()).then_some(raw) +} + +/// One-call birth of a fresh (non-singleton) capturing closure: allocation, +/// header, capture slots and layout in a single runtime entry. +/// +/// Replaces `js_closure_alloc` + one `js_closure_set_capture_bits` per +/// capture, where every setter re-resolved the header, re-checked forwarding, +/// re-dispatched on the object kind for the layout note and paid the write +/// barrier's page-table classification again. Here the slots are copied in +/// bulk from `captures_ptr`, the layout is classified once from the finished +/// slots, and the barrier resolves the parent once for all slots. The +/// pointer-free sentinel fill `js_closure_alloc` performs is unnecessary: +/// every slot is written before the object is reachable from anywhere. +/// +/// `captures_ptr` slots are plain capture bits; box-cell captures (which need +/// `set_closure_box_capture` bookkeeping) keep the per-slot setter path in +/// codegen and never reach this entry. +#[no_mangle] +pub extern "C" fn js_closure_alloc_init( + func_ptr: *const u8, + capture_count: u32, + captures_ptr: *const u64, +) -> *mut ClosureHeader { + crate::promise::bump(&CLOSURE_ALLOC_COUNT); + let actual_count = real_capture_count(capture_count) as usize; + if actual_count == 0 || captures_ptr.is_null() { + return js_closure_alloc(func_ptr, capture_count); + } + // The no-collect arm keeps `captures_ptr`'s VALUES valid raw: nothing on + // the heap moved. The collecting fallback may have moved what those bits + // point at, so it re-reads them through roots — exactly the original + // per-setter path's contract, kept by taking that path. + let raw = match closure_alloc_storage_no_collect(actual_count) { + Some(raw) => raw, + None => { + let closure = js_closure_alloc(func_ptr, capture_count); + for i in 0..actual_count { + js_closure_set_capture_bits(closure, i as u32, unsafe { *captures_ptr.add(i) }); + } + return closure; + } + }; + let ptr = raw as *mut ClosureHeader; + unsafe { + (*ptr).func_ptr = func_ptr; + (*ptr).capture_count = capture_count; + (*ptr).type_tag = CLOSURE_MAGIC; + let slots = closure_capture_slots_mut(ptr); + // A handful of captures is the common case; a counted store loop + // beats the `memcpy` PLT call the runtime-length copy compiles to + // (perf: 2.6% of a one-capture birth was that call). + if actual_count <= 8 { + for i in 0..actual_count { + // GC_STORE_AUDIT(BARRIERED): copied captures are followed by + // the closure layout/barrier rebuild below (`any_pointer`), + // exactly as the `copy_nonoverlapping` arm beneath this one. + std::ptr::write(slots.add(i), *captures_ptr.add(i)); + } + } else { + std::ptr::copy_nonoverlapping(captures_ptr, slots, actual_count); + } + let any_pointer = + crate::gc::layout_init_from_slots(ptr as *mut u8, slots as *const u64, actual_count); + // Pointer-free births (numbers, booleans, SSO strings) have nothing + // for a barrier to remember or shade; the classification above + // already proved it. + if any_pointer { + crate::gc::runtime_write_barrier_newborn_slots( + ptr as usize, + slots as *const u64, + actual_count, + ); + } + } + ptr +} + #[inline] pub unsafe fn closure_capture_slots_mut(closure: *mut ClosureHeader) -> *mut u64 { (closure as *mut u8).add(std::mem::size_of::()) as *mut u64 diff --git a/crates/perry-runtime/src/gc/barrier_store.rs b/crates/perry-runtime/src/gc/barrier_store.rs index f39ddc4bf6..c77a3eb4bd 100644 --- a/crates/perry-runtime/src/gc/barrier_store.rs +++ b/crates/perry-runtime/src/gc/barrier_store.rs @@ -155,6 +155,36 @@ pub(crate) fn runtime_write_barrier_gc_slot(parent_addr: usize, slot_addr: usize write_barrier_slot_decoded(parent_addr, slot_addr, child_bits, parent_is_malloc_gc); } +/// Barrier for `slot_count` capture/field slots of a NEWBORN parent that were +/// just bulk-initialized. Same observable contract as calling +/// [`runtime_write_barrier_gc_slot`] once per slot, but the parent's +/// generation/malloc classification — a page-table lookup — is resolved once +/// for the whole run instead of per slot, and when barriers are off the loop +/// reduces to the incremental-mark shade check per value. +/// +/// # Safety +/// `slots` must point at `slot_count` initialized u64 slots inside `parent_addr`. +pub(crate) unsafe fn runtime_write_barrier_newborn_slots( + parent_addr: usize, + slots: *const u64, + slot_count: usize, +) { + if !write_barriers_enabled() { + for i in 0..slot_count { + incremental_mark_barrier_value(*slots.add(i)); + } + return; + } + let parent_is_malloc_gc = matches!( + crate::arena::classify_heap_generation(parent_addr), + crate::arena::HeapGeneration::Unknown + ) && malloc_gc_parent_addr(parent_addr); + for i in 0..slot_count { + let slot = slots.add(i); + write_barrier_slot_decoded(parent_addr, slot as usize, *slot, parent_is_malloc_gc); + } +} + // --- slot-form barrier entry points (moved from `barrier/mod.rs`, #2000-line cap) --- /// Gen-GC Phase C1: slot-aware write barrier. Called by diff --git a/crates/perry-runtime/src/gc/layout.rs b/crates/perry-runtime/src/gc/layout.rs index 12b8ae396c..b32956d7c8 100644 --- a/crates/perry-runtime/src/gc/layout.rs +++ b/crates/perry-runtime/src/gc/layout.rs @@ -1165,6 +1165,81 @@ pub(super) unsafe fn layout_rebuild_from_slots_with_policy( } } +/// Layout for a NEWBORN whose slots were just bulk-initialized: the same +/// classification as [`layout_rebuild_from_slots`] (pointer-free / unknown / +/// side mask), but with one `layout_forget_object` up front — a recycled +/// address may carry stale entries — instead of per-table removes interleaved +/// with the rebuild, and no per-slot `layout_note_slot` round trips (each of +/// which re-resolved the header, re-checked forwarding and re-dispatched on the +/// object kind). Callers must treat the object as fully initialized after +/// this returns. +/// +/// Returns `true` when at least one slot holds a pointer-bearing value, so +/// the caller can skip the write barrier entirely for a pointer-free birth +/// (the barrier's own child check would reject every slot anyway, after a +/// call and a page classification per slot). +pub(crate) unsafe fn layout_init_from_slots( + user_ptr: *mut u8, + slots: *const u64, + slot_count: usize, +) -> bool { + let Some(header) = layout_header_for_user(user_ptr as usize) else { + return true; + }; + if super::layout_tables::per_object_layouts_maybe_nonempty() { + layout_forget_object(user_ptr as usize); + } + header_clear_typed_layout_intact(header); + if slots.is_null() || slot_count == 0 { + set_layout_state(header, GC_LAYOUT_POINTER_FREE); + return false; + } + // Small births (the common case: a handful of captures) classify with a + // register-resident mask and no heap `Vec`; the min-slots threshold is + // read once here, not per slot. + let mut any_pointer = false; + if slot_count <= 64 { + let mut bits: u64 = 0; + for i in 0..slot_count { + if layout_pointer_bearing_bits(*slots.add(i)) { + bits |= 1u64 << i; + } + } + if bits == 0 { + set_layout_state(header, GC_LAYOUT_POINTER_FREE); + return false; + } + any_pointer = true; + if super::layout_tables::immortal_layout_scope_active() + || slot_count < super::layout_tables::layout_mask_min_slots() + { + set_layout_state(header, GC_LAYOUT_UNKNOWN); + } else { + set_layout_state(header, GC_LAYOUT_SIDE_MASK); + slot_masks_insert(user_ptr as usize, LayoutSlotMask::Inline(bits)); + } + return any_pointer; + } + let mut mask = LayoutSlotMask::Heap(vec![0; slot_count.div_ceil(64)]); + for i in 0..slot_count { + if layout_pointer_bearing_bits(*slots.add(i)) { + mask.set_slot(i); + any_pointer = true; + } + } + if !any_pointer { + set_layout_state(header, GC_LAYOUT_POINTER_FREE); + } else if super::layout_tables::immortal_layout_scope_active() + || slot_count < super::layout_tables::layout_mask_min_slots() + { + set_layout_state(header, GC_LAYOUT_UNKNOWN); + } else { + set_layout_state(header, GC_LAYOUT_SIDE_MASK); + slot_masks_insert(user_ptr as usize, mask); + } + any_pointer +} + pub(crate) unsafe fn layout_rebuild_from_slots( user_ptr: *mut u8, slots: *const u64, diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index eb6cd93a83..449e658ba7 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -1784,6 +1784,24 @@ "name": "WINDOW_ROOTS", "verdict": "not_a_gc_pointer", "why": "Window-root registry maps numeric window handles to numeric root-widget handles; neither value is a JavaScript heap pointer." + }, + { + "file": "crates/perry-runtime/src/set.rs", + "name": "SET_FOREACH_STACK", + "count": 1, + "classification": "not_a_gc_pointer", + "verdict": "not_a_gc_pointer", + "why": "Holds Set/Map header ADDRESSES as identity keys for in-flight forEach walks (#9082); the values are only ever compared (`contains`, `pop` equality) and truncated, never dereferenced, so no collector edge originates here. Correctness across evacuation is maintained by set_header_moved_for_gc / map_header_moved_for_gc, which rewrite every matching entry when a header moves \u2014 see the SET_FOREACH_STACK/MAP_FOREACH_STACK rewrite loops there.", + "reason": "Holds Set/Map header ADDRESSES as identity keys for in-flight forEach walks (#9082); the values are only ever compared (`contains`, `pop` equality) and truncated, never dereferenced, so no collector edge originates here. Correctness across evacuation is maintained by set_header_moved_for_gc / map_header_moved_for_gc, which rewrite every matching entry when a header moves \u2014 see the SET_FOREACH_STACK/MAP_FOREACH_STACK rewrite loops there." + }, + { + "file": "crates/perry-runtime/src/map.rs", + "name": "MAP_FOREACH_STACK", + "count": 1, + "classification": "not_a_gc_pointer", + "verdict": "not_a_gc_pointer", + "why": "Holds Set/Map header ADDRESSES as identity keys for in-flight forEach walks (#9082); the values are only ever compared (`contains`, `pop` equality) and truncated, never dereferenced, so no collector edge originates here. Correctness across evacuation is maintained by set_header_moved_for_gc / map_header_moved_for_gc, which rewrite every matching entry when a header moves \u2014 see the SET_FOREACH_STACK/MAP_FOREACH_STACK rewrite loops there.", + "reason": "Holds Set/Map header ADDRESSES as identity keys for in-flight forEach walks (#9082); the values are only ever compared (`contains`, `pop` equality) and truncated, never dereferenced, so no collector edge originates here. Correctness across evacuation is maintained by set_header_moved_for_gc / map_header_moved_for_gc, which rewrite every matching entry when a header moves \u2014 see the SET_FOREACH_STACK/MAP_FOREACH_STACK rewrite loops there." } ], "_FRONTIER_README": "Identity-pinned debt ratchet over new perry-ui* candidates and otherwise-unclassified core perry_thread_local! declarations (see the census docstring, \u201cThe identity-pinned frontier\u201d). A new uncovered holder fails until it is scanned, receives a researched holders verdict, or is deliberately pinned as debt. Moving a researched false positive to holders graduates it from this list. A fixed or classified holder makes its old frontier pin stale, so the receipt must be deleted.",