perf(runtime,codegen): one-call birth for fresh capturing closures (−15%)

crates/perry-codegen/src/expr/closure.rs

@@ -340,6 +340,16 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result<String> {
                 None
             };
 
+            // Bulk-init admission: a fresh closure whose captures are all plain
+            // bits. Box-cell captures keep the per-slot setter path — their
+            // `set_closure_box_capture` bookkeeping has no bulk twin.
+            let bulk_fresh_init = !no_capture_singleton
+                && !captured_singleton
+                && total_caps > 0
+                && !captured_value_bits.is_empty()
+                && auto_captures
+                    .iter()
+                    .all(|cap_id| is_plain_async_step || !ctx.boxed_vars.contains(cap_id));
             let closure_handle = if no_capture_singleton {
                 let blk = ctx.block();
                 blk.call(I64, "js_closure_alloc_singleton", &[(PTR, &func_ref)])
@@ -374,6 +384,32 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result<String> {
                     "js_closure_alloc_with_captures_singleton",
                     &[(PTR, &func_ref), (I32, &cap_count), (PTR, &buf)],
                 )
+            } else if bulk_fresh_init {
+                // Fresh (identity-carrying) closure with plain-bits captures:
+                // ONE runtime call does allocation + slots + layout instead of
+                // `js_closure_alloc` plus a `js_closure_set_capture_bits` per
+                // capture (each re-resolving the header, forwarding, kind
+                // dispatch and the barrier's page classification). The reserved
+                // `this` / `new.target` slots are pre-filled with the pointer-free
+                // sentinel the per-slot path relied on `js_closure_alloc` writing;
+                // the post-create patch below fills them exactly as before.
+                let buf = ctx.func.alloca_entry_array(I64, total_caps);
+                {
+                    let blk = ctx.block();
+                    for i in 0..total_caps {
+                        let slot = blk.gep(I64, &buf, &[(I64, &format!("{}", i))]);
+                        match captured_value_bits.get(i) {
+                            Some(v_bits) => blk.store(I64, v_bits, &slot),
+                            None => blk.store(I64, crate::nanbox::TAG_UNDEFINED_I64, &slot),
+                        }
+                    }
+                }
+                let blk = ctx.block();
+                blk.call(
+                    I64,
+                    "js_closure_alloc_init",
+                    &[(PTR, &func_ref), (I32, &cap_count), (PTR, &buf)],
+                )
             } else {
                 let blk = ctx.block();
                 blk.call(
@@ -410,6 +446,10 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result<String> {
             let blk = ctx.block();
             for (idx, val_bits) in captured_value_bits.iter().enumerate() {
                 let track_box_capture = boxed_capture_slots[idx] && !is_plain_async_step;
+                if bulk_fresh_init {
+                    // Every slot was written by `js_closure_alloc_init`.
+                    continue;
+                }
                 if !captured_singleton || track_box_capture {
                     let idx_str = idx.to_string();
                     let setter = if track_box_capture {

crates/perry-codegen/src/runtime_decls/strings.rs

@@ -195,6 +195,7 @@ pub fn declare_phase_b_strings(module: &mut LlModule) {
     //     (see crates/perry-runtime/src/closure.rs); the call site cap in
     //     lower_call.rs matches.
     module.declare_function("js_closure_alloc", I64, &[PTR, I32]);
+    module.declare_function("js_closure_alloc_init", I64, &[PTR, I32, PTR]);
     // Singleton-cached variant for non-capturing closures and FuncRef
     // wrappers — same `func_ptr` returns the same cached ClosureHeader,
     // skipping per-evaluation closure allocation on the hot loop. See

crates/perry-runtime/src/closure/alloc.rs

@@ -266,6 +266,98 @@ pub fn closure_alloc_storage(actual_count: usize) -> *mut u8 {
     }
 }
 
+/// [`closure_alloc_storage`] with the no-collect contract: `Some` came out of
+/// the already-open nursery block (nothing moved, no trigger check); `None`
+/// means the caller must take the collecting path.
+#[inline(always)]
+fn closure_alloc_storage_no_collect(actual_count: usize) -> Option<*mut u8> {
+    let payload = closure_payload_size(actual_count);
+    if crate::gc::GC_HEADER_SIZE + payload > crate::gc::LARGE_OBJECT_THRESHOLD_BYTES {
+        return None;
+    }
+    let raw = crate::arena::arena_alloc_gc_no_collect(
+        payload,
+        std::mem::align_of::<ClosureHeader>(),
+        crate::gc::GC_TYPE_CLOSURE,
+    );
+    (!raw.is_null()).then_some(raw)
+}
+
+/// One-call birth of a fresh (non-singleton) capturing closure: allocation,
+/// header, capture slots and layout in a single runtime entry.
+///
+/// Replaces `js_closure_alloc` + one `js_closure_set_capture_bits` per
+/// capture, where every setter re-resolved the header, re-checked forwarding,
+/// re-dispatched on the object kind for the layout note and paid the write
+/// barrier's page-table classification again. Here the slots are copied in
+/// bulk from `captures_ptr`, the layout is classified once from the finished
+/// slots, and the barrier resolves the parent once for all slots. The
+/// pointer-free sentinel fill `js_closure_alloc` performs is unnecessary:
+/// every slot is written before the object is reachable from anywhere.
+///
+/// `captures_ptr` slots are plain capture bits; box-cell captures (which need
+/// `set_closure_box_capture` bookkeeping) keep the per-slot setter path in
+/// codegen and never reach this entry.
+#[no_mangle]
+pub extern "C" fn js_closure_alloc_init(
+    func_ptr: *const u8,
+    capture_count: u32,
+    captures_ptr: *const u64,
+) -> *mut ClosureHeader {
+    crate::promise::bump(&CLOSURE_ALLOC_COUNT);
+    let actual_count = real_capture_count(capture_count) as usize;
+    if actual_count == 0 || captures_ptr.is_null() {
+        return js_closure_alloc(func_ptr, capture_count);
+    }
+    // The no-collect arm keeps `captures_ptr`'s VALUES valid raw: nothing on
+    // the heap moved. The collecting fallback may have moved what those bits
+    // point at, so it re-reads them through roots — exactly the original
+    // per-setter path's contract, kept by taking that path.
+    let raw = match closure_alloc_storage_no_collect(actual_count) {
+        Some(raw) => raw,
+        None => {
+            let closure = js_closure_alloc(func_ptr, capture_count);
+            for i in 0..actual_count {
+                js_closure_set_capture_bits(closure, i as u32, unsafe { *captures_ptr.add(i) });
+            }
+            return closure;
+        }
+    };
+    let ptr = raw as *mut ClosureHeader;
+    unsafe {
+        (*ptr).func_ptr = func_ptr;
+        (*ptr).capture_count = capture_count;
+        (*ptr).type_tag = CLOSURE_MAGIC;
+        let slots = closure_capture_slots_mut(ptr);
+        // A handful of captures is the common case; a counted store loop
+        // beats the `memcpy` PLT call the runtime-length copy compiles to
+        // (perf: 2.6% of a one-capture birth was that call).
+        if actual_count <= 8 {
+            for i in 0..actual_count {
+                // GC_STORE_AUDIT(BARRIERED): copied captures are followed by
+                // the closure layout/barrier rebuild below (`any_pointer`),
+                // exactly as the `copy_nonoverlapping` arm beneath this one.
+                std::ptr::write(slots.add(i), *captures_ptr.add(i));
+            }
+        } else {
+            std::ptr::copy_nonoverlapping(captures_ptr, slots, actual_count);
+        }
+        let any_pointer =
+            crate::gc::layout_init_from_slots(ptr as *mut u8, slots as *const u64, actual_count);
+        // Pointer-free births (numbers, booleans, SSO strings) have nothing
+        // for a barrier to remember or shade; the classification above
+        // already proved it.
+        if any_pointer {
+            crate::gc::runtime_write_barrier_newborn_slots(
+                ptr as usize,
+                slots as *const u64,
+                actual_count,
+            );
+        }
+    }
+    ptr
+}
+
 #[inline]
 pub unsafe fn closure_capture_slots_mut(closure: *mut ClosureHeader) -> *mut u64 {
     (closure as *mut u8).add(std::mem::size_of::<ClosureHeader>()) as *mut u64

crates/perry-runtime/src/gc/barrier_store.rs

@@ -155,6 +155,36 @@ pub(crate) fn runtime_write_barrier_gc_slot(parent_addr: usize, slot_addr: usize
     write_barrier_slot_decoded(parent_addr, slot_addr, child_bits, parent_is_malloc_gc);
 }
 
+/// Barrier for `slot_count` capture/field slots of a NEWBORN parent that were
+/// just bulk-initialized. Same observable contract as calling
+/// [`runtime_write_barrier_gc_slot`] once per slot, but the parent's
+/// generation/malloc classification — a page-table lookup — is resolved once
+/// for the whole run instead of per slot, and when barriers are off the loop
+/// reduces to the incremental-mark shade check per value.
+///
+/// # Safety
+/// `slots` must point at `slot_count` initialized u64 slots inside `parent_addr`.
+pub(crate) unsafe fn runtime_write_barrier_newborn_slots(
+    parent_addr: usize,
+    slots: *const u64,
+    slot_count: usize,
+) {
+    if !write_barriers_enabled() {
+        for i in 0..slot_count {
+            incremental_mark_barrier_value(*slots.add(i));
+        }
+        return;
+    }
+    let parent_is_malloc_gc = matches!(
+        crate::arena::classify_heap_generation(parent_addr),
+        crate::arena::HeapGeneration::Unknown
+    ) && malloc_gc_parent_addr(parent_addr);
+    for i in 0..slot_count {
+        let slot = slots.add(i);
+        write_barrier_slot_decoded(parent_addr, slot as usize, *slot, parent_is_malloc_gc);
+    }
+}
+
 // --- slot-form barrier entry points (moved from `barrier/mod.rs`, #2000-line cap) ---
 
 /// Gen-GC Phase C1: slot-aware write barrier. Called by

crates/perry-runtime/src/gc/layout.rs

@@ -1165,6 +1165,81 @@ pub(super) unsafe fn layout_rebuild_from_slots_with_policy(
     }
 }
 
+/// Layout for a NEWBORN whose slots were just bulk-initialized: the same
+/// classification as [`layout_rebuild_from_slots`] (pointer-free / unknown /
+/// side mask), but with one `layout_forget_object` up front — a recycled
+/// address may carry stale entries — instead of per-table removes interleaved
+/// with the rebuild, and no per-slot `layout_note_slot` round trips (each of
+/// which re-resolved the header, re-checked forwarding and re-dispatched on the
+/// object kind). Callers must treat the object as fully initialized after
+/// this returns.
+///
+/// Returns `true` when at least one slot holds a pointer-bearing value, so
+/// the caller can skip the write barrier entirely for a pointer-free birth
+/// (the barrier's own child check would reject every slot anyway, after a
+/// call and a page classification per slot).
+pub(crate) unsafe fn layout_init_from_slots(
+    user_ptr: *mut u8,
+    slots: *const u64,
+    slot_count: usize,
+) -> bool {
+    let Some(header) = layout_header_for_user(user_ptr as usize) else {
+        return true;
+    };
+    if super::layout_tables::per_object_layouts_maybe_nonempty() {
+        layout_forget_object(user_ptr as usize);
+    }
+    header_clear_typed_layout_intact(header);
+    if slots.is_null() || slot_count == 0 {
+        set_layout_state(header, GC_LAYOUT_POINTER_FREE);
+        return false;
+    }
+    // Small births (the common case: a handful of captures) classify with a
+    // register-resident mask and no heap `Vec`; the min-slots threshold is
+    // read once here, not per slot.
+    let mut any_pointer = false;
+    if slot_count <= 64 {
+        let mut bits: u64 = 0;
+        for i in 0..slot_count {
+            if layout_pointer_bearing_bits(*slots.add(i)) {
+                bits |= 1u64 << i;
+            }
+        }
+        if bits == 0 {
+            set_layout_state(header, GC_LAYOUT_POINTER_FREE);
+            return false;
+        }
+        any_pointer = true;
+        if super::layout_tables::immortal_layout_scope_active()
+            || slot_count < super::layout_tables::layout_mask_min_slots()
+        {
+            set_layout_state(header, GC_LAYOUT_UNKNOWN);
+        } else {
+            set_layout_state(header, GC_LAYOUT_SIDE_MASK);
+            slot_masks_insert(user_ptr as usize, LayoutSlotMask::Inline(bits));
+        }
+        return any_pointer;
+    }
+    let mut mask = LayoutSlotMask::Heap(vec![0; slot_count.div_ceil(64)]);
+    for i in 0..slot_count {
+        if layout_pointer_bearing_bits(*slots.add(i)) {
+            mask.set_slot(i);
+            any_pointer = true;
+        }
+    }
+    if !any_pointer {
+        set_layout_state(header, GC_LAYOUT_POINTER_FREE);
+    } else if super::layout_tables::immortal_layout_scope_active()
+        || slot_count < super::layout_tables::layout_mask_min_slots()
+    {
+        set_layout_state(header, GC_LAYOUT_UNKNOWN);
+    } else {
+        set_layout_state(header, GC_LAYOUT_SIDE_MASK);
+        slot_masks_insert(user_ptr as usize, mask);
+    }
+    any_pointer
+}
+
 pub(crate) unsafe fn layout_rebuild_from_slots(
     user_ptr: *mut u8,
     slots: *const u64,

scripts/gc_runtime_root_holders.json

@@ -1784,6 +1784,24 @@
       "name": "WINDOW_ROOTS",
       "verdict": "not_a_gc_pointer",
       "why": "Window-root registry maps numeric window handles to numeric root-widget handles; neither value is a JavaScript heap pointer."
+    },
+    {
+      "file": "crates/perry-runtime/src/set.rs",
+      "name": "SET_FOREACH_STACK",
+      "count": 1,
+      "classification": "not_a_gc_pointer",
+      "verdict": "not_a_gc_pointer",
+      "why": "Holds Set/Map header ADDRESSES as identity keys for in-flight forEach walks (#9082); the values are only ever compared (`contains`, `pop` equality) and truncated, never dereferenced, so no collector edge originates here. Correctness across evacuation is maintained by set_header_moved_for_gc / map_header_moved_for_gc, which rewrite every matching entry when a header moves \u2014 see the SET_FOREACH_STACK/MAP_FOREACH_STACK rewrite loops there.",
+      "reason": "Holds Set/Map header ADDRESSES as identity keys for in-flight forEach walks (#9082); the values are only ever compared (`contains`, `pop` equality) and truncated, never dereferenced, so no collector edge originates here. Correctness across evacuation is maintained by set_header_moved_for_gc / map_header_moved_for_gc, which rewrite every matching entry when a header moves \u2014 see the SET_FOREACH_STACK/MAP_FOREACH_STACK rewrite loops there."
+    },
+    {
+      "file": "crates/perry-runtime/src/map.rs",
+      "name": "MAP_FOREACH_STACK",
+      "count": 1,
+      "classification": "not_a_gc_pointer",
+      "verdict": "not_a_gc_pointer",
+      "why": "Holds Set/Map header ADDRESSES as identity keys for in-flight forEach walks (#9082); the values are only ever compared (`contains`, `pop` equality) and truncated, never dereferenced, so no collector edge originates here. Correctness across evacuation is maintained by set_header_moved_for_gc / map_header_moved_for_gc, which rewrite every matching entry when a header moves \u2014 see the SET_FOREACH_STACK/MAP_FOREACH_STACK rewrite loops there.",
+      "reason": "Holds Set/Map header ADDRESSES as identity keys for in-flight forEach walks (#9082); the values are only ever compared (`contains`, `pop` equality) and truncated, never dereferenced, so no collector edge originates here. Correctness across evacuation is maintained by set_header_moved_for_gc / map_header_moved_for_gc, which rewrite every matching entry when a header moves \u2014 see the SET_FOREACH_STACK/MAP_FOREACH_STACK rewrite loops there."
     }
   ],
   "_FRONTIER_README": "Identity-pinned debt ratchet over new perry-ui* candidates and otherwise-unclassified core perry_thread_local! declarations (see the census docstring, \u201cThe identity-pinned frontier\u201d). A new uncovered holder fails until it is scanned, receives a researched holders verdict, or is deliberately pinned as debt. Moving a researched false positive to holders graduates it from this list. A fixed or classified holder makes its old frontier pin stale, so the receipt must be deleted.",