diff --git a/changelog.d/9091-unboxed-clone-accumulators.md b/changelog.d/9091-unboxed-clone-accumulators.md new file mode 100644 index 0000000000..f3a511c4fd --- /dev/null +++ b/changelog.d/9091-unboxed-clone-accumulators.md @@ -0,0 +1 @@ +Reduce accumulators in packed fast clones now live in register-promotable unboxed F64 slots for the clone's duration (write-back at every exit) — `s += a[i]` reduce loops reach node parity or better (literal-bound 0.98 ns/element vs Node's 1.01). diff --git a/crates/perry-codegen/src/codegen/closure.rs b/crates/perry-codegen/src/codegen/closure.rs index c06a359850..d5a143be12 100644 --- a/crates/perry-codegen/src/codegen/closure.rs +++ b/crates/perry-codegen/src/codegen/closure.rs @@ -1188,6 +1188,7 @@ pub(super) fn compile_closure( class_field_loop_facts: Vec::new(), element_shape_loop_facts: Vec::new(), i32_counter_slots: HashMap::new(), + numeric_accumulator_f64_slots: HashMap::new(), local_slot_reps: HashMap::new(), repsel_context_allows_canonical_i32: repsel_allows, // #7109 split the FIELD out of `repsel_context_allows_canonical_i32`; diff --git a/crates/perry-codegen/src/codegen/entry.rs b/crates/perry-codegen/src/codegen/entry.rs index 3ca86ad220..bf38be77b3 100644 --- a/crates/perry-codegen/src/codegen/entry.rs +++ b/crates/perry-codegen/src/codegen/entry.rs @@ -879,6 +879,7 @@ pub(super) fn compile_module_entry( class_field_loop_facts: Vec::new(), element_shape_loop_facts: Vec::new(), i32_counter_slots: HashMap::new(), + numeric_accumulator_f64_slots: HashMap::new(), local_slot_reps: HashMap::new(), // #7109: this entry body selects canonical i32/u32/Str on the same // per-value rules as a function body. Phase 1 (#6903) excluded it @@ -1595,6 +1596,7 @@ pub(super) fn compile_module_entry( class_field_loop_facts: Vec::new(), element_shape_loop_facts: Vec::new(), i32_counter_slots: HashMap::new(), + numeric_accumulator_f64_slots: HashMap::new(), local_slot_reps: HashMap::new(), // #7109: this entry body selects canonical i32/u32/Str on the same // per-value rules as a function body. Phase 1 (#6903) excluded it diff --git a/crates/perry-codegen/src/codegen/function.rs b/crates/perry-codegen/src/codegen/function.rs index af581de982..d114dcce0e 100644 --- a/crates/perry-codegen/src/codegen/function.rs +++ b/crates/perry-codegen/src/codegen/function.rs @@ -1147,6 +1147,7 @@ pub(super) fn compile_function( .map(|id| (*id, crate::expr::SlotRep::I32)) .collect(), i32_counter_slots: spec_i32_param_slots, + numeric_accumulator_f64_slots: HashMap::new(), repsel_context_allows_canonical_i32: repsel_allows, // #7109 split the FIELD out of `repsel_context_allows_canonical_i32`; // #7128 split the VALUE, which is what the knob actually reads. Until diff --git a/crates/perry-codegen/src/codegen/method.rs b/crates/perry-codegen/src/codegen/method.rs index 720a75712c..927a0cd68b 100644 --- a/crates/perry-codegen/src/codegen/method.rs +++ b/crates/perry-codegen/src/codegen/method.rs @@ -549,6 +549,7 @@ pub(super) fn compile_method( class_field_loop_facts: Vec::new(), element_shape_loop_facts: Vec::new(), i32_counter_slots: index_i32_param_slots, + numeric_accumulator_f64_slots: HashMap::new(), local_slot_reps: HashMap::new(), repsel_context_allows_canonical_i32: repsel_allows, // #7109 split the FIELD out of `repsel_context_allows_canonical_i32`; @@ -1714,6 +1715,7 @@ pub(super) fn compile_static_method( class_field_loop_facts: Vec::new(), element_shape_loop_facts: Vec::new(), i32_counter_slots: HashMap::new(), + numeric_accumulator_f64_slots: HashMap::new(), local_slot_reps: HashMap::new(), repsel_context_allows_canonical_i32: repsel_allows, // #7109 split the FIELD out of `repsel_context_allows_canonical_i32`; diff --git a/crates/perry-codegen/src/expr/literals_vars.rs b/crates/perry-codegen/src/expr/literals_vars.rs index 56ad7bafe1..14013aab5a 100644 --- a/crates/perry-codegen/src/expr/literals_vars.rs +++ b/crates/perry-codegen/src/expr/literals_vars.rs @@ -537,6 +537,12 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // double round-trip. The double slot is still maintained (for // closures or escape sites) but mem2reg + DSE will eliminate // it when the i32 path covers every read. + // Unboxed accumulator redirect (packed fast clones): the + // live value sits in a plain F64 alloca; its bits are the + // nanbox, so the load IS the value. + if let Some(f64_slot) = ctx.numeric_accumulator_f64_slots.get(id).cloned() { + return Ok(ctx.block().load(DOUBLE, &f64_slot)); + } if let Some(i32_slot) = ctx.i32_counter_slots.get(id).cloned() { let i = ctx.block().load(I32, &i32_slot); let v = if ctx.unsigned_i32_locals.contains(id) { @@ -579,6 +585,17 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { Expr::LocalSet(id, value) => { super::invalidate_local_write_facts(ctx, *id); super::record_local_value_alias_for_write(ctx, *id, value.as_ref()); + // Unboxed accumulator redirect (packed fast clones): the matcher + // proved every in-clone write numeric-preserving, so the store is + // a bare F64 alloca store — no shadow bookkeeping (the real slot + // holds a stale NUMBER for the clone's duration, consistent with + // whatever shadow state preceded the loop), no barrier (numbers + // carry no heap edge). Exits write the value back. + if let Some(f64_slot) = ctx.numeric_accumulator_f64_slots.get(id).cloned() { + let v = lower_expr(ctx, value)?; + ctx.block().store(DOUBLE, &v, &f64_slot); + return Ok(v); + } if let Some(v) = lower_pod_local_reassignment(ctx, *id, value)? { super::record_native_arena_owner_assignment(ctx, *id, value.as_ref()); return Ok(v); diff --git a/crates/perry-codegen/src/expr/mod.rs b/crates/perry-codegen/src/expr/mod.rs index d3652d83f0..bbcf1d48c1 100644 --- a/crates/perry-codegen/src/expr/mod.rs +++ b/crates/perry-codegen/src/expr/mod.rs @@ -1046,6 +1046,17 @@ pub(crate) struct FnCtx<'a> { /// on hot array-walking loops like `for (let i = 0; i < arr.length; /// i++) arr[i] = expr`. pub i32_counter_slots: std::collections::HashMap, + /// Unboxed reduce-accumulator redirect, active only while a packed fast + /// clone is being lowered: local id -> plain (addrspace-0) F64 alloca. + /// The clone's preheader tag-tested the local as a Number and moved its + /// value here; every in-clone read/write of the local goes through this + /// alloca (mem2reg promotes it to a register — the GC-root slot's + /// store-to-load-forward chain was the reduce rows' latency floor), and + /// every clone exit (fall-through and side-exit trampoline) writes the + /// value back to the real slot. A genuine double's bits ARE its nanbox, + /// so no conversion exists on either edge; the stale number left in the + /// root slot during the clone is harmless to a GC scan. + pub numeric_accumulator_f64_slots: std::collections::HashMap, /// Representation-selection Phase 1 (RFC `docs/representation-selection- /// rfc.md`): LocalId → selected slot representation. Absent = `Boxed` diff --git a/crates/perry-codegen/src/stmt/loops.rs b/crates/perry-codegen/src/stmt/loops.rs index fd9ed7dfb2..307261e756 100644 --- a/crates/perry-codegen/src/stmt/loops.rs +++ b/crates/perry-codegen/src/stmt/loops.rs @@ -534,17 +534,17 @@ fn emit_range_loop_accumulator_admission( body: &[Stmt], slow_pre_label: &str, block_prefix: &str, -) -> Vec { +) -> PackedAccumulatorScope { let mut counter_arrays = matched .arrays .iter() .filter(|access| access.counter.is_some()) .map(|access| access.array_id); let Some(array_id) = counter_arrays.next() else { - return Vec::new(); + return PackedAccumulatorScope::empty(); }; if counter_arrays.next().is_some() { - return Vec::new(); + return PackedAccumulatorScope::empty(); } emit_packed_numeric_accumulator_admission( ctx, @@ -556,6 +556,56 @@ fn emit_range_loop_accumulator_admission( ) } +/// The live state of a packed clone's accumulator admission: the admitted +/// ids (they ride the scope's fact so `is_numeric_expr` sees them), the +/// unboxed subset (id, F64 alloca, real slot) whose reads/writes redirect +/// through `ctx.numeric_accumulator_f64_slots`, and the side-exit trampoline +/// that writes the live values back before entering the slow clone. +struct PackedAccumulatorScope { + accumulators: Vec, + unboxed: Vec<(u32, String, String)>, + side_exit_override: Option, +} + +impl PackedAccumulatorScope { + fn empty() -> Self { + Self { + accumulators: Vec::new(), + unboxed: Vec::new(), + side_exit_override: None, + } + } + + /// The label packed facts should carry as their side exit: the + /// write-back trampoline when unboxed accumulators exist, else the slow + /// preheader itself. + fn fact_side_exit(&self, slow_pre_label: &str) -> String { + self.side_exit_override + .clone() + .unwrap_or_else(|| slow_pre_label.to_string()) + } + + /// Fall-through exit: write the live values back to the real slots and + /// end the redirect scope. Must run right after the fast clone's + /// lowering, BEFORE the slow clone is lowered (the slow clone reads and + /// writes the real slots). + fn finish(self, ctx: &mut FnCtx<'_>) { + let emit_writeback = !ctx.block().is_terminated(); + for (id, alloca, real_slot) in &self.unboxed { + if emit_writeback { + let value = ctx.block().load(DOUBLE, alloca); + // A genuine double's bits are its nanbox; numbers carry no + // heap edge, so no barrier. Leaving the shadow state + // conservative is always safe — shadow slots only license + // SKIPPING a root scan, and scanning a number nanbox is + // harmless. + ctx.block().store(DOUBLE, &value, real_slot); + } + ctx.numeric_accumulator_f64_slots.remove(id); + } + } +} + fn emit_packed_numeric_accumulator_admission( ctx: &mut FnCtx<'_>, body: &[Stmt], @@ -563,24 +613,35 @@ fn emit_packed_numeric_accumulator_admission( counter_id: u32, slow_pre_label: &str, block_prefix: &str, -) -> Vec { +) -> PackedAccumulatorScope { let accumulators = super::stable_packed_accumulator::collect_numeric_accumulators( ctx, body, array_id, counter_id, ); if accumulators.is_empty() { - return accumulators; + return PackedAccumulatorScope::empty(); } + let mut loaded: Vec<(u32, String, String)> = Vec::new(); let mut all_numbers: Option = None; for id in &accumulators { let Some(slot) = ctx.locals.get(id).cloned() else { - return Vec::new(); + return PackedAccumulatorScope::empty(); }; let value = ctx.block().load(DOUBLE, &slot); + // `emit_js_value_is_number` IS the strict genuine-double window: + // SHORT_STRING (0x7FF9) .. STRING (0x7FFF) is the ENTIRE boxed tag + // band (INT32/POINTER/BIGINT/singletons included), so + // `tag < SHORT_STRING || tag > STRING` accepts exactly non-boxed + // doubles. That strictness is required here — the fact these ids + // ride lets consumers use bare fadd/fcmp on the value, and an + // INT32-boxed number's bits are not a valid double; it takes the + // slow clone instead. Shared with the stable clone's admission so + // the two cannot drift. let is_number = emit_js_value_is_number(ctx, &value); all_numbers = Some(match all_numbers { Some(prev) => ctx.block().and(I1, &prev, &is_number), None => is_number, }); + loaded.push((*id, slot, value)); } let all_numbers = all_numbers.expect("at least one accumulator"); let acc_ok_idx = ctx.new_block(&format!("{block_prefix}.acc.ok")); @@ -591,7 +652,51 @@ fn emit_packed_numeric_accumulator_admission( ctx.block() .cond_br(&all_numbers, &acc_ok_label, slow_pre_label); ctx.current_block = acc_ok_idx; - accumulators + + // Unbox LocalSet-only accumulators into plain F64 allocas (mem2reg + // promotes them to registers — the GC-root slot's per-iteration + // store-to-load-forward chain was the reduce rows' latency floor). + // Update-written accumulators (`c++`) keep the slot: the Update lowering + // does not consult the redirect, and the int-slot machinery already + // serves counters well. + let mut writes = std::collections::BTreeMap::new(); + super::stable_packed_accumulator::collect_local_writes(body, &mut writes); + let mut unboxed: Vec<(u32, String, String)> = Vec::new(); + for (id, slot, value) in loaded { + let localset_only = writes + .get(&id) + .is_some_and(|ws| ws.iter().all(|w| w.is_some())); + if !localset_only { + continue; + } + let alloca = ctx.func.alloca_entry(DOUBLE); + ctx.block().store(DOUBLE, &value, &alloca); + ctx.numeric_accumulator_f64_slots.insert(id, alloca.clone()); + unboxed.push((id, alloca, slot)); + } + let side_exit_override = if unboxed.is_empty() { + None + } else { + // Side-exit trampoline: any mid-iteration exit (a hole-checked load, + // a masked store's value check) lands here, writes the live values + // back, and only then enters the slow clone — which re-executes the + // current iteration against correct slot state. + let tramp_idx = ctx.new_block(&format!("{block_prefix}.acc.writeback_exit")); + let saved = ctx.current_block; + ctx.current_block = tramp_idx; + for (_, alloca, real_slot) in &unboxed { + let value = ctx.block().load(DOUBLE, alloca); + ctx.block().store(DOUBLE, &value, real_slot); + } + ctx.block().br(slow_pre_label); + ctx.current_block = saved; + Some(ctx.block_label(tramp_idx)) + }; + PackedAccumulatorScope { + accumulators, + unboxed, + side_exit_override, + } } fn lower_packed_f64_versioned_for( @@ -662,7 +767,7 @@ fn lower_packed_f64_versioned_for( let packed_scope_id = ctx.next_loop_proof_scope_id(); ctx.current_block = fast_pre_idx; - let numeric_accumulators = emit_packed_numeric_accumulator_admission( + let acc_scope = emit_packed_numeric_accumulator_admission( ctx, body, matched.array_id, @@ -675,11 +780,11 @@ fn lower_packed_f64_versioned_for( array_local_id: matched.array_id, scope_id: packed_scope_id, guard_id: guard_id.to_string(), - store_side_exit_label: slow_pre_label.clone(), + store_side_exit_label: acc_scope.fact_side_exit(&slow_pre_label), array_kind: matched.array_kind, allow_holes: false, window_validated: false, - numeric_accumulators, + numeric_accumulators: acc_scope.accumulators.clone(), }); // The guard just proved a live, non-forwarded plain array, and the // matched body cannot change its length (in-bounds stores only, no @@ -708,6 +813,7 @@ fn lower_packed_f64_versioned_for( )?; ctx.packed_f64_loop_facts .retain(|fact| fact.scope_id != packed_scope_id); + acc_scope.finish(ctx); if !ctx.block().is_terminated() { ctx.block().br(&merge_label); } @@ -2119,22 +2225,23 @@ fn lower_packed_f64_range_versioned_for( ctx.current_block = fast_i32_pre_idx; let scope_i32 = ctx.next_loop_proof_scope_id(); - let range_numeric_accumulators = emit_range_loop_accumulator_admission( + let acc_scope = emit_range_loop_accumulator_admission( ctx, &matched, body, &slow_pre_label, "packed_f64_range.fast_i32", ); + let fact_side_exit = acc_scope.fact_side_exit(&slow_pre_label); push_packed_f64_range_facts( ctx, &matched, scope_i32, "packed_f64_range_loop_guard_dense_i32", - &slow_pre_label, + &fact_side_exit, true, false, - &range_numeric_accumulators, + &acc_scope.accumulators, ); lower_for_after_init_with_i32_bound( ctx, @@ -2149,6 +2256,7 @@ fn lower_packed_f64_range_versioned_for( .retain(|fact| fact.scope_id != scope_i32); ctx.masked_window_array_facts .retain(|fact| fact.scope_id != scope_i32); + acc_scope.finish(ctx); if !ctx.block().is_terminated() { ctx.block().br(&merge_label); } @@ -2156,22 +2264,23 @@ fn lower_packed_f64_range_versioned_for( ctx.current_block = fast_pre_idx; let scope_f64 = ctx.next_loop_proof_scope_id(); - let range_numeric_accumulators = emit_range_loop_accumulator_admission( + let acc_scope = emit_range_loop_accumulator_admission( ctx, &matched, body, &slow_pre_label, "packed_f64_range.fast", ); + let fact_side_exit = acc_scope.fact_side_exit(&slow_pre_label); push_packed_f64_range_facts( ctx, &matched, scope_f64, "packed_f64_range_loop_guard_dense", - &slow_pre_label, + &fact_side_exit, false, has_stores, - &range_numeric_accumulators, + &acc_scope.accumulators, ); lower_for_after_init_with_i32_bound( ctx, @@ -2186,6 +2295,7 @@ fn lower_packed_f64_range_versioned_for( .retain(|fact| fact.scope_id != scope_f64); ctx.masked_window_array_facts .retain(|fact| fact.scope_id != scope_f64); + acc_scope.finish(ctx); if !ctx.block().is_terminated() { ctx.block().br(&merge_label); } @@ -2203,22 +2313,23 @@ fn lower_packed_f64_range_versioned_for( let packed_scope_id = ctx.next_loop_proof_scope_id(); ctx.current_block = fast_pre_idx; - let range_numeric_accumulators = emit_range_loop_accumulator_admission( + let acc_scope = emit_range_loop_accumulator_admission( ctx, &matched, body, &slow_pre_label, "packed_f64_range.classic", ); + let fact_side_exit = acc_scope.fact_side_exit(&slow_pre_label); push_packed_f64_range_facts( ctx, &matched, packed_scope_id, "packed_f64_range_loop_guard", - &slow_pre_label, + &fact_side_exit, false, false, - &range_numeric_accumulators, + &acc_scope.accumulators, ); lower_for_after_init_with_i32_bound( ctx, @@ -2233,6 +2344,7 @@ fn lower_packed_f64_range_versioned_for( .retain(|fact| fact.scope_id != packed_scope_id); ctx.masked_window_array_facts .retain(|fact| fact.scope_id != packed_scope_id); + acc_scope.finish(ctx); if !ctx.block().is_terminated() { ctx.block().br(&merge_label); } diff --git a/crates/perry-codegen/src/stmt/stable_packed_accumulator.rs b/crates/perry-codegen/src/stmt/stable_packed_accumulator.rs index c3fe8c14a7..398de2b935 100644 --- a/crates/perry-codegen/src/stmt/stable_packed_accumulator.rs +++ b/crates/perry-codegen/src/stmt/stable_packed_accumulator.rs @@ -86,7 +86,7 @@ fn accumulator_rhs_is_numeric( /// Collect every write (`LocalSet` / `Update`) per local in `body`, without /// descending into nested closures (their writes go through boxes, and a /// boxed local is excluded from admission anyway). -fn collect_local_writes<'a>( +pub(super) fn collect_local_writes<'a>( stmts: &'a [Stmt], out: &mut std::collections::BTreeMap>>, ) {