diff --git a/changelog.d/9060-packed-loop-numeric-accumulator.md b/changelog.d/9060-packed-loop-numeric-accumulator.md new file mode 100644 index 0000000000..700ea8901e --- /dev/null +++ b/changelog.d/9060-packed-loop-numeric-accumulator.md @@ -0,0 +1,3 @@ +### Changed + +- Reduce loops — `for (let i = 0; i < arr.length; i++) s += arr[i]` — now run their fast versioned clone at full speed: the accumulator is tag-tested once in the preheader and proven a Number for the whole clone, so the per-element addition is a native `fadd` instead of a dynamic-`+` runtime call; and a proven-numeric accumulator's shadow-slot clear no longer disqualifies the clone from the call-free fast path. The isolated reduce loop went from 5.3× node to node parity. diff --git a/crates/perry-codegen/src/expr/mod.rs b/crates/perry-codegen/src/expr/mod.rs index 692f626b7c..d94b5ada4b 100644 --- a/crates/perry-codegen/src/expr/mod.rs +++ b/crates/perry-codegen/src/expr/mod.rs @@ -1727,6 +1727,15 @@ pub(crate) struct StablePackedReadCache { pub(crate) struct StablePackedLoopFact { pub counter_local_id: u32, pub array_local_id: u32, + /// Plain locals the fast preheader proved to hold a Number (one tag test + /// per admitted accumulator) and whose every write inside the loop body is + /// numeric-preserving with all leaves provable numeric in-loop, so the + /// value stays a Number by induction for the whole fast clone. + /// `is_numeric_expr` consults this for `LocalGet`, exactly like the + /// element-shape clone's `numeric_accumulator` — it is what lets + /// `s += arr[i]` lower to a native `fadd` instead of + /// `js_dynamic_string_or_number_add` on every iteration. + pub numeric_accumulators: Vec, pub side_exit_label: String, pub descriptor: String, /// Boxed bound passed to the runtime guard (`-1` requests live length). diff --git a/crates/perry-codegen/src/inst.rs b/crates/perry-codegen/src/inst.rs index 6988ac2906..c4859f17c8 100644 --- a/crates/perry-codegen/src/inst.rs +++ b/crates/perry-codegen/src/inst.rs @@ -410,9 +410,20 @@ impl LlInst { return false; }; let callee = &s[pos..]; - !callee.contains("@llvm.") && !callee.contains(" asm ") + !callee.contains("@llvm.") + && !callee.contains(" asm ") + // `js_shadow_slot_set` is a bounds-checked TLS store + // (`gc/roots/shadow_stack.rs`): it cannot allocate, + // collect, or revoke a layout, which is precisely what the + // two call-free clone scans exist to exclude. Without the + // exemption a proven-numeric accumulator's per-statement + // shadow CLEAR — itself emitted BECAUSE the value is a + // known non-pointer — condemned the whole fast clone. + && !callee.contains("@js_shadow_slot_set") + } + LlInst::Call { callee, .. } => { + !callee.starts_with("llvm.") && callee != "js_shadow_slot_set" } - LlInst::Call { callee, .. } => !callee.starts_with("llvm."), LlInst::CallIndirect { .. } => true, _ => false, } diff --git a/crates/perry-codegen/src/stmt/mod.rs b/crates/perry-codegen/src/stmt/mod.rs index 6a2f571648..7bca7fa9ac 100644 --- a/crates/perry-codegen/src/stmt/mod.rs +++ b/crates/perry-codegen/src/stmt/mod.rs @@ -29,6 +29,7 @@ mod loops; mod masked_window_region; #[cfg(test)] mod prealloc_module_global_tests; +pub(crate) mod stable_packed_accumulator; pub(crate) mod stable_packed_loop; mod stable_packed_typed_array; mod switch_stmt; diff --git a/crates/perry-codegen/src/stmt/stable_packed_accumulator.rs b/crates/perry-codegen/src/stmt/stable_packed_accumulator.rs new file mode 100644 index 0000000000..c3fe8c14a7 --- /dev/null +++ b/crates/perry-codegen/src/stmt/stable_packed_accumulator.rs @@ -0,0 +1,273 @@ +//! Reduce-accumulator admission for the stable-packed fast clone. +//! +//! Split out of `stable_packed_loop.rs` to keep it under the 2,000-line file +//! gate. The entry point is [`collect_numeric_accumulators`]; the fact it +//! feeds is consumed by `type_analysis::is_numeric_expr`'s `LocalGet` arm and +//! the tag tests the caller emits in the fast preheader. + +use perry_hir::{Expr, Stmt}; + +use crate::expr::FnCtx; + +/// `PERRY_PACKED_LOOP_NUMERIC_ACCUMULATOR` gate (default on): admit reduce +/// accumulators into the fast clone's numeric proof. `=0`/`off`/`false` keeps +/// the pre-existing lowering (dynamic `+` per element) for A/B bisection. +pub(super) fn packed_loop_numeric_accumulators_enabled() -> bool { + use std::sync::OnceLock; + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| { + !matches!( + std::env::var("PERRY_PACKED_LOOP_NUMERIC_ACCUMULATOR").as_deref(), + Ok("0") | Ok("off") | Ok("false") + ) + }) +} + +/// Fail-closed walk: is `expr` numeric with every leaf provable numeric +/// INSIDE the fast clone? Leaves are literals, the exact `array[counter]` +/// read (proven raw f64 by the clone's guard — the caller only admits +/// accumulators when `numeric_elements` is set), locals that are numeric on +/// their own, and the candidate accumulators themselves (the induction +/// hypothesis — the preheader tag test is the base case). Anything else — +/// calls, property reads, other indexed reads, closures — declines the +/// accumulator. `Add` is safe here for the same reason the element-shape walk +/// documents: with every admitted leaf numeric, string concatenation is +/// unreachable. +fn accumulator_rhs_is_numeric( + ctx: &FnCtx<'_>, + expr: &Expr, + array_id: u32, + counter_id: u32, + candidates: &std::collections::BTreeSet, +) -> bool { + match expr { + Expr::Number(_) | Expr::Integer(_) => true, + Expr::IndexGet { object, index } => matches!( + (object.as_ref(), index.as_ref()), + (Expr::LocalGet(a), Expr::LocalGet(i)) if *a == array_id && *i == counter_id + ), + Expr::LocalGet(id) => { + candidates.contains(id) || crate::type_analysis::is_numeric_expr(ctx, expr) + } + Expr::Binary { left, right, .. } => { + accumulator_rhs_is_numeric(ctx, left, array_id, counter_id, candidates) + && accumulator_rhs_is_numeric(ctx, right, array_id, counter_id, candidates) + } + Expr::NumberCoerce(operand) => { + accumulator_rhs_is_numeric(ctx, operand, array_id, counter_id, candidates) + } + Expr::Unary { op, operand } => { + matches!( + op, + perry_hir::UnaryOp::Neg | perry_hir::UnaryOp::Pos | perry_hir::UnaryOp::BitNot + ) && accumulator_rhs_is_numeric(ctx, operand, array_id, counter_id, candidates) + } + Expr::MathAbs(v) + | Expr::MathSqrt(v) + | Expr::MathFloor(v) + | Expr::MathCeil(v) + | Expr::MathRound(v) + | Expr::MathTrunc(v) + | Expr::MathSign(v) + | Expr::MathFround(v) => { + accumulator_rhs_is_numeric(ctx, v, array_id, counter_id, candidates) + } + Expr::MathImul(l, r) | Expr::MathPow(l, r) => { + accumulator_rhs_is_numeric(ctx, l, array_id, counter_id, candidates) + && accumulator_rhs_is_numeric(ctx, r, array_id, counter_id, candidates) + } + Expr::MathMin(values) | Expr::MathMax(values) => values + .iter() + .all(|v| accumulator_rhs_is_numeric(ctx, v, array_id, counter_id, candidates)), + _ => false, + } +} + +/// Collect every write (`LocalSet` / `Update`) per local in `body`, without +/// descending into nested closures (their writes go through boxes, and a +/// boxed local is excluded from admission anyway). +fn collect_local_writes<'a>( + stmts: &'a [Stmt], + out: &mut std::collections::BTreeMap>>, +) { + fn walk_expr<'a>( + expr: &'a Expr, + out: &mut std::collections::BTreeMap>>, + ) { + match expr { + Expr::LocalSet(id, value) => { + out.entry(*id).or_default().push(Some(value)); + walk_expr(value, out); + } + Expr::Update { id, .. } => { + out.entry(*id).or_default().push(None); + } + Expr::Closure { .. } => {} + other => { + perry_hir::walker::walk_expr_children(other, &mut |child| walk_expr(child, out)); + } + } + } + fn walk_stmt<'a>( + stmt: &'a Stmt, + out: &mut std::collections::BTreeMap>>, + ) { + match stmt { + Stmt::Let { init, .. } => { + if let Some(init) = init { + walk_expr(init, out); + } + } + Stmt::Expr(e) | Stmt::Throw(e) => walk_expr(e, out), + Stmt::Return(e) => { + if let Some(e) = e { + walk_expr(e, out); + } + } + Stmt::If { + condition, + then_branch, + else_branch, + } => { + walk_expr(condition, out); + for s in then_branch { + walk_stmt(s, out); + } + if let Some(body) = else_branch { + for s in body { + walk_stmt(s, out); + } + } + } + Stmt::While { condition, body } => { + walk_expr(condition, out); + for s in body { + walk_stmt(s, out); + } + } + Stmt::DoWhile { body, condition } => { + for s in body { + walk_stmt(s, out); + } + walk_expr(condition, out); + } + Stmt::For { + init, + condition, + update, + body, + } => { + if let Some(init) = init { + walk_stmt(init, out); + } + if let Some(condition) = condition { + walk_expr(condition, out); + } + if let Some(update) = update { + walk_expr(update, out); + } + for s in body { + walk_stmt(s, out); + } + } + Stmt::Try { + body, + catch, + finally, + } => { + for s in body { + walk_stmt(s, out); + } + if let Some(catch) = catch { + for s in &catch.body { + walk_stmt(s, out); + } + } + if let Some(body) = finally { + for s in body { + walk_stmt(s, out); + } + } + } + Stmt::Switch { + discriminant, + cases, + } => { + walk_expr(discriminant, out); + for case in cases { + if let Some(test) = &case.test { + walk_expr(test, out); + } + for s in &case.body { + walk_stmt(s, out); + } + } + } + Stmt::Labeled { body, .. } => walk_stmt(body, out), + Stmt::Break + | Stmt::Continue + | Stmt::LabeledBreak(_) + | Stmt::LabeledContinue(_) + | Stmt::PreallocateBoxes(_) + | Stmt::PreallocateTdzBoxes(_) + | Stmt::ReleaseBoxes(_) => {} + } + } + for stmt in stmts { + walk_stmt(stmt, out); + } +} + +/// Reduce accumulators the fast clone may prove numeric: plain uncaptured +/// locals whose every write in `body` is numeric-preserving under the +/// fixpoint. `Update` writes preserve Number-ness on a Number (BigInt cannot +/// appear: the preheader proves the base case and no admitted write produces +/// one). Fail-closed at every step. +pub(super) fn collect_numeric_accumulators( + ctx: &FnCtx<'_>, + body: &[Stmt], + array_id: u32, + counter_id: u32, +) -> Vec { + if !packed_loop_numeric_accumulators_enabled() { + return Vec::new(); + } + let mut writes = std::collections::BTreeMap::new(); + collect_local_writes(body, &mut writes); + let mut candidates: std::collections::BTreeSet = writes + .keys() + .copied() + .filter(|id| { + *id != array_id + && *id != counter_id + && ctx.locals.contains_key(id) + && !ctx.boxed_vars.contains(id) + && !ctx.closure_captures.contains_key(id) + && !ctx.module_globals.contains_key(id) + && !ctx.i32_counter_slots.contains_key(id) + && ctx.shadow_slot_map.contains_key(id) + }) + .collect(); + loop { + let rejected: Vec = candidates + .iter() + .copied() + .filter(|id| { + !writes[id].iter().all(|write| match write { + Some(rhs) => { + accumulator_rhs_is_numeric(ctx, rhs, array_id, counter_id, &candidates) + } + // `Update` (++/--): ToNumeric(Number) ± 1 is a Number. + None => true, + }) + }) + .collect(); + if rejected.is_empty() { + break; + } + for id in rejected { + candidates.remove(&id); + } + } + candidates.into_iter().collect() +} diff --git a/crates/perry-codegen/src/stmt/stable_packed_loop.rs b/crates/perry-codegen/src/stmt/stable_packed_loop.rs index 31967e4477..40c5c20f7f 100644 --- a/crates/perry-codegen/src/stmt/stable_packed_loop.rs +++ b/crates/perry-codegen/src/stmt/stable_packed_loop.rs @@ -1321,6 +1321,8 @@ fn finish_revalidated_read( .phi(DOUBLE, &[(&direct, &direct_end), (&generic, &fallback_end)]) } +use super::stable_packed_accumulator::collect_numeric_accumulators; + pub(crate) fn has_numeric_index_fact(ctx: &FnCtx<'_>, expr: &Expr) -> bool { let Expr::IndexGet { object, index } = expr else { return false; @@ -1598,6 +1600,41 @@ pub(super) fn lower( } else { None }; + // Reduce accumulators: one tag test each here in the fast preheader (the + // induction base case), then the fact below carries the proof through the + // fast clone so `s += arr[counter]` lowers to a native `fadd`. Admission + // requires `numeric_elements` — without the element proof the accumulator + // walk's `array[counter]` leaf has nothing to stand on. + let numeric_accumulators = if candidate.numeric_elements { + collect_numeric_accumulators(ctx, body, candidate.array_id, candidate.counter_id) + } else { + Vec::new() + }; + if !numeric_accumulators.is_empty() { + let mut all_numbers: Option = None; + for id in &numeric_accumulators { + let slot = ctx + .locals + .get(id) + .cloned() + .expect("admitted accumulator has a plain slot"); + let value = ctx.block().load(DOUBLE, &slot); + let is_number = super::loops::emit_js_value_is_number(ctx, &value); + all_numbers = Some(match all_numbers { + Some(prev) => ctx.block().and(I1, &prev, &is_number), + None => is_number, + }); + } + let all_numbers = all_numbers.expect("at least one accumulator"); + let acc_ok_idx = ctx.new_block("stable_packed.acc.ok"); + let acc_ok_label = ctx.block_label(acc_ok_idx); + // A non-Number accumulator (a string total, a BigInt) takes the slow + // clone before the first fast iteration; nothing has run yet, so the + // slow clone sees pristine state. + ctx.block() + .cond_br(&all_numbers, &acc_ok_label, &slow_pre_label); + ctx.current_block = acc_ok_idx; + } let revalidation_dirty_slot = candidate .nested_requires_access_revalidation .then(|| ctx.func.alloca_entry(I1)); @@ -1653,6 +1690,7 @@ pub(super) fn lower( .map(|installed| installed.common_length.clone()), u32_out_of_bounds_label: None, numeric_access, + numeric_accumulators, derived_locals: std::collections::HashSet::new(), u32_view_derived_locals: std::collections::HashMap::new(), }); diff --git a/crates/perry-codegen/src/type_analysis/numeric.rs b/crates/perry-codegen/src/type_analysis/numeric.rs index b01bced962..f264aa171a 100644 --- a/crates/perry-codegen/src/type_analysis/numeric.rs +++ b/crates/perry-codegen/src/type_analysis/numeric.rs @@ -150,6 +150,16 @@ pub(crate) fn is_numeric_expr(ctx: &FnCtx<'_>, e: &Expr) -> bool { .iter() .rev() .any(|fact| fact.numeric_accumulator == *id) + // The stable-packed twin: the fast preheader tag-tested the + // accumulator and every in-clone write is numeric-preserving, + // so within the fast clone the local provably holds a Number. + // The fact is pushed around the fast-clone lowering only, so + // the slow clone and post-loop code never see it. + || ctx + .stable_packed_loop_facts + .iter() + .rev() + .any(|fact| fact.numeric_accumulators.contains(id)) || ctx.integer_locals.contains(id) || ctx.unsigned_i32_locals.contains(id) || ctx.int_valued_i64_locals.contains_key(id) diff --git a/crates/perry/src/commands/compile/build_cache.rs b/crates/perry/src/commands/compile/build_cache.rs index 143d274d90..8aca884bcb 100644 --- a/crates/perry/src/commands/compile/build_cache.rs +++ b/crates/perry/src/commands/compile/build_cache.rs @@ -49,6 +49,11 @@ const BUILD_CACHE_ENV_VARS: &[&str] = &[ // `disable-tail-calls` before the optimizer. It changes the generated // code of the functions it trips on, so it is a cache input. "PERRY_LL_TRE_MAX_ALLOCA_WALK", + // #9060: gates whether a reduce accumulator earns the stable-packed fast + // clone's numeric proof — with it on, `s += arr[i]` lowers to an inline + // fadd instead of `js_dynamic_string_or_number_add`, so the two settings + // emit different code and must not share a cached object. + "PERRY_PACKED_LOOP_NUMERIC_ACCUMULATOR", // #9026: gates the once-per-closure-entry resolution of read-only boxed // capture cells — flipping it changes every closure body that qualifies. "PERRY_BOX_CAPTURE_ENTRY_CELLS",