perf: ECS command path follow-up to #8872 — −40% on the codehz/ecs 5k-entity sync row

changelog.d/8885-ecs-followup-gc-bookkeeping.md

@@ -0,0 +1,18 @@
+Removed most of the remaining GC-bookkeeping and dispatch overhead on the ECS
+command path (follow-up to #8872). General compiler/runtime mechanisms:
+branded `number & {…}` intersections and generic/imported type aliases now
+lower to their primitive (they were `Any`, so every entity id was dynamic);
+inline plain-double fast paths for dynamic compares and truthiness;
+beta-reduction of called arrow-literal locals left behind by inlining;
+`Map.get` heals stale array-growth forwarding stubs in place; a header-bit
+fast lane in `clean_arr_ptr` and every strict array helper; a process-global
+address sketch, an object/closure per-object-mask threshold of eight slots, a
+death prune for the per-object layout tables and a live-young-record count
+that together let the inline allocator skip the stale-layout probe; the
+strict store lane, `pop` and `length =` resolve the header once; the
+dirty-page and `typeof` caches move to hot TLS; the string-demote tag test
+and plain closure-capture reads are inlined; a validated-parent write-barrier
+entry; and `length = 0` keeps an all-pointer array all-pointer. On the
+upstream `codehz/ecs` "5k entities: 3 commands each + sync" row the compiled
+benchmark went from 7.30 ms/op to ~4.7 ms/op (−36%, paired runs on an idle
+Mac mini; Node 26.5.1 is 1.76 ms/op).

crates/perry-codegen/src/codegen/closure.rs

@@ -1109,6 +1109,7 @@ pub(super) fn compile_closure(
         declared_only_numeric_locals: std::collections::HashSet::new(),
         shadow_slot_clears_after_stmt,
         arena_state_slot: None,
+        arena_state_lazy: false,
         class_keys_slots: HashMap::new(),
         class_shape_slots: HashMap::new(),
         class_header_images: HashMap::new(),

crates/perry-codegen/src/codegen/declared_string_add_tests.rs

@@ -628,6 +628,20 @@ fn assigning_one_local_to_another_demotes_a_possible_string_alias() {
         ir.contains("call void @js_string_addref_if_heap_string("),
         "assignment aliases need the same demote as declaration aliases:\n{ir}"
     );
+    // The helper's tag test is hoisted into IR: the call sits behind an
+    // inline `STRING_TAG` compare, so a numeric copy never leaves the
+    // function. Pin the compare AND the call — the demote must still reach
+    // the runtime for a real heap string.
+    let tag_at = ir
+        .find("icmp ne i64 %")
+        .expect("inline STRING_TAG compare before the demote call");
+    let call_at = ir
+        .find("call void @js_string_addref_if_heap_string(")
+        .expect("demote call");
+    assert!(
+        ir.contains(", 9223090561878065152") && tag_at < call_at,
+        "the demote call must be guarded by an inline 0x7FFF_0000_0000_0000 tag compare:\n{ir}"
+    );
 }
 
 // ------------------------------------------------------- untouched tiers

crates/perry-codegen/src/codegen/entry.rs

@@ -871,6 +871,7 @@ pub(super) fn compile_module_entry(
             declared_only_numeric_locals: std::collections::HashSet::new(),
             shadow_slot_clears_after_stmt: main_shadow_slot_clears_after_stmt,
             arena_state_slot: None,
+            arena_state_lazy: false,
             class_keys_slots: HashMap::new(),
             class_shape_slots: HashMap::new(),
             class_header_images: HashMap::new(),
@@ -1583,6 +1584,7 @@ pub(super) fn compile_module_entry(
             declared_only_numeric_locals: std::collections::HashSet::new(),
             shadow_slot_clears_after_stmt: init_shadow_slot_clears_after_stmt,
             arena_state_slot: None,
+            arena_state_lazy: false,
             class_keys_slots: HashMap::new(),
             class_shape_slots: HashMap::new(),
             class_header_images: HashMap::new(),

crates/perry-codegen/src/codegen/function.rs

@@ -1125,6 +1125,7 @@ pub(super) fn compile_function(
         shadow_slots_bound: bound_param_slots,
         temp_roots: crate::rooting::TempRootPool::default(),
         arena_state_slot,
+        arena_state_lazy: false,
         class_keys_slots: HashMap::new(),
         class_shape_slots: HashMap::new(),
         class_header_images: HashMap::new(),

crates/perry-codegen/src/codegen/method.rs

@@ -481,6 +481,7 @@ pub(super) fn compile_method(
         declared_only_numeric_locals: std::collections::HashSet::new(),
         shadow_slot_clears_after_stmt,
         arena_state_slot: None,
+        arena_state_lazy: false,
         class_keys_slots: HashMap::new(),
         class_shape_slots: HashMap::new(),
         class_header_images: HashMap::new(),
@@ -1779,6 +1780,7 @@ pub(super) fn compile_static_method(
         declared_only_numeric_locals: std::collections::HashSet::new(),
         shadow_slot_clears_after_stmt,
         arena_state_slot: None,
+        arena_state_lazy: false,
         class_keys_slots: HashMap::new(),
         class_shape_slots: HashMap::new(),
         class_header_images: HashMap::new(),

crates/perry-codegen/src/expr/barrier_stem_census_tests.rs

@@ -94,7 +94,7 @@ pub(super) const VERIFIED_BARRIER_STEMS: &[(&str, StemKind)] = &[
     ("put.pic", StemKind::PointerTestedStore),
 ];
 
-const BARRIER_CALL: &str = "call void @js_write_barrier_slot(";
+const BARRIER_CALL: &str = "call void @js_write_barrier_slot";
 const INCREMENTAL_GLOBAL: &str = "@PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT";
 
 // ---------------------------------------------------------------------------

crates/perry-codegen/src/expr/compare.rs

@@ -339,6 +339,85 @@ fn lower_string_literal_strict_eq(
 ///
 /// Returns an i64 holding `TAG_TRUE`/`TAG_FALSE` (or `js_eq`'s own tagged
 /// boolean), i.e. the same value the bare call produced.
+/// Quiet-NaN prefix (`0x7FF8_0000_0000_0000`) shared by every Perry NaN-box
+/// tag and by the canonical NaN itself.
+const QNAN_PREFIX_I64: &str = "9221120237041090560";
+
+/// `(bits & 0x7FF8…) != 0x7FF8…`: the operand is an ordinary IEEE double —
+/// finite, ±Infinity, or a signaling-NaN pattern no Perry encoding occupies.
+/// Every NaN-box tag (top-16 `0x7FF9`..=`0x7FFF`, sign-clear) and the quiet
+/// NaN carry the prefix, so one mask+compare separates "plain number" from
+/// "tagged or NaN" without decoding either side. Two plain numbers answer
+/// every relational and (strict or loose) equality operator with the raw
+/// `fcmp`; the helper keeps NaN, so the unordered edge never reaches the
+/// inline predicate.
+fn emit_is_plain_double(ctx: &mut FnCtx<'_>, bits: &str) -> String {
+    let blk = ctx.block();
+    let masked = blk.and(I64, bits, QNAN_PREFIX_I64);
+    blk.icmp_ne(I64, &masked, QNAN_PREFIX_I64)
+}
+
+/// Dynamic-operand comparison with an inline plain-number fast path.
+///
+/// When both NaN-boxed operands are ordinary doubles the result is
+/// `select(fcmp <pred> l, r, TAG_TRUE, TAG_FALSE)`; every other shape —
+/// strings, BigInt, objects with `valueOf`/`toString`, null/undefined/boolean
+/// coercions, NaN — takes `helper`, which owns the full ECMAScript semantics.
+/// `helper_takes_bits` selects the `(i64, i64) -> i64` helper ABI
+/// (`js_eq`, `js_loose_eq`) over the `(double, double) -> double` one
+/// (`js_rel_*`). Returns the NaN-boxed boolean as i64 bits.
+fn lower_dynamic_compare_bits(
+    ctx: &mut FnCtx<'_>,
+    l: &str,
+    r: &str,
+    pred: &str,
+    helper: &str,
+    helper_takes_bits: bool,
+) -> String {
+    let l_bits = ctx.block().bitcast_double_to_i64(l);
+    let r_bits = ctx.block().bitcast_double_to_i64(r);
+    let l_plain = emit_is_plain_double(ctx, &l_bits);
+    let r_plain = emit_is_plain_double(ctx, &r_bits);
+    let both_plain = ctx.block().and(I1, &l_plain, &r_plain);
+
+    let fast_idx = ctx.new_block("dyncmp.num");
+    let slow_idx = ctx.new_block("dyncmp.slow");
+    let merge_idx = ctx.new_block("dyncmp.merge");
+    let fast_l = ctx.block_label(fast_idx);
+    let slow_l = ctx.block_label(slow_idx);
+    let merge_l = ctx.block_label(merge_idx);
+    ctx.block().cond_br(&both_plain, &fast_l, &slow_l);
+
+    ctx.current_block = fast_idx;
+    let bit = ctx.block().fcmp(pred, l, r);
+    let fast_res = ctx.block().select(
+        I1,
+        &bit,
+        I64,
+        crate::nanbox::TAG_TRUE_I64,
+        crate::nanbox::TAG_FALSE_I64,
+    );
+    let fast_pred = ctx.block().label.clone();
+    ctx.block().br(&merge_l);
+
+    ctx.current_block = slow_idx;
+    let slow_res = if helper_takes_bits {
+        ctx.block()
+            .call(I64, helper, &[(I64, &l_bits), (I64, &r_bits)])
+    } else {
+        let boxed = ctx
+            .block()
+            .call(DOUBLE, helper, &[(DOUBLE, l), (DOUBLE, r)]);
+        ctx.block().bitcast_double_to_i64(&boxed)
+    };
+    let slow_pred = ctx.block().label.clone();
+    ctx.block().br(&merge_l);
+
+    ctx.current_block = merge_idx;
+    ctx.block()
+        .phi(I64, &[(&fast_res, &fast_pred), (&slow_res, &slow_pred)])
+}
+
 fn lower_strict_eq_inline_any(ctx: &mut FnCtx<'_>, l: &str, r: &str) -> String {
     let l_bits = ctx.block().bitcast_double_to_i64(l);
     let r_bits = ctx.block().bitcast_double_to_i64(r);
@@ -374,10 +453,26 @@ fn lower_strict_eq_inline_any(ctx: &mut FnCtx<'_>, l: &str, r: &str) -> String {
     let same_ok = ctx.block().or(I1, &tagged, &not_nan);
     ctx.block().cond_br(&same_ok, &true_l, &slow_l);
 
-    // Different bits: only a same-tag pair whose encoding is canonical is
-    // decidable here. Pointer pairs need the runtime's allocation registries
-    // before either payload can safely be treated as a GC allocation.
+    // Different bits: two plain numbers are decided by `fcmp` (only `+0`/`-0`
+    // differ in bits yet compare equal); otherwise only a same-tag pair whose
+    // encoding is canonical is decidable here. Pointer pairs need the
+    // runtime's allocation registries before either payload can safely be
+    // treated as a GC allocation.
     ctx.current_block = diff_idx;
+    let num_idx = ctx.new_block("anyeq.num");
+    let tag_idx = ctx.new_block("anyeq.tag");
+    let num_l = ctx.block_label(num_idx);
+    let tag_l = ctx.block_label(tag_idx);
+    let l_plain = emit_is_plain_double(ctx, &l_bits);
+    let r_plain = emit_is_plain_double(ctx, &r_bits);
+    let both_plain = ctx.block().and(I1, &l_plain, &r_plain);
+    ctx.block().cond_br(&both_plain, &num_l, &tag_l);
+
+    ctx.current_block = num_idx;
+    let num_eq = ctx.block().fcmp("oeq", l, r);
+    ctx.block().cond_br(&num_eq, &true_l, &false_l);
+
+    ctx.current_block = tag_idx;
     let l_tag = ctx.block().lshr(I64, &l_bits, "48");
     let r_tag = ctx.block().lshr(I64, &r_bits, "48");
     let l_sso = ctx
@@ -985,10 +1080,7 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result<String> {
                     // call. Loose `==`'s cross-type coercions are not
                     // bit-decidable, so it keeps the bare call.
                     let result_bits = if matches!(op, CompareOp::LooseEq | CompareOp::LooseNe) {
-                        let blk = ctx.block();
-                        let l_bits = blk.bitcast_double_to_i64(&l);
-                        let r_bits = blk.bitcast_double_to_i64(&r);
-                        blk.call(I64, "js_loose_eq", &[(I64, &l_bits), (I64, &r_bits)])
+                        lower_dynamic_compare_bits(ctx, &l, &r, "oeq", "js_loose_eq", true)
                     } else {
                         lower_strict_eq_inline_any(ctx, &l, &r)
                     };
@@ -1162,11 +1254,9 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result<String> {
                 // "1"==1, false==0, etc.). Strict === already handled
                 // above by the typed fast paths.
                 if matches!(op, CompareOp::LooseEq | CompareOp::LooseNe) {
-                    let blk = ctx.block();
-                    let l_bits = blk.bitcast_double_to_i64(&l);
-                    let r_bits = blk.bitcast_double_to_i64(&r);
                     let result_bits =
-                        blk.call(I64, "js_loose_eq", &[(I64, &l_bits), (I64, &r_bits)]);
+                        lower_dynamic_compare_bits(ctx, &l, &r, "oeq", "js_loose_eq", true);
+                    let blk = ctx.block();
                     if matches!(op, CompareOp::LooseNe) {
                         let cmp = blk.icmp_eq(I64, &result_bits, crate::nanbox::TAG_TRUE_I64);
                         let inv = blk.xor(crate::types::I1, &cmp, "true");
@@ -1198,16 +1288,18 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result<String> {
                     && !is_bigint_expr(ctx, left)
                     && !is_bigint_expr(ctx, right);
                 if is_relational_op && !both_numeric {
-                    let blk = ctx.block();
-                    let fname = match op {
-                        CompareOp::Lt => "js_rel_lt",
-                        CompareOp::Le => "js_rel_le",
-                        CompareOp::Gt => "js_rel_gt",
-                        CompareOp::Ge => "js_rel_ge",
+                    let (pred, fname) = match op {
+                        CompareOp::Lt => ("olt", "js_rel_lt"),
+                        CompareOp::Le => ("ole", "js_rel_le"),
+                        CompareOp::Gt => ("ogt", "js_rel_gt"),
+                        CompareOp::Ge => ("oge", "js_rel_ge"),
                         _ => unreachable!(),
                     };
-                    let res = blk.call(DOUBLE, fname, &[(DOUBLE, &l), (DOUBLE, &r)]);
-                    return Ok(res);
+                    // Two plain numbers are the overwhelmingly common dynamic
+                    // shape (erased ids, PIC-loaded fields); they take the raw
+                    // `fcmp` inline and everything else keeps the helper.
+                    let bits = lower_dynamic_compare_bits(ctx, &l, &r, pred, fname, false);
+                    return Ok(ctx.block().bitcast_i64_to_double(&bits));
                 }
                 // Strict ===/!== where the operands are NOT both certainly
                 // numeric must NOT fall to the bare fcmp tail: a declared
@@ -1217,10 +1309,8 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result<String> {
                 // js_eq answers correctly for every runtime shape, including
                 // the honest number-vs-object case (#3576 probe family).
                 if matches!(op, CompareOp::Eq | CompareOp::Ne) && !both_numeric {
+                    let result_bits = lower_dynamic_compare_bits(ctx, &l, &r, "oeq", "js_eq", true);
                     let blk = ctx.block();
-                    let l_bits = blk.bitcast_double_to_i64(&l);
-                    let r_bits = blk.bitcast_double_to_i64(&r);
-                    let result_bits = blk.call(I64, "js_eq", &[(I64, &l_bits), (I64, &r_bits)]);
                     if matches!(op, CompareOp::Ne) {
                         let cmp = blk.icmp_eq(I64, &result_bits, crate::nanbox::TAG_TRUE_I64);
                         let inv = blk.xor(crate::types::I1, &cmp, "true");

crates/perry-codegen/src/expr/compare_tests.rs

@@ -559,3 +559,115 @@ fn i8_literal_writes_high_bytes_in_twos_complement() {
     assert_eq!(i8_literal(0xC3), "-61");
     assert_eq!(i8_literal(0xFF), "-1");
 }
+
+// ---------------------------------------------------------------------------
+// Dynamic-operand plain-number fast paths (#8872 follow-up).
+//
+// Two erased operands used to go straight to the runtime helper for every
+// relational and equality operator. Two ordinary doubles — the overwhelmingly
+// common dynamic shape (erased ids, PIC-loaded fields) — now decide inline
+// with the raw `fcmp`; the helper stays reachable for every other shape.
+// ---------------------------------------------------------------------------
+
+const JS_REL_GE_CALL: &str = "call double @js_rel_ge(";
+const JS_REL_LT_CALL: &str = "call double @js_rel_lt(";
+
+#[test]
+fn dynamic_relational_decides_plain_numbers_inline_and_keeps_the_helper() {
+    let ir = cmp_ir(
+        "dynrel_ge",
+        CompareOp::Ge,
+        Expr::LocalGet(X),
+        Expr::LocalGet(Y),
+    );
+    assert!(
+        ir.contains("fcmp oge double"),
+        "dynamic `>=` has no inline number arm:\n{ir}"
+    );
+    assert!(
+        ir.contains(JS_REL_GE_CALL),
+        "dynamic `>=` lost its coercing helper fallback:\n{ir}"
+    );
+    let ir = cmp_ir(
+        "dynrel_lt",
+        CompareOp::Lt,
+        Expr::LocalGet(X),
+        Expr::LocalGet(Y),
+    );
+    assert!(ir.contains("fcmp olt double"), "no inline `<` arm:\n{ir}");
+    assert!(ir.contains(JS_REL_LT_CALL), "no `<` helper fallback:\n{ir}");
+}
+
+#[test]
+fn dynamic_strict_eq_decides_plain_numbers_inline_and_keeps_js_eq() {
+    for (name, op) in [("dynseq", CompareOp::Eq), ("dynsne", CompareOp::Ne)] {
+        let ir = cmp_ir(name, op, Expr::LocalGet(X), Expr::LocalGet(Y));
+        assert!(
+            ir.contains("fcmp oeq double"),
+            "{name}: no inline number arm:\n{ir}"
+        );
+        assert!(
+            ir.contains(JS_EQ_CALL),
+            "{name}: lost the js_eq fallback:\n{ir}"
+        );
+    }
+}
+
+#[test]
+fn dynamic_loose_eq_decides_plain_numbers_inline_and_keeps_js_loose_eq() {
+    let ir = cmp_ir(
+        "dynleq",
+        CompareOp::LooseEq,
+        Expr::LocalGet(X),
+        Expr::LocalGet(Y),
+    );
+    assert!(
+        ir.contains("fcmp oeq double"),
+        "dynamic `==` has no inline number arm:\n{ir}"
+    );
+    assert!(
+        ir.contains(JS_LOOSE_EQ_CALL),
+        "dynamic `==` lost its coercing helper:\n{ir}"
+    );
+}
+
+#[test]
+fn dynamic_truthiness_decides_numbers_and_tag_singletons_inline() {
+    let ir = ir_for(
+        "dyntruthy",
+        vec![
+            Stmt::Let {
+                id: X,
+                name: "x".to_string(),
+                ty: Type::Any,
+                mutable: true,
+                init: Some(Expr::Undefined),
+            },
+            Stmt::Let {
+                id: R,
+                name: "r".to_string(),
+                ty: Type::Any,
+                mutable: true,
+                init: Some(Expr::Undefined),
+            },
+            Stmt::If {
+                // An element read of an erased local: no initializer proof can
+                // settle it, so the condition reaches the dynamic predicate.
+                condition: Expr::IndexGet {
+                    object: Box::new(Expr::LocalGet(X)),
+                    index: Box::new(Expr::Integer(0)),
+                },
+                then_branch: vec![Stmt::Expr(Expr::LocalSet(R, Box::new(Expr::Integer(1))))],
+                else_branch: None,
+            },
+        ],
+    );
+    assert!(
+        ir.contains("truthy.num") && ir.contains("fcmp one double"),
+        "dynamic truthiness has no inline number arm:\n{ir}"
+    );
+    assert!(
+        ir.contains("call i32 @js_is_truthy("),
+        "dynamic truthiness lost the runtime predicate for strings/BigInt:\n{ir}"
+    );
+}