perf(runtime): IC hits stop re-deriving shape-immutable facts (combined loop now beats node)

changelog.d/9022-ic-hit-immutable-facts.md

@@ -0,0 +1,32 @@
+IC hits stop re-deriving shape-immutable facts, and the combined
+overwrite+read loop goes past node: 31 → **24 ms against node's 29**. The
+write-only loop drops to **15 ms against node's 23**.
+
+Every stub or way hit re-proved the receiver's kind (a registry probe via
+`is_class_object_ptr`), the plain-ordinary verdict, and the inline slot bound
+(a descriptor fetch) — together 16% of the combined loop
+(`write_fast_path_receiver_kind_ok` 6.8%, `shape_object_kind_by_id` 5.1%,
+`shape_live_inline_slot_count_by_id` 4.4%).
+
+All of those are facts of the SHAPE ID. The shape table only ever inserts
+descriptors — its sole in-place mutations are GC bookkeeping (the relocated
+`keys` address and the carrier liveness bits) — so `object_kind`,
+`live_inline_slot_count` and `logical_key_count` cannot change under a fixed
+id. A hit whose token matches the receiver's CURRENT stamp has therefore
+already proved everything prime time proved about kind and bounds.
+
+The slot word now carries the one prime-time verdict a hit needs:
+`IC_SLOT_OVERFLOW_BIT`, choosing the inline region vs the spill store.
+Overflow entries are bound-checked against `logical_key_count` at prime. Hits
+keep exactly the checks that ARE mutable per object — header type, forwarded,
+and the blocking flags the `Object.freeze` family sets — plus the token
+compare. Applied to the write stub, the per-site dyn ways, and both read-stub
+hit sites.
+
+Interleaved A/B, min-of-21 at quiet load (~0.8): combined overwrite 31 →
+24 ms; computed-key read, realistic-name read and populated delete unchanged;
+write-only measured on the branch at 15 ms (node 23).
+
+Verification: suite 2790 passed, all 60 lint gates pass, no warnings; the
+adversarial, computed-key, and stale-slot differentials all byte-identical to
+node.

crates/perry-runtime/src/object/field_get_set/get_field_by_name.rs

@@ -142,15 +142,12 @@ pub extern "C" fn js_object_get_field_by_name(
                             if let Some(slot) =
                                 super::super::read_stub::read_stub_probe(token, key_bits)
                             {
-                                let live = crate::object::object_live_slot_count(o);
-                                let limit =
-                                    std::cmp::max(live, crate::object::INLINE_SLOT_FLOOR as u32);
-                                if slot < limit {
-                                    return super::accessors::js_object_get_field(o, slot);
-                                }
-                                if let Some(bits) = super::super::overflow_get(addr, slot as usize)
+                                // The slot word carries the inline/overflow
+                                // verdict from prime time; no bound fetch.
+                                if let Some(v) =
+                                    super::super::read_stub::read_slot_by_tag(o, addr, slot)
                                 {
-                                    return JSValue::from_bits(bits);
+                                    return JSValue::from_bits(v.to_bits());
                                 }
                             }
                         }
@@ -228,7 +225,7 @@ pub extern "C" fn js_object_get_field_by_name(
                                 keys as usize,
                                 key as usize,
                             ) {
-                                prime_read_stub(o, key, idx);
+                                prime_read_stub(o, key, idx, (idx as usize) < alloc_limit);
                                 return if (idx as usize) < alloc_limit {
                                     super::accessors::js_object_get_field(o, idx)
                                 } else {
@@ -264,7 +261,7 @@ pub extern "C" fn js_object_get_field_by_name(
                                         key,
                                     ) {
                                         let i = i as usize;
-                                        prime_read_stub(o, key, i as u32);
+                                        prime_read_stub(o, key, i as u32, i < alloc_limit);
                                         super::super::prop_plan::read_plan_record(
                                             keys as usize,
                                             key as usize,
@@ -1758,7 +1755,17 @@ mod null_key_guard_5972 {
 /// semantics. Keys that cannot be represented as content bits are skipped by
 /// `read_stub_key_bits`.
 #[inline]
-fn prime_read_stub(obj: *const ObjectHeader, key: *const crate::StringHeader, slot: u32) {
+fn prime_read_stub(
+    obj: *const ObjectHeader,
+    key: *const crate::StringHeader,
+    slot: u32,
+    inline: bool,
+) {
+    let slot = if inline {
+        slot
+    } else {
+        slot | crate::proxy::IC_SLOT_OVERFLOW_BIT
+    };
     if let Some(key_bits) = super::super::read_stub::read_stub_key_bits(key) {
         if let Some(token) = unsafe { super::super::read_stub::receiver_shape_token(obj) } {
             super::super::read_stub::read_stub_insert(token, key_bits, slot);

crates/perry-runtime/src/object/read_stub.rs

@@ -159,14 +159,32 @@ pub(crate) unsafe fn try_read_by_content_bits(
     }
     let token = receiver_shape_token(obj)?;
     let slot = read_stub_probe(token, key_bits)?;
-    let limit = std::cmp::max(
-        crate::object::object_live_slot_count(obj),
-        crate::object::INLINE_SLOT_FLOOR as u32,
-    );
-    if slot < limit {
-        return Some(f64::from_bits(
-            crate::object::js_object_get_field(obj, slot).bits(),
-        ));
+    read_slot_by_tag(obj, addr, slot)
+}
+
+/// Read the value a bit-tagged cached slot names. The inline/overflow verdict
+/// was decided at prime time (`IC_SLOT_OVERFLOW_BIT`, see `proxy::put_value`)
+/// under the exact shape id the caller's token match just re-proved, so no
+/// bound is fetched here — the shape table only ever inserts descriptors, so
+/// the verdict cannot have changed under the same id.
+#[inline(always)]
+pub(crate) unsafe fn read_slot_by_tag(
+    obj: *const ObjectHeader,
+    addr: usize,
+    slot: u32,
+) -> Option<f64> {
+    use crate::proxy::IC_SLOT_OVERFLOW_BIT;
+    if slot & IC_SLOT_OVERFLOW_BIT != 0 {
+        return crate::object::overflow_get(addr, (slot & !IC_SLOT_OVERFLOW_BIT) as usize)
+            .map(f64::from_bits);
+    }
+    let fields_ptr =
+        (obj as *const u8).add(std::mem::size_of::<ObjectHeader>()) as *const crate::JSValue;
+    let val = *fields_ptr.add(slot as usize);
+    // Same null-POINTER_TAG guard as `js_object_get_field`'s inline half: the
+    // pattern is never a legitimate stored value.
+    if val.bits() == 0x7FFD_0000_0000_0000 {
+        return Some(f64::from_bits(crate::value::TAG_UNDEFINED));
     }
-    crate::object::overflow_get(addr, slot as usize).map(f64::from_bits)
+    Some(f64::from_bits(val.bits()))
 }

crates/perry-runtime/src/proxy.rs

@@ -31,7 +31,9 @@ pub use has_delete::{js_proxy_delete, js_proxy_has};
 mod invariants;
 mod put_value;
 pub use put_value::{js_proxy_set, js_put_value_set};
-pub(crate) use put_value::{js_put_value_set_ic_miss, proxy_set_with_receiver};
+pub(crate) use put_value::{
+    js_put_value_set_ic_miss, proxy_set_with_receiver, IC_SLOT_OVERFLOW_BIT,
+};
 mod json;
 mod metadata;
 mod own_keys;

crates/perry-runtime/src/proxy/put_value.rs

@@ -726,12 +726,24 @@ fn write_stub_insert(token: u64, key_bits: u64, slot: u32) {
     });
 }
 
-/// Validated fast store: the receiver must still be an ordinary,
-/// non-forwarded, unblocked, class-tagged heap object whose CURRENT shape
-/// token equals the cached one and whose inline region covers the slot.
-/// Every check reads the receiver's live state — the cached token/slot are
-/// never dereferenced — so a GC between prime and hit at worst causes a
-/// miss, never a wrong store.
+/// Slot-word bit marking a cached slot as living in the OVERFLOW store.
+///
+/// Set at prime time, where the full receiver validation runs and the
+/// inline-vs-overflow verdict is known. `object_kind`,
+/// `live_inline_slot_count` and `logical_key_count` are IMMUTABLE per shape
+/// id (the shape table only ever inserts descriptors; the sole in-place
+/// mutations are GC bookkeeping — the relocated `keys` address and the
+/// carrier liveness bits), so a hit whose token matches the receiver's
+/// CURRENT stamp has already proved everything the prime proved about kind
+/// and bounds. What remains mutable per object is the GC header — type,
+/// forwarded, and the blocking flags `Object.freeze`-family operations set —
+/// and the hit still checks those on every store.
+pub(crate) const IC_SLOT_OVERFLOW_BIT: u32 = 1 << 30;
+
+/// Validated fast store for a cached `(token, slot)` hit. Header checks and
+/// the token compare read the receiver's LIVE state; everything shape-derived
+/// was proved at prime time and is immutable per id (see
+/// [`IC_SLOT_OVERFLOW_BIT`]). A stale entry misses; it cannot store wrongly.
 #[inline]
 unsafe fn dyn_ic_try_store(target: f64, token: u64, slot: u32, value: f64) -> Option<f64> {
     let target_bits = target.to_bits();
@@ -753,41 +765,24 @@ unsafe fn dyn_ic_try_store(target: f64, token: u64, slot: u32, value: f64) -> Op
         return None;
     }
     let obj = obj_addr as *mut crate::ObjectHeader;
-    if !crate::object::object_is_regular(obj)
-        || !write_fast_path_receiver_kind_ok(obj, gc_header._reserved)
-    {
-        return None;
-    }
-    // ONE descriptor lookup, not two. `object_shape_id` runs a full lookup —
-    // and copies the whole `ShapeDescriptor` out of the table — purely to
-    // prove the stamped id is live, then throws the descriptor away; the bound
-    // check below then looked the SAME id up again. `shape_descriptor_by_id`
-    // was 10.5% of self time in a computed-key write loop, the single largest
-    // item, and half of that was this duplicate.
-    //
-    // Read the stamp straight off the header, reject on the token first (a
-    // load and a compare, so a wrong-shape receiver costs no table work at
-    // all), and let the one lookup that supplies the bound double as the
-    // liveness proof: a stamp with no live descriptor returns `None` here,
-    // exactly as `object_shape_id`'s 0 made the token compare fail before.
+    // Token compare against the receiver's CURRENT stamp. On a match, the
+    // shape id is the one every prime-time check ran under — receiver kind,
+    // class-object exclusion, and the slot's inline/overflow placement (now
+    // carried in the slot word) — so none of it is re-derived here. This
+    // took `write_fast_path_receiver_kind_ok` (6.8%), `shape_object_kind_by_id`
+    // (5.1%) and the descriptor bound fetch (4.4%) off every hit of the
+    // combined overwrite loop.
     let stamp = crate::object::shapes::object_shape_stamp(obj);
     if (crate::object::shapes::PIC_ID_TOKEN_BIT | stamp as u64) != token {
         return None;
     }
-    let shape = crate::object::shapes::shape_descriptor_by_id(stamp)?;
-    if slot >= shape.live_inline_slot_count {
-        // Overflow slot. The token compare above already proved the receiver
-        // is in the exact shape the (key → slot) pair was learned in, so the
-        // slot names this key's storage; it just lives in the spill store
-        // instead of the inline region. `overflow_set` is the same store the
-        // full walk bottoms out in (barriers, layout notes, remembered set),
-        // minus the walk. Wide objects — where EVERY data property is an
-        // overflow slot — are exactly the receivers the stub cache exists for.
-        if slot < shape.logical_key_count {
-            crate::object::overflow_set(obj_addr, slot as usize, value.to_bits());
-            return Some(value);
-        }
-        return None;
+    if slot & IC_SLOT_OVERFLOW_BIT != 0 {
+        crate::object::overflow_set(
+            obj_addr,
+            (slot & !IC_SLOT_OVERFLOW_BIT) as usize,
+            value.to_bits(),
+        );
+        return Some(value);
     }
     crate::object::store_object_field_slot(obj, slot as usize, value.to_bits());
     Some(value)
@@ -949,12 +944,20 @@ pub extern "C" fn js_put_value_set_dyn_ic_miss(
         // Feed the megamorphic stub BEFORE the inline-slot bail below: a wide
         // object keeps every data property in an overflow slot, so gating the
         // stub on the inline region would starve it for exactly the receivers
-        // it exists for. `dyn_ic_try_store` stores overflow slots via
-        // `overflow_set` under the same token validation.
+        // it exists for. The slot word carries the inline/overflow verdict
+        // (IC_SLOT_OVERFLOW_BIT), decided HERE where the descriptor is in
+        // hand, so the hit never re-fetches the bound — the verdict is a
+        // fact of the shape id the token pins.
+        let alloc_limit = shape.live_inline_slot_count;
         if let Some(kb) = stub_key_bits(key) {
-            write_stub_insert(shape_token, kb, idx);
+            if idx < alloc_limit {
+                write_stub_insert(shape_token, kb, idx);
+            } else if idx < shape.logical_key_count {
+                // Overflow: bound-checked against the key count at prime,
+                // which the hit-time token match preserves.
+                write_stub_insert(shape_token, kb, idx | IC_SLOT_OVERFLOW_BIT);
+            }
         }
-        let alloc_limit = shape.live_inline_slot_count;
         if idx >= alloc_limit {
             return result;
         }