diff --git a/changelog.d/9016-computed-read-by-value.md b/changelog.d/9016-computed-read-by-value.md deleted file mode 100644 index dc7655b15d..0000000000 --- a/changelog.d/9016-computed-read-by-value.md +++ /dev/null @@ -1,36 +0,0 @@ -Computed property reads take the key by value, and the combined -overwrite+read loop drops 38 → 30 ms (node: 26); the computed-key read loop -drops 21 → **13 ms against node's 24** — nearly twice as fast as node. - -The computed-read lowering called `js_get_string_pointer_unified` before the -by-name entry, because that entry's signature wants a `*const StringHeader`. -For an SSO key that means materialising inline bytes onto the heap — an intern -hash and table probe on every read — purely to satisfy a pointer signature. -`intern_dispatch_bytes` was 5.5% of the combined loop, essentially all of it -that. - -The new by-value entry (`js_typed_feedback_object_get_field_by_value_f64`) -takes the key NaN-boxed: - -- a heap-tagged key is unmasked and passed through — no work; -- an SSO key probes the megamorphic read stub on its CONTENT bits — a hit - never builds a `StringHeader` at all; -- a stub-missing SSO key takes a READ-ONLY intern probe (an intern hit cannot - allocate or move anything) and calls through with the canonical pointer — - still no rooting; -- only a key read before its first write materialises, with the receiver - rooted across the allocation (the hazard the old codegen comment worked - around by re-deriving its handle below the unbox — that workaround is gone - because the fast path no longer allocates). - -Everything downstream is unchanged: feedback recording, exotic receivers and -prototype resolution take exactly the previous path. - -Interleaved A/B, min-of-21 at quiet load (~1.1): combined overwrite 38 → -30 ms, computed-key read 21 → 13 ms, realistic-name read and populated delete -unchanged. - -Verification: suite 2790 passed, no warnings; adversarial property -differential, computed-key differential, and a targeted stale-slot -differential (192 reads across every rotation state of a delete/re-add cycle -with a stable keys-array address) all byte-identical to node. diff --git a/changelog.d/9022-ic-hit-immutable-facts.md b/changelog.d/9022-ic-hit-immutable-facts.md new file mode 100644 index 0000000000..c0a51ecfee --- /dev/null +++ b/changelog.d/9022-ic-hit-immutable-facts.md @@ -0,0 +1,32 @@ +IC hits stop re-deriving shape-immutable facts, and the combined +overwrite+read loop goes past node: 31 → **24 ms against node's 29**. The +write-only loop drops to **15 ms against node's 23**. + +Every stub or way hit re-proved the receiver's kind (a registry probe via +`is_class_object_ptr`), the plain-ordinary verdict, and the inline slot bound +(a descriptor fetch) — together 16% of the combined loop +(`write_fast_path_receiver_kind_ok` 6.8%, `shape_object_kind_by_id` 5.1%, +`shape_live_inline_slot_count_by_id` 4.4%). + +All of those are facts of the SHAPE ID. The shape table only ever inserts +descriptors — its sole in-place mutations are GC bookkeeping (the relocated +`keys` address and the carrier liveness bits) — so `object_kind`, +`live_inline_slot_count` and `logical_key_count` cannot change under a fixed +id. A hit whose token matches the receiver's CURRENT stamp has therefore +already proved everything prime time proved about kind and bounds. + +The slot word now carries the one prime-time verdict a hit needs: +`IC_SLOT_OVERFLOW_BIT`, choosing the inline region vs the spill store. +Overflow entries are bound-checked against `logical_key_count` at prime. Hits +keep exactly the checks that ARE mutable per object — header type, forwarded, +and the blocking flags the `Object.freeze` family sets — plus the token +compare. Applied to the write stub, the per-site dyn ways, and both read-stub +hit sites. + +Interleaved A/B, min-of-21 at quiet load (~0.8): combined overwrite 31 → +24 ms; computed-key read, realistic-name read and populated delete unchanged; +write-only measured on the branch at 15 ms (node 23). + +Verification: suite 2790 passed, all 60 lint gates pass, no warnings; the +adversarial, computed-key, and stale-slot differentials all byte-identical to +node. diff --git a/crates/perry-runtime/src/object/field_get_set/get_field_by_name.rs b/crates/perry-runtime/src/object/field_get_set/get_field_by_name.rs index e8b49bb5a9..2996423e4a 100644 --- a/crates/perry-runtime/src/object/field_get_set/get_field_by_name.rs +++ b/crates/perry-runtime/src/object/field_get_set/get_field_by_name.rs @@ -142,15 +142,12 @@ pub extern "C" fn js_object_get_field_by_name( if let Some(slot) = super::super::read_stub::read_stub_probe(token, key_bits) { - let live = crate::object::object_live_slot_count(o); - let limit = - std::cmp::max(live, crate::object::INLINE_SLOT_FLOOR as u32); - if slot < limit { - return super::accessors::js_object_get_field(o, slot); - } - if let Some(bits) = super::super::overflow_get(addr, slot as usize) + // The slot word carries the inline/overflow + // verdict from prime time; no bound fetch. + if let Some(v) = + super::super::read_stub::read_slot_by_tag(o, addr, slot) { - return JSValue::from_bits(bits); + return JSValue::from_bits(v.to_bits()); } } } @@ -228,7 +225,7 @@ pub extern "C" fn js_object_get_field_by_name( keys as usize, key as usize, ) { - prime_read_stub(o, key, idx); + prime_read_stub(o, key, idx, (idx as usize) < alloc_limit); return if (idx as usize) < alloc_limit { super::accessors::js_object_get_field(o, idx) } else { @@ -264,7 +261,7 @@ pub extern "C" fn js_object_get_field_by_name( key, ) { let i = i as usize; - prime_read_stub(o, key, i as u32); + prime_read_stub(o, key, i as u32, i < alloc_limit); super::super::prop_plan::read_plan_record( keys as usize, key as usize, @@ -1758,7 +1755,17 @@ mod null_key_guard_5972 { /// semantics. Keys that cannot be represented as content bits are skipped by /// `read_stub_key_bits`. #[inline] -fn prime_read_stub(obj: *const ObjectHeader, key: *const crate::StringHeader, slot: u32) { +fn prime_read_stub( + obj: *const ObjectHeader, + key: *const crate::StringHeader, + slot: u32, + inline: bool, +) { + let slot = if inline { + slot + } else { + slot | crate::proxy::IC_SLOT_OVERFLOW_BIT + }; if let Some(key_bits) = super::super::read_stub::read_stub_key_bits(key) { if let Some(token) = unsafe { super::super::read_stub::receiver_shape_token(obj) } { super::super::read_stub::read_stub_insert(token, key_bits, slot); diff --git a/crates/perry-runtime/src/object/read_stub.rs b/crates/perry-runtime/src/object/read_stub.rs index c74ad052b0..5b7b271911 100644 --- a/crates/perry-runtime/src/object/read_stub.rs +++ b/crates/perry-runtime/src/object/read_stub.rs @@ -159,14 +159,32 @@ pub(crate) unsafe fn try_read_by_content_bits( } let token = receiver_shape_token(obj)?; let slot = read_stub_probe(token, key_bits)?; - let limit = std::cmp::max( - crate::object::object_live_slot_count(obj), - crate::object::INLINE_SLOT_FLOOR as u32, - ); - if slot < limit { - return Some(f64::from_bits( - crate::object::js_object_get_field(obj, slot).bits(), - )); + read_slot_by_tag(obj, addr, slot) +} + +/// Read the value a bit-tagged cached slot names. The inline/overflow verdict +/// was decided at prime time (`IC_SLOT_OVERFLOW_BIT`, see `proxy::put_value`) +/// under the exact shape id the caller's token match just re-proved, so no +/// bound is fetched here — the shape table only ever inserts descriptors, so +/// the verdict cannot have changed under the same id. +#[inline(always)] +pub(crate) unsafe fn read_slot_by_tag( + obj: *const ObjectHeader, + addr: usize, + slot: u32, +) -> Option { + use crate::proxy::IC_SLOT_OVERFLOW_BIT; + if slot & IC_SLOT_OVERFLOW_BIT != 0 { + return crate::object::overflow_get(addr, (slot & !IC_SLOT_OVERFLOW_BIT) as usize) + .map(f64::from_bits); + } + let fields_ptr = + (obj as *const u8).add(std::mem::size_of::()) as *const crate::JSValue; + let val = *fields_ptr.add(slot as usize); + // Same null-POINTER_TAG guard as `js_object_get_field`'s inline half: the + // pattern is never a legitimate stored value. + if val.bits() == 0x7FFD_0000_0000_0000 { + return Some(f64::from_bits(crate::value::TAG_UNDEFINED)); } - crate::object::overflow_get(addr, slot as usize).map(f64::from_bits) + Some(f64::from_bits(val.bits())) } diff --git a/crates/perry-runtime/src/proxy.rs b/crates/perry-runtime/src/proxy.rs index db09ed5520..0f601bbdf9 100644 --- a/crates/perry-runtime/src/proxy.rs +++ b/crates/perry-runtime/src/proxy.rs @@ -31,7 +31,9 @@ pub use has_delete::{js_proxy_delete, js_proxy_has}; mod invariants; mod put_value; pub use put_value::{js_proxy_set, js_put_value_set}; -pub(crate) use put_value::{js_put_value_set_ic_miss, proxy_set_with_receiver}; +pub(crate) use put_value::{ + js_put_value_set_ic_miss, proxy_set_with_receiver, IC_SLOT_OVERFLOW_BIT, +}; mod json; mod metadata; mod own_keys; diff --git a/crates/perry-runtime/src/proxy/put_value.rs b/crates/perry-runtime/src/proxy/put_value.rs index 7268ddf41d..52a3bb5772 100644 --- a/crates/perry-runtime/src/proxy/put_value.rs +++ b/crates/perry-runtime/src/proxy/put_value.rs @@ -726,12 +726,24 @@ fn write_stub_insert(token: u64, key_bits: u64, slot: u32) { }); } -/// Validated fast store: the receiver must still be an ordinary, -/// non-forwarded, unblocked, class-tagged heap object whose CURRENT shape -/// token equals the cached one and whose inline region covers the slot. -/// Every check reads the receiver's live state — the cached token/slot are -/// never dereferenced — so a GC between prime and hit at worst causes a -/// miss, never a wrong store. +/// Slot-word bit marking a cached slot as living in the OVERFLOW store. +/// +/// Set at prime time, where the full receiver validation runs and the +/// inline-vs-overflow verdict is known. `object_kind`, +/// `live_inline_slot_count` and `logical_key_count` are IMMUTABLE per shape +/// id (the shape table only ever inserts descriptors; the sole in-place +/// mutations are GC bookkeeping — the relocated `keys` address and the +/// carrier liveness bits), so a hit whose token matches the receiver's +/// CURRENT stamp has already proved everything the prime proved about kind +/// and bounds. What remains mutable per object is the GC header — type, +/// forwarded, and the blocking flags `Object.freeze`-family operations set — +/// and the hit still checks those on every store. +pub(crate) const IC_SLOT_OVERFLOW_BIT: u32 = 1 << 30; + +/// Validated fast store for a cached `(token, slot)` hit. Header checks and +/// the token compare read the receiver's LIVE state; everything shape-derived +/// was proved at prime time and is immutable per id (see +/// [`IC_SLOT_OVERFLOW_BIT`]). A stale entry misses; it cannot store wrongly. #[inline] unsafe fn dyn_ic_try_store(target: f64, token: u64, slot: u32, value: f64) -> Option { let target_bits = target.to_bits(); @@ -753,41 +765,24 @@ unsafe fn dyn_ic_try_store(target: f64, token: u64, slot: u32, value: f64) -> Op return None; } let obj = obj_addr as *mut crate::ObjectHeader; - if !crate::object::object_is_regular(obj) - || !write_fast_path_receiver_kind_ok(obj, gc_header._reserved) - { - return None; - } - // ONE descriptor lookup, not two. `object_shape_id` runs a full lookup — - // and copies the whole `ShapeDescriptor` out of the table — purely to - // prove the stamped id is live, then throws the descriptor away; the bound - // check below then looked the SAME id up again. `shape_descriptor_by_id` - // was 10.5% of self time in a computed-key write loop, the single largest - // item, and half of that was this duplicate. - // - // Read the stamp straight off the header, reject on the token first (a - // load and a compare, so a wrong-shape receiver costs no table work at - // all), and let the one lookup that supplies the bound double as the - // liveness proof: a stamp with no live descriptor returns `None` here, - // exactly as `object_shape_id`'s 0 made the token compare fail before. + // Token compare against the receiver's CURRENT stamp. On a match, the + // shape id is the one every prime-time check ran under — receiver kind, + // class-object exclusion, and the slot's inline/overflow placement (now + // carried in the slot word) — so none of it is re-derived here. This + // took `write_fast_path_receiver_kind_ok` (6.8%), `shape_object_kind_by_id` + // (5.1%) and the descriptor bound fetch (4.4%) off every hit of the + // combined overwrite loop. let stamp = crate::object::shapes::object_shape_stamp(obj); if (crate::object::shapes::PIC_ID_TOKEN_BIT | stamp as u64) != token { return None; } - let shape = crate::object::shapes::shape_descriptor_by_id(stamp)?; - if slot >= shape.live_inline_slot_count { - // Overflow slot. The token compare above already proved the receiver - // is in the exact shape the (key → slot) pair was learned in, so the - // slot names this key's storage; it just lives in the spill store - // instead of the inline region. `overflow_set` is the same store the - // full walk bottoms out in (barriers, layout notes, remembered set), - // minus the walk. Wide objects — where EVERY data property is an - // overflow slot — are exactly the receivers the stub cache exists for. - if slot < shape.logical_key_count { - crate::object::overflow_set(obj_addr, slot as usize, value.to_bits()); - return Some(value); - } - return None; + if slot & IC_SLOT_OVERFLOW_BIT != 0 { + crate::object::overflow_set( + obj_addr, + (slot & !IC_SLOT_OVERFLOW_BIT) as usize, + value.to_bits(), + ); + return Some(value); } crate::object::store_object_field_slot(obj, slot as usize, value.to_bits()); Some(value) @@ -949,12 +944,20 @@ pub extern "C" fn js_put_value_set_dyn_ic_miss( // Feed the megamorphic stub BEFORE the inline-slot bail below: a wide // object keeps every data property in an overflow slot, so gating the // stub on the inline region would starve it for exactly the receivers - // it exists for. `dyn_ic_try_store` stores overflow slots via - // `overflow_set` under the same token validation. + // it exists for. The slot word carries the inline/overflow verdict + // (IC_SLOT_OVERFLOW_BIT), decided HERE where the descriptor is in + // hand, so the hit never re-fetches the bound — the verdict is a + // fact of the shape id the token pins. + let alloc_limit = shape.live_inline_slot_count; if let Some(kb) = stub_key_bits(key) { - write_stub_insert(shape_token, kb, idx); + if idx < alloc_limit { + write_stub_insert(shape_token, kb, idx); + } else if idx < shape.logical_key_count { + // Overflow: bound-checked against the key count at prime, + // which the hit-time token match preserves. + write_stub_insert(shape_token, kb, idx | IC_SLOT_OVERFLOW_BIT); + } } - let alloc_limit = shape.live_inline_slot_count; if idx >= alloc_limit { return result; }