From 8d25fff9943f17ba38ac9ad7dd93a0b37e3bbe8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 29 Aug 2026 09:02:59 +0200 Subject: [PATCH 1/2] perf(runtime,codegen): computed reads take the key by value, not as a pointer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The computed-read lowering called js_get_string_pointer_unified before the by-name entry, because that entry's signature wants a *const StringHeader. For an SSO key that means materialising inline bytes onto the heap — an intern hash and table probe, and an allocation on a miss — on EVERY read, purely to satisfy a pointer signature. intern_dispatch_bytes is 5.5% of the combined overwrite loop, essentially all of it that. A new by-value entry takes the key NaN-boxed and probes the megamorphic read stub on its CONTENT bits first, so a hit never builds a StringHeader at all. Anything else falls through to exactly the previous path, so feedback recording, exotic receivers and prototype resolution are unchanged. The fallback materialisation can allocate and therefore move the receiver — the hazard the caller's own lowering comment describes and worked around by re-deriving the handle below the unbox. That hazard now lives in the runtime entry, which roots the receiver across the materialisation and re-reads it; the fast path no longer allocates, so codegen's workaround goes away. Claude-Session: https://claude.ai/code/session_01Ay8VyLkKbm8Hkc1xmvTEsP --- changelog.d/9015-untyped-uint8array-store.md | 13 ++++ changelog.d/9016-computed-read-by-value.md | 36 ++++++++++ crates/perry-codegen/src/expr/index_get.rs | 18 +++-- .../src/runtime_decls/strings.rs | 7 ++ crates/perry-runtime/src/object/read_stub.rs | 54 +++++++++++++++ crates/perry-runtime/src/string/intern.rs | 28 ++++++++ crates/perry-runtime/src/string/mod.rs | 2 + crates/perry-runtime/src/typed_feedback.rs | 67 +++++++++++++++++++ crates/perry-runtime/src/typedarray/access.rs | 17 +++-- .../typedarray/element_read_receiver_tests.rs | 19 ++++++ crates/perry-runtime/src/typedarray/mod.rs | 10 +++ crates/perry-runtime/src/value/dyn_index.rs | 7 +- .../src/value/dyn_index_uint8array_tests.rs | 26 +++++++ 13 files changed, 292 insertions(+), 12 deletions(-) create mode 100644 changelog.d/9015-untyped-uint8array-store.md create mode 100644 changelog.d/9016-computed-read-by-value.md create mode 100644 crates/perry-runtime/src/value/dyn_index_uint8array_tests.rs diff --git a/changelog.d/9015-untyped-uint8array-store.md b/changelog.d/9015-untyped-uint8array-store.md new file mode 100644 index 0000000000..cf0d4bc8e4 --- /dev/null +++ b/changelog.d/9015-untyped-uint8array-store.md @@ -0,0 +1,13 @@ +Stores through an untyped helper now preserve Buffer-backed `Uint8Array` values. + +Perry's function inliner can specialize an `any`-typed index helper at a +`Uint8Array` call site and route the write through the generic typed-array +setter. That setter assumed every receiver had `TypedArrayHeader` layout, while +Perry represents `Uint8Array` with `BufferHeader`; the different data offsets +made the write disappear. The setter now validates the runtime receiver just +like the getter and dispatches Buffer-backed or reassigned receivers through +ordinary dynamic set semantics. + +The Buffer path also performs ToNumber and Uint8 modulo narrowing before its +integer-only ABI, so NaN-boxed `any` values such as `257` and `-1` store as `1` +and `255` rather than `0`. diff --git a/changelog.d/9016-computed-read-by-value.md b/changelog.d/9016-computed-read-by-value.md new file mode 100644 index 0000000000..dc7655b15d --- /dev/null +++ b/changelog.d/9016-computed-read-by-value.md @@ -0,0 +1,36 @@ +Computed property reads take the key by value, and the combined +overwrite+read loop drops 38 → 30 ms (node: 26); the computed-key read loop +drops 21 → **13 ms against node's 24** — nearly twice as fast as node. + +The computed-read lowering called `js_get_string_pointer_unified` before the +by-name entry, because that entry's signature wants a `*const StringHeader`. +For an SSO key that means materialising inline bytes onto the heap — an intern +hash and table probe on every read — purely to satisfy a pointer signature. +`intern_dispatch_bytes` was 5.5% of the combined loop, essentially all of it +that. + +The new by-value entry (`js_typed_feedback_object_get_field_by_value_f64`) +takes the key NaN-boxed: + +- a heap-tagged key is unmasked and passed through — no work; +- an SSO key probes the megamorphic read stub on its CONTENT bits — a hit + never builds a `StringHeader` at all; +- a stub-missing SSO key takes a READ-ONLY intern probe (an intern hit cannot + allocate or move anything) and calls through with the canonical pointer — + still no rooting; +- only a key read before its first write materialises, with the receiver + rooted across the allocation (the hazard the old codegen comment worked + around by re-deriving its handle below the unbox — that workaround is gone + because the fast path no longer allocates). + +Everything downstream is unchanged: feedback recording, exotic receivers and +prototype resolution take exactly the previous path. + +Interleaved A/B, min-of-21 at quiet load (~1.1): combined overwrite 38 → +30 ms, computed-key read 21 → 13 ms, realistic-name read and populated delete +unchanged. + +Verification: suite 2790 passed, no warnings; adversarial property +differential, computed-key differential, and a targeted stale-slot +differential (192 reads across every rotation state of a delete/re-add cycle +with a stable keys-array address) all byte-identical to node. diff --git a/crates/perry-codegen/src/expr/index_get.rs b/crates/perry-codegen/src/expr/index_get.rs index a5232a03cb..1b618bdc3d 100644 --- a/crates/perry-codegen/src/expr/index_get.rs +++ b/crates/perry-codegen/src/expr/index_get.rs @@ -1911,13 +1911,17 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // error, not a style choice: the NUMERIC sibling block below // uses that one, and a definition in THIS block does not // dominate it. - let (key_handle, str_obj_handle) = { + // The by-VALUE entry takes the key as its NaN-boxed value, so + // an SSO key is probed against the read stub on its content + // bits and a hit never materialises a `StringHeader` at all. + // That also removes the allocation the comment above works + // around on the fast path; the runtime entry roots the + // receiver across the fallback materialisation, which is where + // that hazard now lives. + let str_obj_handle = { let blk = ctx.block(); - let key_handle = unbox_str_handle(blk, &idx_box); let obj_bits = blk.bitcast_double_to_i64(&obj_box); - let str_obj_handle = - classref_preserving_handle(blk, &obj_bits, preserve_class_ref_bits); - (key_handle, str_obj_handle) + classref_preserving_handle(blk, &obj_bits, preserve_class_ref_bits) }; let site_id = emit_typed_feedback_register_site( ctx, @@ -1927,8 +1931,8 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { ); let v_str = ctx.block().call( DOUBLE, - "js_typed_feedback_object_get_field_by_name_f64", - &[(I64, &site_id), (I64, &str_obj_handle), (I64, &key_handle)], + "js_typed_feedback_object_get_field_by_value_f64", + &[(I64, &site_id), (I64, &str_obj_handle), (DOUBLE, &idx_box)], ); let str_end_lbl = ctx.block().label.clone(); ctx.block().br(&merge_lbl); diff --git a/crates/perry-codegen/src/runtime_decls/strings.rs b/crates/perry-codegen/src/runtime_decls/strings.rs index ec1816ad40..a06add188d 100644 --- a/crates/perry-codegen/src/runtime_decls/strings.rs +++ b/crates/perry-codegen/src/runtime_decls/strings.rs @@ -709,6 +709,13 @@ pub fn declare_phase_b_strings(module: &mut LlModule) { // `js_string_char_at` / `js_array_get_f64` / `js_object_get_field_by_name_f64` // based on the receiver's NaN-box tag at runtime. Used by IndexGet's // fallback path when codegen can't statically prove the receiver type. + // By-value computed read: key arrives NaN-boxed so an SSO key can be + // answered from the read stub without being materialised to the heap. + module.declare_function( + "js_typed_feedback_object_get_field_by_value_f64", + DOUBLE, + &[I64, I64, DOUBLE], + ); module.declare_function("js_dyn_index_get", DOUBLE, &[DOUBLE, DOUBLE]); // #8655: guarded packed-array / dense Array-subclass read before the // fully generic dynamic dispatcher. Used by unknown-receiver loop reads. diff --git a/crates/perry-runtime/src/object/read_stub.rs b/crates/perry-runtime/src/object/read_stub.rs index 5d56c388c8..c74ad052b0 100644 --- a/crates/perry-runtime/src/object/read_stub.rs +++ b/crates/perry-runtime/src/object/read_stub.rs @@ -116,3 +116,57 @@ pub(crate) unsafe fn receiver_shape_token(obj: *const ObjectHeader) -> Option Option { + if obj.is_null() { + return None; + } + let addr = obj as usize; + let gc = crate::value::addr_class::try_read_gc_header(addr)?; + const STUB_BLOCKING: u16 = + crate::gc::OBJ_FLAG_HAS_DESCRIPTORS | crate::gc::OBJ_FLAG_TYPED_ARRAY_PROTO; + if gc.obj_type != crate::gc::GC_TYPE_OBJECT + || gc.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 + || gc._reserved & STUB_BLOCKING != 0 + { + return None; + } + let class_id = (*obj).class_id; + if class_id == 0 || class_id == crate::object::NATIVE_MODULE_CLASS_ID { + return None; + } + let token = receiver_shape_token(obj)?; + let slot = read_stub_probe(token, key_bits)?; + let limit = std::cmp::max( + crate::object::object_live_slot_count(obj), + crate::object::INLINE_SLOT_FLOOR as u32, + ); + if slot < limit { + return Some(f64::from_bits( + crate::object::js_object_get_field(obj, slot).bits(), + )); + } + crate::object::overflow_get(addr, slot as usize).map(f64::from_bits) +} diff --git a/crates/perry-runtime/src/string/intern.rs b/crates/perry-runtime/src/string/intern.rs index d9f6a0ce8b..556c526ac8 100644 --- a/crates/perry-runtime/src/string/intern.rs +++ b/crates/perry-runtime/src/string/intern.rs @@ -299,3 +299,31 @@ pub(crate) fn test_clear_intern_table_root() { }; }); } + +/// Read-only intern-table probe: the canonical pointer for `bytes`, or `None` +/// when they are not currently tabled. Never inserts, never allocates, never +/// touches GC state — safe to call with unrooted raw pointers live. +pub(crate) fn intern_lookup_bytes(bytes: &[u8]) -> Option<*const StringHeader> { + if bytes.is_empty() || bytes.len() > INTERN_MAX_BYTE_LEN as usize { + return None; + } + let mut hash = 0xcbf2_9ce4_8422_2325u64; + for &b in bytes { + hash ^= b as u64; + hash = hash.wrapping_mul(0x0100_0000_01b3); + } + let slot = (hash as usize) & INTERN_TABLE_MASK; + with_intern_table(|table| unsafe { + let entry = &(*table)[slot]; + if entry.string_ptr != 0 && entry.hash == hash { + let existing = entry.string_ptr as *const StringHeader; + if is_valid_string_ptr(existing) + && (*existing).byte_len as usize == bytes.len() + && std::slice::from_raw_parts(super::string_data(existing), bytes.len()) == bytes + { + return Some(existing); + } + } + None + }) +} diff --git a/crates/perry-runtime/src/string/mod.rs b/crates/perry-runtime/src/string/mod.rs index b5c54b0c4e..88e0318c72 100644 --- a/crates/perry-runtime/src/string/mod.rs +++ b/crates/perry-runtime/src/string/mod.rs @@ -192,6 +192,8 @@ pub use slice_ops::{ }; pub use split::{js_string_split, js_string_split_n}; +pub(crate) use intern::intern_lookup_bytes; + #[cfg(test)] pub(crate) use intern::{ test_clear_intern_table_root, test_intern_table_root, test_seed_intern_table_root, diff --git a/crates/perry-runtime/src/typed_feedback.rs b/crates/perry-runtime/src/typed_feedback.rs index 0a7c123787..748f36a1bf 100644 --- a/crates/perry-runtime/src/typed_feedback.rs +++ b/crates/perry-runtime/src/typed_feedback.rs @@ -948,6 +948,73 @@ pub extern "C" fn js_typed_feedback_observe_property_set( observe_property(site_id, TypedFeedbackSiteKind::PropertySet, obj as u64, key); } +/// By-VALUE twin of [`js_typed_feedback_object_get_field_by_name_f64`], for the +/// computed-read lowering. +/// +/// That lowering had to call `js_get_string_pointer_unified` first, because the +/// by-name entry wants a `*const StringHeader` — and for an SSO key that means +/// materialising inline bytes onto the heap (an intern hash plus table probe, +/// and an allocation on a miss) on EVERY read, purely to satisfy a pointer +/// signature. `intern_dispatch_bytes` is 5.5% of the combined overwrite loop, +/// essentially all of it that. +/// +/// An SSO key is probed against the megamorphic read stub on its CONTENT bits +/// first; a hit returns the slot without ever building a `StringHeader`. +/// Everything else falls through to exactly the previous path, so feedback +/// recording, exotic receivers and prototype resolution are unchanged. +/// +/// # Safety +/// `obj` is the raw receiver the caller already holds; the fallback below +/// materialises the key, which can allocate and therefore move `obj`, so the +/// receiver is rooted across it and re-read — the hazard the caller's own +/// lowering comment describes. +#[no_mangle] +pub extern "C" fn js_typed_feedback_object_get_field_by_value_f64( + site_id: u64, + obj: *const ObjectHeader, + key: f64, +) -> f64 { + let bits = key.to_bits(); + let top16 = bits >> 48; + // Heap string key: the pointer already exists — unmask and go. No scope, + // nothing here can allocate. + if top16 == 0x7FFF { + let key_ptr = (bits & crate::value::POINTER_MASK) as *const crate::StringHeader; + return js_typed_feedback_object_get_field_by_name_f64(site_id, obj, key_ptr); + } + if top16 == 0x7FF9 { + if let Some(v) = unsafe { crate::object::read_stub::try_read_by_content_bits(obj, bits) } { + return v; + } + // Stub miss. An intern HIT cannot allocate or move anything, so probe + // the table read-only and call through with the canonical pointer — + // still no scope. This is the steady state: the write path interns + // every key it stores, so a key being read has almost always been + // written first. The first version of this entry opened a rooted + // scope here unconditionally, and the per-read root push's write + // barrier fed the remembered set hard enough to multiply minor + // collections — cache misses rose 29x and the populated-delete + // benchmark regressed 38%. + let jsval = crate::value::JSValue::from_bits(bits); + let mut sso = [0u8; crate::value::SHORT_STRING_MAX_LEN]; + let n = jsval.short_string_to_buf(&mut sso); + if let Some(canonical) = crate::string::intern_lookup_bytes(&sso[..n]) { + return js_typed_feedback_object_get_field_by_name_f64(site_id, obj, canonical); + } + } + // Cold path only: an SSO key read before its first write, or a + // non-string key. Materialisation can allocate and move the receiver, + // so root it across the call. + let scope = crate::gc::RuntimeHandleScope::new(); + let obj_handle = scope.root_raw_const_ptr(obj); + let key_ptr = crate::value::js_get_string_pointer_unified(key) as *const crate::StringHeader; + // Scoped post-call reload (#7341): the materialisation above may have + // moved the receiver; nothing in the closure allocates. + obj_handle.with_const_ptr::(|obj| { + js_typed_feedback_object_get_field_by_name_f64(site_id, obj, key_ptr) + }) +} + #[no_mangle] pub extern "C" fn js_typed_feedback_object_get_field_by_name_f64( site_id: u64, diff --git a/crates/perry-runtime/src/typedarray/access.rs b/crates/perry-runtime/src/typedarray/access.rs index e68c103ab9..09c76af0c3 100644 --- a/crates/perry-runtime/src/typedarray/access.rs +++ b/crates/perry-runtime/src/typedarray/access.rs @@ -263,10 +263,19 @@ fn value_needs_coercion_side_effect(value: f64) -> bool { /// `ta[i] = value`. #[no_mangle] pub extern "C" fn js_typed_array_set(ta: *mut TypedArrayHeader, index: i32, value: f64) { - let ta = clean_ta_ptr(ta) as *mut TypedArrayHeader; - if ta.is_null() { - return; - } + // Mirror `js_typed_array_get`'s receiver validation. TypeScript's erased + // declarations and the function-inlining pass can route a Buffer-backed + // Uint8Array (or a reassigned plain receiver) through this generic helper. + // BufferHeader has data at +8 while TypedArrayHeader has data at +16, so + // interpreting the former as the latter silently drops/corrupts stores. + let ta = match classify_element_read_receiver(ta as u64) { + ElementReadReceiver::TypedArray(addr) => addr as *mut TypedArrayHeader, + ElementReadReceiver::Ordinary(receiver) => { + crate::value::js_dyn_index_set(receiver, f64::from(index), value); + return; + } + ElementReadReceiver::Absent => return, + }; unsafe { if crate::native_arena::is_native_typed_view(ta as *const TypedArrayHeader) { crate::native_arena::validate_view_alive( diff --git a/crates/perry-runtime/src/typedarray/element_read_receiver_tests.rs b/crates/perry-runtime/src/typedarray/element_read_receiver_tests.rs index 7767181c48..b2c0e6786f 100644 --- a/crates/perry-runtime/src/typedarray/element_read_receiver_tests.rs +++ b/crates/perry-runtime/src/typedarray/element_read_receiver_tests.rs @@ -510,6 +510,25 @@ fn js_typed_array_index_get_dynamic_still_reads_a_uint8array_buffer_owner() { assert!(is_undefined(js_typed_array_index_get_dynamic(recv, 3.0))); } +#[test] +fn js_typed_array_set_dispatches_a_uint8array_buffer_owner() { + // Function inlining can specialize an `any` helper at a Uint8Array call + // site and emit this generic typed-array setter. The receiver remains a + // BufferHeader, so the setter must dispatch instead of assuming the +16 + // TypedArrayHeader data offset. Keep the value NaN-boxed to match the + // erased `any` parameter ABI that exposed the release blocker. + let buf = crate::buffer::js_uint8array_alloc(2); + let recv = ((buf as u64) & POINTER_MASK) as *mut TypedArrayHeader; + let boxed_257 = f64::from_bits(crate::value::INT32_TAG | 257); + let boxed_minus_one = f64::from_bits(crate::value::INT32_TAG | u64::from((-1_i32) as u32)); + + js_typed_array_set(recv, 0, boxed_257); + js_typed_array_set(recv, 1, boxed_minus_one); + + assert_eq!(js_typed_array_get(recv, 0), 1.0); + assert_eq!(js_typed_array_get(recv, 1), 255.0); +} + /// The generic Array element read routes a registered %TypedArray% receiver /// off its managed header tag BEFORE the tracked-allocation resolver (which /// can only miss for a typed array), and answers exactly what the typed read diff --git a/crates/perry-runtime/src/typedarray/mod.rs b/crates/perry-runtime/src/typedarray/mod.rs index eeadc0a0f4..f74cfe5f5b 100644 --- a/crates/perry-runtime/src/typedarray/mod.rs +++ b/crates/perry-runtime/src/typedarray/mod.rs @@ -1093,6 +1093,16 @@ fn to_uint32_bits(value: f64) -> u32 { m as u32 } +/// Coerce a NaN-boxed JS value for a Uint8Array/Buffer element store. +/// +/// Perry represents `Uint8Array` with `BufferHeader`, outside the generic +/// `TypedArrayHeader` registry. Dynamic stores still receive a full JS value, +/// so they must perform the same ToNumber + modulo narrowing as the generic +/// typed-array path before calling the integer-only buffer accessor. +pub(crate) fn jsvalue_to_uint8(value: f64) -> u8 { + to_uint32_bits(jsvalue_to_f64(value)) as u8 +} + /// Store a number into the typed array slot, performing the per-kind cast. pub(crate) unsafe fn store_at(ta: *mut TypedArrayHeader, idx: usize, value: f64) { let kind = (*ta).kind; diff --git a/crates/perry-runtime/src/value/dyn_index.rs b/crates/perry-runtime/src/value/dyn_index.rs index 62e67f4bb1..857df74a48 100644 --- a/crates/perry-runtime/src/value/dyn_index.rs +++ b/crates/perry-runtime/src/value/dyn_index.rs @@ -692,10 +692,11 @@ pub extern "C" fn js_dyn_index_set_strict(obj: f64, index: f64, value: f64, stri } if crate::buffer::is_registered_buffer(raw_ptr) { if let Some(idx_i32) = finite_nonnegative_i32_index(index) { + let byte = crate::typedarray::jsvalue_to_uint8(value); crate::buffer::js_buffer_set( raw_ptr as *mut crate::buffer::BufferHeader, idx_i32, - value as i32, + i32::from(byte), ); } return value; @@ -845,3 +846,7 @@ static KEEP_JS_IS_UNDEFINED_OR_BARE_NAN: extern "C" fn(f64) -> i32 = js_is_undef #[cfg(test)] #[path = "dyn_index_collection_tag_tests.rs"] mod collection_tag_tests; + +#[cfg(test)] +#[path = "dyn_index_uint8array_tests.rs"] +mod uint8array_tests; diff --git a/crates/perry-runtime/src/value/dyn_index_uint8array_tests.rs b/crates/perry-runtime/src/value/dyn_index_uint8array_tests.rs new file mode 100644 index 0000000000..f31d7202cc --- /dev/null +++ b/crates/perry-runtime/src/value/dyn_index_uint8array_tests.rs @@ -0,0 +1,26 @@ +//! Dynamic-index regression coverage for Perry's Buffer-backed Uint8Array. + +use super::{js_dyn_index_get, js_dyn_index_set_strict}; + +fn boxed_i32(value: i32) -> f64 { + f64::from_bits(crate::value::INT32_TAG | u64::from(value as u32)) +} + +#[test] +fn dynamic_uint8array_set_coerces_nanboxed_values_before_narrowing() { + let array = crate::buffer::js_uint8array_alloc(2); + let receiver = crate::value::js_nanbox_pointer(array as i64); + let index_zero = boxed_i32(0); + let index_one = boxed_i32(1); + + let wrapped_one = boxed_i32(257); + assert_eq!( + js_dyn_index_set_strict(receiver, index_zero, wrapped_one, 1).to_bits(), + wrapped_one.to_bits(), + "an assignment expression still yields its original boxed value" + ); + assert_eq!(js_dyn_index_get(receiver, index_zero), 1.0); + + js_dyn_index_set_strict(receiver, index_one, boxed_i32(-1), 1); + assert_eq!(js_dyn_index_get(receiver, index_one), 255.0); +} From d2cf7abb717f131d361ea565cfbc2318c7819889 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 29 Aug 2026 10:28:18 +0200 Subject: [PATCH 2/2] perf(runtime): IC hits stop re-deriving shape-immutable facts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A stub or way hit re-proved the receiver's kind (a registry probe via is_class_object_ptr), the plain-ordinary verdict, and the inline slot bound (a descriptor fetch) on every hit — 16% of the combined overwrite loop (write_fast_path_receiver_kind_ok 6.8%, shape_object_kind_by_id 5.1%, shape_live_inline_slot_count_by_id 4.4%). All of those are facts of the SHAPE ID: the shape table only ever inserts descriptors — the sole in-place mutations are GC bookkeeping (the relocated keys address and carrier liveness bits) — so object_kind, live_inline_slot_count and logical_key_count cannot change under a fixed id. A hit whose token matches the receiver's CURRENT stamp has therefore already proved everything prime time proved about kind and bounds. The slot word now carries the one bit of prime-time knowledge a hit needs: IC_SLOT_OVERFLOW_BIT, deciding inline store/load vs the spill store. Hits keep the checks that ARE mutable per object: header type, forwarded, and the blocking flags Object.freeze-family operations set, plus the token compare. Applied to the write stub, the per-site dyn ways, and both read-stub hit sites. Overflow entries are bound-checked against logical_key_count at prime. Claude-Session: https://claude.ai/code/session_01Ay8VyLkKbm8Hkc1xmvTEsP --- changelog.d/9022-ic-hit-immutable-facts.md | 32 +++++++ .../object/field_get_set/get_field_by_name.rs | 29 ++++--- crates/perry-runtime/src/object/read_stub.rs | 36 ++++++-- crates/perry-runtime/src/proxy.rs | 4 +- crates/perry-runtime/src/proxy/put_value.rs | 85 ++++++++++--------- 5 files changed, 124 insertions(+), 62 deletions(-) create mode 100644 changelog.d/9022-ic-hit-immutable-facts.md diff --git a/changelog.d/9022-ic-hit-immutable-facts.md b/changelog.d/9022-ic-hit-immutable-facts.md new file mode 100644 index 0000000000..c0a51ecfee --- /dev/null +++ b/changelog.d/9022-ic-hit-immutable-facts.md @@ -0,0 +1,32 @@ +IC hits stop re-deriving shape-immutable facts, and the combined +overwrite+read loop goes past node: 31 → **24 ms against node's 29**. The +write-only loop drops to **15 ms against node's 23**. + +Every stub or way hit re-proved the receiver's kind (a registry probe via +`is_class_object_ptr`), the plain-ordinary verdict, and the inline slot bound +(a descriptor fetch) — together 16% of the combined loop +(`write_fast_path_receiver_kind_ok` 6.8%, `shape_object_kind_by_id` 5.1%, +`shape_live_inline_slot_count_by_id` 4.4%). + +All of those are facts of the SHAPE ID. The shape table only ever inserts +descriptors — its sole in-place mutations are GC bookkeeping (the relocated +`keys` address and the carrier liveness bits) — so `object_kind`, +`live_inline_slot_count` and `logical_key_count` cannot change under a fixed +id. A hit whose token matches the receiver's CURRENT stamp has therefore +already proved everything prime time proved about kind and bounds. + +The slot word now carries the one prime-time verdict a hit needs: +`IC_SLOT_OVERFLOW_BIT`, choosing the inline region vs the spill store. +Overflow entries are bound-checked against `logical_key_count` at prime. Hits +keep exactly the checks that ARE mutable per object — header type, forwarded, +and the blocking flags the `Object.freeze` family sets — plus the token +compare. Applied to the write stub, the per-site dyn ways, and both read-stub +hit sites. + +Interleaved A/B, min-of-21 at quiet load (~0.8): combined overwrite 31 → +24 ms; computed-key read, realistic-name read and populated delete unchanged; +write-only measured on the branch at 15 ms (node 23). + +Verification: suite 2790 passed, all 60 lint gates pass, no warnings; the +adversarial, computed-key, and stale-slot differentials all byte-identical to +node. diff --git a/crates/perry-runtime/src/object/field_get_set/get_field_by_name.rs b/crates/perry-runtime/src/object/field_get_set/get_field_by_name.rs index e8b49bb5a9..2996423e4a 100644 --- a/crates/perry-runtime/src/object/field_get_set/get_field_by_name.rs +++ b/crates/perry-runtime/src/object/field_get_set/get_field_by_name.rs @@ -142,15 +142,12 @@ pub extern "C" fn js_object_get_field_by_name( if let Some(slot) = super::super::read_stub::read_stub_probe(token, key_bits) { - let live = crate::object::object_live_slot_count(o); - let limit = - std::cmp::max(live, crate::object::INLINE_SLOT_FLOOR as u32); - if slot < limit { - return super::accessors::js_object_get_field(o, slot); - } - if let Some(bits) = super::super::overflow_get(addr, slot as usize) + // The slot word carries the inline/overflow + // verdict from prime time; no bound fetch. + if let Some(v) = + super::super::read_stub::read_slot_by_tag(o, addr, slot) { - return JSValue::from_bits(bits); + return JSValue::from_bits(v.to_bits()); } } } @@ -228,7 +225,7 @@ pub extern "C" fn js_object_get_field_by_name( keys as usize, key as usize, ) { - prime_read_stub(o, key, idx); + prime_read_stub(o, key, idx, (idx as usize) < alloc_limit); return if (idx as usize) < alloc_limit { super::accessors::js_object_get_field(o, idx) } else { @@ -264,7 +261,7 @@ pub extern "C" fn js_object_get_field_by_name( key, ) { let i = i as usize; - prime_read_stub(o, key, i as u32); + prime_read_stub(o, key, i as u32, i < alloc_limit); super::super::prop_plan::read_plan_record( keys as usize, key as usize, @@ -1758,7 +1755,17 @@ mod null_key_guard_5972 { /// semantics. Keys that cannot be represented as content bits are skipped by /// `read_stub_key_bits`. #[inline] -fn prime_read_stub(obj: *const ObjectHeader, key: *const crate::StringHeader, slot: u32) { +fn prime_read_stub( + obj: *const ObjectHeader, + key: *const crate::StringHeader, + slot: u32, + inline: bool, +) { + let slot = if inline { + slot + } else { + slot | crate::proxy::IC_SLOT_OVERFLOW_BIT + }; if let Some(key_bits) = super::super::read_stub::read_stub_key_bits(key) { if let Some(token) = unsafe { super::super::read_stub::receiver_shape_token(obj) } { super::super::read_stub::read_stub_insert(token, key_bits, slot); diff --git a/crates/perry-runtime/src/object/read_stub.rs b/crates/perry-runtime/src/object/read_stub.rs index c74ad052b0..5b7b271911 100644 --- a/crates/perry-runtime/src/object/read_stub.rs +++ b/crates/perry-runtime/src/object/read_stub.rs @@ -159,14 +159,32 @@ pub(crate) unsafe fn try_read_by_content_bits( } let token = receiver_shape_token(obj)?; let slot = read_stub_probe(token, key_bits)?; - let limit = std::cmp::max( - crate::object::object_live_slot_count(obj), - crate::object::INLINE_SLOT_FLOOR as u32, - ); - if slot < limit { - return Some(f64::from_bits( - crate::object::js_object_get_field(obj, slot).bits(), - )); + read_slot_by_tag(obj, addr, slot) +} + +/// Read the value a bit-tagged cached slot names. The inline/overflow verdict +/// was decided at prime time (`IC_SLOT_OVERFLOW_BIT`, see `proxy::put_value`) +/// under the exact shape id the caller's token match just re-proved, so no +/// bound is fetched here — the shape table only ever inserts descriptors, so +/// the verdict cannot have changed under the same id. +#[inline(always)] +pub(crate) unsafe fn read_slot_by_tag( + obj: *const ObjectHeader, + addr: usize, + slot: u32, +) -> Option { + use crate::proxy::IC_SLOT_OVERFLOW_BIT; + if slot & IC_SLOT_OVERFLOW_BIT != 0 { + return crate::object::overflow_get(addr, (slot & !IC_SLOT_OVERFLOW_BIT) as usize) + .map(f64::from_bits); + } + let fields_ptr = + (obj as *const u8).add(std::mem::size_of::()) as *const crate::JSValue; + let val = *fields_ptr.add(slot as usize); + // Same null-POINTER_TAG guard as `js_object_get_field`'s inline half: the + // pattern is never a legitimate stored value. + if val.bits() == 0x7FFD_0000_0000_0000 { + return Some(f64::from_bits(crate::value::TAG_UNDEFINED)); } - crate::object::overflow_get(addr, slot as usize).map(f64::from_bits) + Some(f64::from_bits(val.bits())) } diff --git a/crates/perry-runtime/src/proxy.rs b/crates/perry-runtime/src/proxy.rs index db09ed5520..0f601bbdf9 100644 --- a/crates/perry-runtime/src/proxy.rs +++ b/crates/perry-runtime/src/proxy.rs @@ -31,7 +31,9 @@ pub use has_delete::{js_proxy_delete, js_proxy_has}; mod invariants; mod put_value; pub use put_value::{js_proxy_set, js_put_value_set}; -pub(crate) use put_value::{js_put_value_set_ic_miss, proxy_set_with_receiver}; +pub(crate) use put_value::{ + js_put_value_set_ic_miss, proxy_set_with_receiver, IC_SLOT_OVERFLOW_BIT, +}; mod json; mod metadata; mod own_keys; diff --git a/crates/perry-runtime/src/proxy/put_value.rs b/crates/perry-runtime/src/proxy/put_value.rs index 7268ddf41d..52a3bb5772 100644 --- a/crates/perry-runtime/src/proxy/put_value.rs +++ b/crates/perry-runtime/src/proxy/put_value.rs @@ -726,12 +726,24 @@ fn write_stub_insert(token: u64, key_bits: u64, slot: u32) { }); } -/// Validated fast store: the receiver must still be an ordinary, -/// non-forwarded, unblocked, class-tagged heap object whose CURRENT shape -/// token equals the cached one and whose inline region covers the slot. -/// Every check reads the receiver's live state — the cached token/slot are -/// never dereferenced — so a GC between prime and hit at worst causes a -/// miss, never a wrong store. +/// Slot-word bit marking a cached slot as living in the OVERFLOW store. +/// +/// Set at prime time, where the full receiver validation runs and the +/// inline-vs-overflow verdict is known. `object_kind`, +/// `live_inline_slot_count` and `logical_key_count` are IMMUTABLE per shape +/// id (the shape table only ever inserts descriptors; the sole in-place +/// mutations are GC bookkeeping — the relocated `keys` address and the +/// carrier liveness bits), so a hit whose token matches the receiver's +/// CURRENT stamp has already proved everything the prime proved about kind +/// and bounds. What remains mutable per object is the GC header — type, +/// forwarded, and the blocking flags `Object.freeze`-family operations set — +/// and the hit still checks those on every store. +pub(crate) const IC_SLOT_OVERFLOW_BIT: u32 = 1 << 30; + +/// Validated fast store for a cached `(token, slot)` hit. Header checks and +/// the token compare read the receiver's LIVE state; everything shape-derived +/// was proved at prime time and is immutable per id (see +/// [`IC_SLOT_OVERFLOW_BIT`]). A stale entry misses; it cannot store wrongly. #[inline] unsafe fn dyn_ic_try_store(target: f64, token: u64, slot: u32, value: f64) -> Option { let target_bits = target.to_bits(); @@ -753,41 +765,24 @@ unsafe fn dyn_ic_try_store(target: f64, token: u64, slot: u32, value: f64) -> Op return None; } let obj = obj_addr as *mut crate::ObjectHeader; - if !crate::object::object_is_regular(obj) - || !write_fast_path_receiver_kind_ok(obj, gc_header._reserved) - { - return None; - } - // ONE descriptor lookup, not two. `object_shape_id` runs a full lookup — - // and copies the whole `ShapeDescriptor` out of the table — purely to - // prove the stamped id is live, then throws the descriptor away; the bound - // check below then looked the SAME id up again. `shape_descriptor_by_id` - // was 10.5% of self time in a computed-key write loop, the single largest - // item, and half of that was this duplicate. - // - // Read the stamp straight off the header, reject on the token first (a - // load and a compare, so a wrong-shape receiver costs no table work at - // all), and let the one lookup that supplies the bound double as the - // liveness proof: a stamp with no live descriptor returns `None` here, - // exactly as `object_shape_id`'s 0 made the token compare fail before. + // Token compare against the receiver's CURRENT stamp. On a match, the + // shape id is the one every prime-time check ran under — receiver kind, + // class-object exclusion, and the slot's inline/overflow placement (now + // carried in the slot word) — so none of it is re-derived here. This + // took `write_fast_path_receiver_kind_ok` (6.8%), `shape_object_kind_by_id` + // (5.1%) and the descriptor bound fetch (4.4%) off every hit of the + // combined overwrite loop. let stamp = crate::object::shapes::object_shape_stamp(obj); if (crate::object::shapes::PIC_ID_TOKEN_BIT | stamp as u64) != token { return None; } - let shape = crate::object::shapes::shape_descriptor_by_id(stamp)?; - if slot >= shape.live_inline_slot_count { - // Overflow slot. The token compare above already proved the receiver - // is in the exact shape the (key → slot) pair was learned in, so the - // slot names this key's storage; it just lives in the spill store - // instead of the inline region. `overflow_set` is the same store the - // full walk bottoms out in (barriers, layout notes, remembered set), - // minus the walk. Wide objects — where EVERY data property is an - // overflow slot — are exactly the receivers the stub cache exists for. - if slot < shape.logical_key_count { - crate::object::overflow_set(obj_addr, slot as usize, value.to_bits()); - return Some(value); - } - return None; + if slot & IC_SLOT_OVERFLOW_BIT != 0 { + crate::object::overflow_set( + obj_addr, + (slot & !IC_SLOT_OVERFLOW_BIT) as usize, + value.to_bits(), + ); + return Some(value); } crate::object::store_object_field_slot(obj, slot as usize, value.to_bits()); Some(value) @@ -949,12 +944,20 @@ pub extern "C" fn js_put_value_set_dyn_ic_miss( // Feed the megamorphic stub BEFORE the inline-slot bail below: a wide // object keeps every data property in an overflow slot, so gating the // stub on the inline region would starve it for exactly the receivers - // it exists for. `dyn_ic_try_store` stores overflow slots via - // `overflow_set` under the same token validation. + // it exists for. The slot word carries the inline/overflow verdict + // (IC_SLOT_OVERFLOW_BIT), decided HERE where the descriptor is in + // hand, so the hit never re-fetches the bound — the verdict is a + // fact of the shape id the token pins. + let alloc_limit = shape.live_inline_slot_count; if let Some(kb) = stub_key_bits(key) { - write_stub_insert(shape_token, kb, idx); + if idx < alloc_limit { + write_stub_insert(shape_token, kb, idx); + } else if idx < shape.logical_key_count { + // Overflow: bound-checked against the key count at prime, + // which the hit-time token match preserves. + write_stub_insert(shape_token, kb, idx | IC_SLOT_OVERFLOW_BIT); + } } - let alloc_limit = shape.live_inline_slot_count; if idx >= alloc_limit { return result; }