From 81515f9304bcee31cc469374f7788c3cbd3749f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 11:14:52 +0200 Subject: [PATCH 01/23] perf(ecs): typed branded ids, inline dynamic compares, Map heal of forwarded arrays, strict dense number store lane (WIP follow-up to #8872) Claude-Session: https://claude.ai/code/session_01FUvFrRNZyc5qknBiJbYbby --- crates/perry-codegen/src/expr/compare.rs | 134 +++- .../perry-codegen/src/expr/compare_tests.rs | 112 +++ crates/perry-codegen/src/lower_conditional.rs | 69 +- .../src/type_analysis/numeric/tests.rs | 21 +- crates/perry-hir/src/lib.rs | 2 + crates/perry-hir/src/lower_types.rs | 1 + .../lower_types/branded_intersection_tests.rs | 105 +++ crates/perry-hir/src/lower_types/extract.rs | 94 ++- crates/perry-hir/src/type_alias_resolve.rs | 694 ++++++++++++++++++ crates/perry-runtime/src/array/indexing.rs | 144 ++++ crates/perry-runtime/src/array/tests.rs | 63 ++ crates/perry-runtime/src/map.rs | 211 +++++- .../src/commands/compile/run_pipeline.rs | 217 ++++++ 13 files changed, 1801 insertions(+), 66 deletions(-) create mode 100644 crates/perry-hir/src/lower_types/branded_intersection_tests.rs create mode 100644 crates/perry-hir/src/type_alias_resolve.rs diff --git a/crates/perry-codegen/src/expr/compare.rs b/crates/perry-codegen/src/expr/compare.rs index 9e4bf53b45..d203130a36 100644 --- a/crates/perry-codegen/src/expr/compare.rs +++ b/crates/perry-codegen/src/expr/compare.rs @@ -339,6 +339,85 @@ fn lower_string_literal_strict_eq( /// /// Returns an i64 holding `TAG_TRUE`/`TAG_FALSE` (or `js_eq`'s own tagged /// boolean), i.e. the same value the bare call produced. +/// Quiet-NaN prefix (`0x7FF8_0000_0000_0000`) shared by every Perry NaN-box +/// tag and by the canonical NaN itself. +const QNAN_PREFIX_I64: &str = "9221120237041090560"; + +/// `(bits & 0x7FF8…) != 0x7FF8…`: the operand is an ordinary IEEE double — +/// finite, ±Infinity, or a signaling-NaN pattern no Perry encoding occupies. +/// Every NaN-box tag (top-16 `0x7FF9`..=`0x7FFF`, sign-clear) and the quiet +/// NaN carry the prefix, so one mask+compare separates "plain number" from +/// "tagged or NaN" without decoding either side. Two plain numbers answer +/// every relational and (strict or loose) equality operator with the raw +/// `fcmp`; the helper keeps NaN, so the unordered edge never reaches the +/// inline predicate. +fn emit_is_plain_double(ctx: &mut FnCtx<'_>, bits: &str) -> String { + let blk = ctx.block(); + let masked = blk.and(I64, bits, QNAN_PREFIX_I64); + blk.icmp_ne(I64, &masked, QNAN_PREFIX_I64) +} + +/// Dynamic-operand comparison with an inline plain-number fast path. +/// +/// When both NaN-boxed operands are ordinary doubles the result is +/// `select(fcmp l, r, TAG_TRUE, TAG_FALSE)`; every other shape — +/// strings, BigInt, objects with `valueOf`/`toString`, null/undefined/boolean +/// coercions, NaN — takes `helper`, which owns the full ECMAScript semantics. +/// `helper_takes_bits` selects the `(i64, i64) -> i64` helper ABI +/// (`js_eq`, `js_loose_eq`) over the `(double, double) -> double` one +/// (`js_rel_*`). Returns the NaN-boxed boolean as i64 bits. +fn lower_dynamic_compare_bits( + ctx: &mut FnCtx<'_>, + l: &str, + r: &str, + pred: &str, + helper: &str, + helper_takes_bits: bool, +) -> String { + let l_bits = ctx.block().bitcast_double_to_i64(l); + let r_bits = ctx.block().bitcast_double_to_i64(r); + let l_plain = emit_is_plain_double(ctx, &l_bits); + let r_plain = emit_is_plain_double(ctx, &r_bits); + let both_plain = ctx.block().and(I1, &l_plain, &r_plain); + + let fast_idx = ctx.new_block("dyncmp.num"); + let slow_idx = ctx.new_block("dyncmp.slow"); + let merge_idx = ctx.new_block("dyncmp.merge"); + let fast_l = ctx.block_label(fast_idx); + let slow_l = ctx.block_label(slow_idx); + let merge_l = ctx.block_label(merge_idx); + ctx.block().cond_br(&both_plain, &fast_l, &slow_l); + + ctx.current_block = fast_idx; + let bit = ctx.block().fcmp(pred, l, r); + let fast_res = ctx.block().select( + I1, + &bit, + I64, + crate::nanbox::TAG_TRUE_I64, + crate::nanbox::TAG_FALSE_I64, + ); + let fast_pred = ctx.block().label.clone(); + ctx.block().br(&merge_l); + + ctx.current_block = slow_idx; + let slow_res = if helper_takes_bits { + ctx.block() + .call(I64, helper, &[(I64, &l_bits), (I64, &r_bits)]) + } else { + let boxed = ctx + .block() + .call(DOUBLE, helper, &[(DOUBLE, l), (DOUBLE, r)]); + ctx.block().bitcast_double_to_i64(&boxed) + }; + let slow_pred = ctx.block().label.clone(); + ctx.block().br(&merge_l); + + ctx.current_block = merge_idx; + ctx.block() + .phi(I64, &[(&fast_res, &fast_pred), (&slow_res, &slow_pred)]) +} + fn lower_strict_eq_inline_any(ctx: &mut FnCtx<'_>, l: &str, r: &str) -> String { let l_bits = ctx.block().bitcast_double_to_i64(l); let r_bits = ctx.block().bitcast_double_to_i64(r); @@ -374,10 +453,26 @@ fn lower_strict_eq_inline_any(ctx: &mut FnCtx<'_>, l: &str, r: &str) -> String { let same_ok = ctx.block().or(I1, &tagged, ¬_nan); ctx.block().cond_br(&same_ok, &true_l, &slow_l); - // Different bits: only a same-tag pair whose encoding is canonical is - // decidable here. Pointer pairs need the runtime's allocation registries - // before either payload can safely be treated as a GC allocation. + // Different bits: two plain numbers are decided by `fcmp` (only `+0`/`-0` + // differ in bits yet compare equal); otherwise only a same-tag pair whose + // encoding is canonical is decidable here. Pointer pairs need the + // runtime's allocation registries before either payload can safely be + // treated as a GC allocation. ctx.current_block = diff_idx; + let num_idx = ctx.new_block("anyeq.num"); + let tag_idx = ctx.new_block("anyeq.tag"); + let num_l = ctx.block_label(num_idx); + let tag_l = ctx.block_label(tag_idx); + let l_plain = emit_is_plain_double(ctx, &l_bits); + let r_plain = emit_is_plain_double(ctx, &r_bits); + let both_plain = ctx.block().and(I1, &l_plain, &r_plain); + ctx.block().cond_br(&both_plain, &num_l, &tag_l); + + ctx.current_block = num_idx; + let num_eq = ctx.block().fcmp("oeq", l, r); + ctx.block().cond_br(&num_eq, &true_l, &false_l); + + ctx.current_block = tag_idx; let l_tag = ctx.block().lshr(I64, &l_bits, "48"); let r_tag = ctx.block().lshr(I64, &r_bits, "48"); let l_sso = ctx @@ -985,10 +1080,7 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // call. Loose `==`'s cross-type coercions are not // bit-decidable, so it keeps the bare call. let result_bits = if matches!(op, CompareOp::LooseEq | CompareOp::LooseNe) { - let blk = ctx.block(); - let l_bits = blk.bitcast_double_to_i64(&l); - let r_bits = blk.bitcast_double_to_i64(&r); - blk.call(I64, "js_loose_eq", &[(I64, &l_bits), (I64, &r_bits)]) + lower_dynamic_compare_bits(ctx, &l, &r, "oeq", "js_loose_eq", true) } else { lower_strict_eq_inline_any(ctx, &l, &r) }; @@ -1162,11 +1254,9 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // "1"==1, false==0, etc.). Strict === already handled // above by the typed fast paths. if matches!(op, CompareOp::LooseEq | CompareOp::LooseNe) { - let blk = ctx.block(); - let l_bits = blk.bitcast_double_to_i64(&l); - let r_bits = blk.bitcast_double_to_i64(&r); let result_bits = - blk.call(I64, "js_loose_eq", &[(I64, &l_bits), (I64, &r_bits)]); + lower_dynamic_compare_bits(ctx, &l, &r, "oeq", "js_loose_eq", true); + let blk = ctx.block(); if matches!(op, CompareOp::LooseNe) { let cmp = blk.icmp_eq(I64, &result_bits, crate::nanbox::TAG_TRUE_I64); let inv = blk.xor(crate::types::I1, &cmp, "true"); @@ -1198,16 +1288,18 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { && !is_bigint_expr(ctx, left) && !is_bigint_expr(ctx, right); if is_relational_op && !both_numeric { - let blk = ctx.block(); - let fname = match op { - CompareOp::Lt => "js_rel_lt", - CompareOp::Le => "js_rel_le", - CompareOp::Gt => "js_rel_gt", - CompareOp::Ge => "js_rel_ge", + let (pred, fname) = match op { + CompareOp::Lt => ("olt", "js_rel_lt"), + CompareOp::Le => ("ole", "js_rel_le"), + CompareOp::Gt => ("ogt", "js_rel_gt"), + CompareOp::Ge => ("oge", "js_rel_ge"), _ => unreachable!(), }; - let res = blk.call(DOUBLE, fname, &[(DOUBLE, &l), (DOUBLE, &r)]); - return Ok(res); + // Two plain numbers are the overwhelmingly common dynamic + // shape (erased ids, PIC-loaded fields); they take the raw + // `fcmp` inline and everything else keeps the helper. + let bits = lower_dynamic_compare_bits(ctx, &l, &r, pred, fname, false); + return Ok(ctx.block().bitcast_i64_to_double(&bits)); } // Strict ===/!== where the operands are NOT both certainly // numeric must NOT fall to the bare fcmp tail: a declared @@ -1217,10 +1309,8 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // js_eq answers correctly for every runtime shape, including // the honest number-vs-object case (#3576 probe family). if matches!(op, CompareOp::Eq | CompareOp::Ne) && !both_numeric { + let result_bits = lower_dynamic_compare_bits(ctx, &l, &r, "oeq", "js_eq", true); let blk = ctx.block(); - let l_bits = blk.bitcast_double_to_i64(&l); - let r_bits = blk.bitcast_double_to_i64(&r); - let result_bits = blk.call(I64, "js_eq", &[(I64, &l_bits), (I64, &r_bits)]); if matches!(op, CompareOp::Ne) { let cmp = blk.icmp_eq(I64, &result_bits, crate::nanbox::TAG_TRUE_I64); let inv = blk.xor(crate::types::I1, &cmp, "true"); diff --git a/crates/perry-codegen/src/expr/compare_tests.rs b/crates/perry-codegen/src/expr/compare_tests.rs index b80300f941..56b3694685 100644 --- a/crates/perry-codegen/src/expr/compare_tests.rs +++ b/crates/perry-codegen/src/expr/compare_tests.rs @@ -559,3 +559,115 @@ fn i8_literal_writes_high_bytes_in_twos_complement() { assert_eq!(i8_literal(0xC3), "-61"); assert_eq!(i8_literal(0xFF), "-1"); } + +// --------------------------------------------------------------------------- +// Dynamic-operand plain-number fast paths (#8872 follow-up). +// +// Two erased operands used to go straight to the runtime helper for every +// relational and equality operator. Two ordinary doubles — the overwhelmingly +// common dynamic shape (erased ids, PIC-loaded fields) — now decide inline +// with the raw `fcmp`; the helper stays reachable for every other shape. +// --------------------------------------------------------------------------- + +const JS_REL_GE_CALL: &str = "call double @js_rel_ge("; +const JS_REL_LT_CALL: &str = "call double @js_rel_lt("; + +#[test] +fn dynamic_relational_decides_plain_numbers_inline_and_keeps_the_helper() { + let ir = cmp_ir( + "dynrel_ge", + CompareOp::Ge, + Expr::LocalGet(X), + Expr::LocalGet(Y), + ); + assert!( + ir.contains("fcmp oge double"), + "dynamic `>=` has no inline number arm:\n{ir}" + ); + assert!( + ir.contains(JS_REL_GE_CALL), + "dynamic `>=` lost its coercing helper fallback:\n{ir}" + ); + let ir = cmp_ir( + "dynrel_lt", + CompareOp::Lt, + Expr::LocalGet(X), + Expr::LocalGet(Y), + ); + assert!(ir.contains("fcmp olt double"), "no inline `<` arm:\n{ir}"); + assert!(ir.contains(JS_REL_LT_CALL), "no `<` helper fallback:\n{ir}"); +} + +#[test] +fn dynamic_strict_eq_decides_plain_numbers_inline_and_keeps_js_eq() { + for (name, op) in [("dynseq", CompareOp::Eq), ("dynsne", CompareOp::Ne)] { + let ir = cmp_ir(name, op, Expr::LocalGet(X), Expr::LocalGet(Y)); + assert!( + ir.contains("fcmp oeq double"), + "{name}: no inline number arm:\n{ir}" + ); + assert!( + ir.contains(JS_EQ_CALL), + "{name}: lost the js_eq fallback:\n{ir}" + ); + } +} + +#[test] +fn dynamic_loose_eq_decides_plain_numbers_inline_and_keeps_js_loose_eq() { + let ir = cmp_ir( + "dynleq", + CompareOp::LooseEq, + Expr::LocalGet(X), + Expr::LocalGet(Y), + ); + assert!( + ir.contains("fcmp oeq double"), + "dynamic `==` has no inline number arm:\n{ir}" + ); + assert!( + ir.contains(JS_LOOSE_EQ_CALL), + "dynamic `==` lost its coercing helper:\n{ir}" + ); +} + +#[test] +fn dynamic_truthiness_decides_numbers_and_tag_singletons_inline() { + let ir = ir_for( + "dyntruthy", + vec![ + Stmt::Let { + id: X, + name: "x".to_string(), + ty: Type::Any, + mutable: true, + init: Some(Expr::Undefined), + }, + Stmt::Let { + id: R, + name: "r".to_string(), + ty: Type::Any, + mutable: true, + init: Some(Expr::Undefined), + }, + Stmt::If { + // An element read of an erased local: no initializer proof can + // settle it, so the condition reaches the dynamic predicate. + condition: Expr::IndexGet { + object: Box::new(Expr::LocalGet(X)), + index: Box::new(Expr::Integer(0)), + }, + then_branch: vec![Stmt::Expr(Expr::LocalSet(R, Box::new(Expr::Integer(1))))], + else_branch: None, + }, + ], + ); + assert!( + ir.contains("truthy.num") && ir.contains("fcmp one double"), + "dynamic truthiness has no inline number arm:\n{ir}" + ); + assert!( + ir.contains("call i32 @js_is_truthy("), + "dynamic truthiness lost the runtime predicate for strings/BigInt:\n{ir}" + ); +} diff --git a/crates/perry-codegen/src/lower_conditional.rs b/crates/perry-codegen/src/lower_conditional.rs index 72824d0472..ba5a1f5ddb 100644 --- a/crates/perry-codegen/src/lower_conditional.rs +++ b/crates/perry-codegen/src/lower_conditional.rs @@ -10,7 +10,7 @@ use crate::expr::{lower_expr, FnCtx}; use crate::type_analysis::{ expr_may_return_boxed_value_from_raw_f64_fallback, is_bool_expr, is_numeric_expr, }; -use crate::types::{DOUBLE, I32, I64}; +use crate::types::{DOUBLE, I1, I32, I64}; /// Convert a lowered condition value to an `i1` for `cond_br`. /// @@ -67,10 +67,65 @@ pub(crate) fn lower_truthy(ctx: &mut FnCtx<'_>, cond_val: &str, cond_expr: &Expr let bits = blk.bitcast_double_to_i64(cond_val); return blk.icmp_eq(I64, &bits, crate::nanbox::TAG_TRUE_I64); } + // Dynamic value: decide the bit-decidable shapes inline and keep the + // runtime predicate for the rest. A plain (non-NaN, untagged) double is + // truthy iff it is non-zero; `true`/`false`/`undefined`/`null` are single + // bit patterns. Strings (empty is falsy), BigInt (`0n` is falsy), pointers, + // handles, int32 boxes, and NaN take `js_is_truthy` exactly as before. + let bits = ctx.block().bitcast_double_to_i64(cond_val); + let masked = ctx.block().and(I64, &bits, QNAN_PREFIX_I64); + let plain = ctx.block().icmp_ne(I64, &masked, QNAN_PREFIX_I64); + + let num_idx = ctx.new_block("truthy.num"); + let tag_idx = ctx.new_block("truthy.tag"); + let slow_idx = ctx.new_block("truthy.slow"); + let merge_idx = ctx.new_block("truthy.merge"); + let num_l = ctx.block_label(num_idx); + let tag_l = ctx.block_label(tag_idx); + let slow_l = ctx.block_label(slow_idx); + let merge_l = ctx.block_label(merge_idx); + ctx.block().cond_br(&plain, &num_l, &tag_l); + + ctx.current_block = num_idx; + let num_res = ctx.block().fcmp("one", cond_val, "0.0"); + let num_pred = ctx.block().label.clone(); + ctx.block().br(&merge_l); + + ctx.current_block = tag_idx; + let is_true = ctx.block().icmp_eq(I64, &bits, crate::nanbox::TAG_TRUE_I64); + let is_false = ctx + .block() + .icmp_eq(I64, &bits, crate::nanbox::TAG_FALSE_I64); + let is_undef = ctx + .block() + .icmp_eq(I64, &bits, crate::nanbox::TAG_UNDEFINED_I64); + let is_null = ctx.block().icmp_eq(I64, &bits, crate::nanbox::TAG_NULL_I64); + let falsy_a = ctx.block().or(I1, &is_false, &is_undef); + let falsy = ctx.block().or(I1, &falsy_a, &is_null); + let decided = ctx.block().or(I1, &is_true, &falsy); + let tag_pred = ctx.block().label.clone(); + ctx.block().cond_br(&decided, &merge_l, &slow_l); + + ctx.current_block = slow_idx; let i32_truthy = ctx.block().call(I32, "js_is_truthy", &[(DOUBLE, cond_val)]); - ctx.block().icmp_ne(I32, &i32_truthy, "0") + let slow_res = ctx.block().icmp_ne(I32, &i32_truthy, "0"); + let slow_pred = ctx.block().label.clone(); + ctx.block().br(&merge_l); + + ctx.current_block = merge_idx; + ctx.block().phi( + I1, + &[ + (&num_res, &num_pred), + (&is_true, &tag_pred), + (&slow_res, &slow_pred), + ], + ) } +/// Quiet-NaN prefix (`0x7FF8_0000_0000_0000`) shared by every Perry NaN-box tag. +const QNAN_PREFIX_I64: &str = "9221120237041090560"; + /// Lower `cond ? then_expr : else_expr` to a 4-block CFG with a phi at /// the merge: condition → conditional cond_br → then → merge ← else. /// Both then and else are always lowered (no short-circuit), but only one @@ -207,11 +262,13 @@ pub(crate) fn lower_logical( // Lower left in the current block. let l = lower_expr(ctx, left)?; - // Capture the post-left block — left's lowering may have created new - // blocks via nested control flow. - let l_block_label = ctx.block().label.clone(); - // Truthiness test: fast fcmp for numeric, js_is_truthy for NaN-boxed. + // Truthiness test: fast fcmp for numeric, inline tag/number decision with + // a `js_is_truthy` fallback for NaN-boxed. let l_bool = lower_truthy(ctx, &l, left); + // Capture the post-condition block — both left's lowering and the + // truthiness test may have created new blocks, and the merge phi must + // name the block that actually branches to it. + let l_block_label = ctx.block().label.clone(); let then_idx = ctx.new_block("logical.then"); let merge_idx = ctx.new_block("logical.merge"); diff --git a/crates/perry-codegen/src/type_analysis/numeric/tests.rs b/crates/perry-codegen/src/type_analysis/numeric/tests.rs index b080f8af97..6be85b091c 100644 --- a/crates/perry-codegen/src/type_analysis/numeric/tests.rs +++ b/crates/perry-codegen/src/type_analysis/numeric/tests.rs @@ -784,11 +784,28 @@ mod symbol_keyed_element_reads { helper, or a NaN-boxed function reads as false:\n{body}" ); assert!( - !body.contains("fcmp one"), + fcmp_one_only_under_the_plain_number_guard(&body), "the numeric fast path must not fire for a non-numeric index:\n{body}" ); } + /// The dynamic truthiness lowering decides a plain (untagged, non-NaN) + /// double inline with `fcmp one` — but only inside its `truthy.num` block, + /// after the bit test that proves the value is a number. An `fcmp one` + /// anywhere else is the unguarded numeric claim these tests forbid. + fn fcmp_one_only_under_the_plain_number_guard(body: &str) -> bool { + let mut label = String::new(); + for line in body.lines() { + let trimmed = line.trim_start(); + if !line.starts_with(' ') && trimmed.ends_with(':') { + label = trimmed.trim_end_matches(':').to_string(); + } else if trimmed.contains("fcmp one") && !label.starts_with("truthy.num") { + return false; + } + } + true + } + #[test] fn a_numeric_index_keeps_the_array_fast_path_but_not_a_truthiness_claim() { // A numeric index preserves the guarded array read, but its boxed @@ -828,7 +845,7 @@ mod symbol_keyed_element_reads { "the result binding must use runtime truthiness:\n{body}" ); assert!( - !body.contains("fcmp one"), + fcmp_one_only_under_the_plain_number_guard(&body), "the read's boxed fallback must not become a numeric proof:\n{body}" ); } diff --git a/crates/perry-hir/src/lib.rs b/crates/perry-hir/src/lib.rs index 2819ddb2dd..b2279eb7dd 100644 --- a/crates/perry-hir/src/lib.rs +++ b/crates/perry-hir/src/lib.rs @@ -26,6 +26,7 @@ pub(crate) mod lower_types; pub mod monomorph; pub mod native_profile; pub mod stable_hash; +pub mod type_alias_resolve; pub mod types; pub mod walker; @@ -66,3 +67,4 @@ pub use lower::{ }; pub use monomorph::monomorphize_module; pub use native_profile::exported_native_pod_abi; +pub use type_alias_resolve::{resolve_type_aliases_in_module, AliasDef, AliasTable}; diff --git a/crates/perry-hir/src/lower_types.rs b/crates/perry-hir/src/lower_types.rs index 053534de19..96f1556078 100644 --- a/crates/perry-hir/src/lower_types.rs +++ b/crates/perry-hir/src/lower_types.rs @@ -1474,6 +1474,7 @@ pub(crate) fn infer_call_return_type(callee: &ast::Expr, ctx: &LoweringContext) } } +mod branded_intersection_tests; mod extract; mod generic_alias_specialization_tests; diff --git a/crates/perry-hir/src/lower_types/branded_intersection_tests.rs b/crates/perry-hir/src/lower_types/branded_intersection_tests.rs new file mode 100644 index 0000000000..928dd80e87 --- /dev/null +++ b/crates/perry-hir/src/lower_types/branded_intersection_tests.rs @@ -0,0 +1,105 @@ +//! Branded-primitive intersections (`number & { __tag: T }`) lower to the +//! primitive they spell, so an id alias gets the same guarded native +//! treatment as a plain `number` annotation. Object-object merges and +//! conflicting primitives keep the `Any` lowering. + +#![cfg(test)] + +use crate::lower_module; +use crate::types::Type; +use crate::Module; +use perry_diagnostics::SourceCache; +use perry_parser::parse_typescript_with_cache; + +fn lower_src(src: &str) -> Module { + let src = src.to_string(); + std::thread::Builder::new() + .stack_size(32 * 1024 * 1024) + .spawn(move || { + let mut cache = SourceCache::new(); + let parsed = parse_typescript_with_cache(&src, "test.ts", &mut cache) + .expect("parse should succeed"); + lower_module(&parsed.module, "test", "test.ts").expect("lowering should succeed") + }) + .expect("spawn") + .join() + .expect("lowering thread") +} + +fn first_param_type(module: &Module, func: &str) -> Type { + module + .functions + .iter() + .find(|f| f.name == func) + .unwrap_or_else(|| panic!("function {func} not lowered")) + .params[0] + .ty + .clone() +} + +#[test] +fn branded_number_and_string_intersections_lower_to_their_primitive() { + let module = lower_src( + r#" +declare const __tag: unique symbol; +type Id = number & { readonly [__tag]?: "id" }; +type Name = string & { readonly __brand: "name" }; +type Loose = string & {}; +type Both = number & { a: 1 } & { b: 2 }; +export function fId(x: Id): boolean { return x >= 0; } +export function fName(x: Name): boolean { return x === "a"; } +export function fLoose(x: Loose): boolean { return x === "a"; } +export function fBoth(x: Both): boolean { return x >= 0; } +"#, + ); + assert_eq!(first_param_type(&module, "fId"), Type::Number); + assert_eq!(first_param_type(&module, "fName"), Type::String); + assert_eq!(first_param_type(&module, "fLoose"), Type::String); + assert_eq!(first_param_type(&module, "fBoth"), Type::Number); +} + +#[test] +fn non_branded_intersections_stay_dynamic() { + let module = lower_src( + r#" +type A = { a: number }; +type B = { b: string }; +type Merged = A & B; +type Conflict = number & string; +type Arr = number[] & { extra: true }; +export function fMerged(x: Merged): number { return x.a; } +export function fConflict(x: Conflict): boolean { return x === 1; } +export function fArr(x: Arr): number { return x.length; } +"#, + ); + assert_eq!(first_param_type(&module, "fMerged"), Type::Any); + assert_eq!(first_param_type(&module, "fConflict"), Type::Any); + assert_eq!(first_param_type(&module, "fArr"), Type::Any); +} + +#[test] +fn generic_branded_alias_reference_stays_generic_until_the_driver_pass() { + // The same-module resolver only settles non-generic aliases; a generic + // instantiation is closed later by `type_alias_resolve` with the alias + // table the driver assembles across modules. + let module = lower_src( + r#" +declare const __tag: unique symbol; +export type EntityId = number & { readonly [__tag]?: T }; +export function f(x: EntityId): boolean { return x >= 0; } +"#, + ); + let alias = module + .type_aliases + .iter() + .find(|a| a.name == "EntityId") + .expect("alias recorded"); + assert_eq!(alias.ty, Type::Number); + assert_eq!( + first_param_type(&module, "f"), + Type::Generic { + base: "EntityId".to_string(), + type_args: vec![Type::String], + } + ); +} diff --git a/crates/perry-hir/src/lower_types/extract.rs b/crates/perry-hir/src/lower_types/extract.rs index 146f32f453..226da864b1 100644 --- a/crates/perry-hir/src/lower_types/extract.rs +++ b/crates/perry-hir/src/lower_types/extract.rs @@ -77,22 +77,19 @@ pub(crate) fn extract_ts_type_with_ctx( } // Union type: A | B | C - TsUnionOrIntersectionType(union_or_inter) => { - match union_or_inter { - ast::TsUnionOrIntersectionType::TsUnionType(union) => { - let types: Vec = union - .types - .iter() - .map(|t| extract_ts_type_with_ctx(t, ctx)) - .collect(); - Type::Union(types) - } - ast::TsUnionOrIntersectionType::TsIntersectionType(_) => { - // Intersection types are complex - treat as Any for now - Type::Any - } + TsUnionOrIntersectionType(union_or_inter) => match union_or_inter { + ast::TsUnionOrIntersectionType::TsUnionType(union) => { + let types: Vec = union + .types + .iter() + .map(|t| extract_ts_type_with_ctx(t, ctx)) + .collect(); + Type::Union(types) } - } + ast::TsUnionOrIntersectionType::TsIntersectionType(inter) => { + lower_intersection_type(&inter.types, ctx) + } + }, // Type reference: Array, MyClass, T (type param), etc. TsTypeRef(type_ref) => { @@ -640,3 +637,70 @@ fn decorator_prop_name(name: &ast::PropName) -> Option { _ => None, } } + +/// `A & B & …`. +/// +/// TypeScript's branded-primitive idiom — `number & { readonly __tag: T }`, +/// `string & {}` — describes a value whose runtime representation is exactly +/// the primitive member: the object-literal "brand" exists only in the type +/// system and is never materialized, since a primitive cannot carry own +/// properties. Lower such an intersection to that primitive when exactly one +/// primitive kind is present and every other member is a brand-shaped object, +/// reference, or type-variable type. This gives a branded id the same +/// guarded-but-native treatment as a plain `number` annotation; it trusts the +/// annotation no further than `number` itself is trusted. Anything else — +/// object-object merges, conflicting primitives, arrays, functions — keeps the +/// prior `Any` lowering. +fn lower_intersection_type(members: &[Box], ctx: Option<&LoweringContext>) -> Type { + let mut primitive: Option = None; + for member in members { + // An object-literal type is a brand whatever it lowers to (`{}` + // extracts as `Any`, which is the identity of `&` here, not TS `any`). + if matches!(member.as_ref(), ast::TsType::TsTypeLit(_)) { + continue; + } + let ty = extract_ts_type_with_ctx(member, ctx); + match ty { + Type::Number + | Type::Int32 + | Type::String + | Type::StringLiteral(_) + | Type::Boolean + | Type::BigInt + | Type::Symbol => match &primitive { + None => primitive = Some(ty), + // `string & "a"` is the literal; `number & number` is number. + Some(prev) if same_primitive_kind(prev, &ty) => { + if matches!(ty, Type::StringLiteral(_)) { + primitive = Some(ty); + } + } + Some(_) => return Type::Any, + }, + // Brand-shaped members: object literal types, named/generic + // references (interfaces, other brands), type variables, and + // `unknown` (the identity of `&`). + Type::Object(_) + | Type::Named(_) + | Type::Generic { .. } + | Type::TypeVar(_) + | Type::Unknown => {} + _ => return Type::Any, + } + } + primitive.unwrap_or(Type::Any) +} + +fn same_primitive_kind(a: &Type, b: &Type) -> bool { + matches!( + (a, b), + (Type::Number | Type::Int32, Type::Number | Type::Int32) + | ( + Type::String | Type::StringLiteral(_), + Type::String | Type::StringLiteral(_) + ) + | (Type::Boolean, Type::Boolean) + | (Type::BigInt, Type::BigInt) + | (Type::Symbol, Type::Symbol) + ) +} diff --git a/crates/perry-hir/src/type_alias_resolve.rs b/crates/perry-hir/src/type_alias_resolve.rs new file mode 100644 index 0000000000..e302c97b53 --- /dev/null +++ b/crates/perry-hir/src/type_alias_resolve.rs @@ -0,0 +1,694 @@ +//! Driver-time resolution of type-alias references that HIR lowering could not +//! settle by itself. +//! +//! Per-module lowering resolves a *non-generic, same-module* alias reference at +//! extraction time (`LoweringContext::resolve_type_alias`). Two shapes stay +//! opaque after that: +//! +//! * an alias imported from another module — `import type { EntityId } from +//! "../entity"` leaves every `EntityId` annotation as `Type::Named("EntityId")`; +//! * a generic alias instantiation — `EntityId` is `Type::Generic { base: +//! "EntityId", .. }` even in the defining module, because the same-module +//! resolver only accepts aliases without type parameters. +//! +//! Both erase what the alias actually spells. For the branded-primitive idiom +//! (`type EntityId = number & { __tag?: T }`) that erasure turns every id in +//! a program into a dynamic value: comparisons take the generic relational and +//! equality helpers, Map keys lose their numeric proofs, and typed calling +//! conventions never fire — even though the same annotation written as plain +//! `number` would be guarded and lowered natively. +//! +//! This pass runs once all modules are lowered. Each module gets a table of the +//! aliases in *its own scope* (its declarations plus the aliases its imports +//! bind, keyed by the local binding name and looked up through the resolved +//! import path, exactly like the enum fix-up), so a name is never resolved +//! against an unrelated module's alias of the same name. Alias bodies are first +//! closed against their defining module's scope, then every type position in +//! the module is rewritten. +//! +//! Resolution is deliberately conservative: an alias reference is replaced only +//! when the instantiated body contains no remaining `TypeVar` and no alias +//! reference it could not resolve. Everything else keeps its original +//! `Named`/`Generic` spelling, so consumers that key on those shapes see exactly +//! what they saw before. + +use std::collections::{BTreeMap, HashMap}; + +use crate::ir::*; +use crate::monomorph::substitute_type; +use crate::types::{ObjectType, PropertyInfo, Type, TypeParam}; +use crate::walker::walk_expr_children_mut; + +/// A type alias definition as seen from a consuming module. +#[derive(Clone, Debug, PartialEq)] +pub struct AliasDef { + pub params: Vec, + pub ty: Type, +} + +/// Alias definitions visible in one module, keyed by the local binding name. +pub type AliasTable = BTreeMap; + +/// Bound on alias-of-alias chasing. Real chains are two or three deep +/// (`ComponentId` → `EntityId` → `number`); a malformed cycle +/// must terminate rather than hang the compiler. +const MAX_DEPTH: usize = 8; + +/// Resolve every alias reference inside `ty` against `table`. +/// +/// Returns `ty` unchanged (structurally) where nothing resolves. A reference +/// whose instantiated body still carries a `TypeVar` or an unresolved alias +/// reference is left as written. +pub fn resolve_type(ty: &Type, table: &AliasTable) -> Type { + resolve_type_inner(ty, table, 0) +} + +fn resolve_type_inner(ty: &Type, table: &AliasTable, depth: usize) -> Type { + if depth > MAX_DEPTH { + return ty.clone(); + } + match ty { + Type::Named(name) => match table.get(name) { + Some(def) => instantiate(def, &[], table, depth).unwrap_or_else(|| ty.clone()), + None => ty.clone(), + }, + Type::Generic { base, type_args } => { + let args: Vec = type_args + .iter() + .map(|t| resolve_type_inner(t, table, depth)) + .collect(); + match table.get(base) { + Some(def) => { + instantiate(def, &args, table, depth).unwrap_or_else(|| Type::Generic { + base: base.clone(), + type_args: args, + }) + } + None => Type::Generic { + base: base.clone(), + type_args: args, + }, + } + } + Type::Array(elem) => Type::Array(Box::new(resolve_type_inner(elem, table, depth))), + Type::Tuple(elems) => Type::Tuple( + elems + .iter() + .map(|e| resolve_type_inner(e, table, depth)) + .collect(), + ), + Type::Promise(inner) => Type::Promise(Box::new(resolve_type_inner(inner, table, depth))), + Type::Union(types) => Type::Union( + types + .iter() + .map(|t| resolve_type_inner(t, table, depth)) + .collect(), + ), + Type::Function(f) => Type::Function(crate::types::FunctionType { + params: f + .params + .iter() + .map(|(n, t, opt)| (n.clone(), resolve_type_inner(t, table, depth), *opt)) + .collect(), + return_type: Box::new(resolve_type_inner(&f.return_type, table, depth)), + is_async: f.is_async, + is_generator: f.is_generator, + }), + Type::Object(obj) => Type::Object(ObjectType { + name: obj.name.clone(), + properties: obj + .properties + .iter() + .map(|(k, p)| { + ( + k.clone(), + PropertyInfo { + ty: resolve_type_inner(&p.ty, table, depth), + optional: p.optional, + readonly: p.readonly, + }, + ) + }) + .collect(), + property_order: obj.property_order.clone(), + index_signature: obj + .index_signature + .as_ref() + .map(|t| Box::new(resolve_type_inner(t, table, depth))), + }), + _ => ty.clone(), + } +} + +/// Instantiate `def` with positional `args` (a missing argument takes the +/// parameter default, else `Any`) and resolve the body. `None` when the result +/// is not closed — it still mentions a type variable or an alias the table +/// cannot resolve — so the caller keeps the original reference. +fn instantiate(def: &AliasDef, args: &[Type], table: &AliasTable, depth: usize) -> Option { + let body = if def.params.is_empty() { + def.ty.clone() + } else { + let mut subs: HashMap = HashMap::new(); + for (i, p) in def.params.iter().enumerate() { + let arg = args + .get(i) + .cloned() + .or_else(|| p.default.as_deref().cloned()) + .unwrap_or(Type::Any); + subs.insert(p.name.clone(), arg); + } + substitute_type_deep(&def.ty, &subs) + }; + let resolved = resolve_type_inner(&body, table, depth + 1); + if type_is_closed(&resolved, table) { + Some(resolved) + } else { + None + } +} + +/// [`substitute_type`] with descent into object-literal types, which the +/// monomorphizer's substitution leaves opaque; an alias body such as +/// `type Box = { value: T }` carries its parameter inside the object. +fn substitute_type_deep(ty: &Type, subs: &HashMap) -> Type { + match ty { + Type::Object(obj) => Type::Object(ObjectType { + name: obj.name.clone(), + properties: obj + .properties + .iter() + .map(|(k, p)| { + ( + k.clone(), + PropertyInfo { + ty: substitute_type_deep(&p.ty, subs), + optional: p.optional, + readonly: p.readonly, + }, + ) + }) + .collect(), + property_order: obj.property_order.clone(), + index_signature: obj + .index_signature + .as_ref() + .map(|t| Box::new(substitute_type_deep(t, subs))), + }), + Type::Array(e) => Type::Array(Box::new(substitute_type_deep(e, subs))), + Type::Promise(e) => Type::Promise(Box::new(substitute_type_deep(e, subs))), + Type::Tuple(v) => Type::Tuple(v.iter().map(|t| substitute_type_deep(t, subs)).collect()), + Type::Union(v) => Type::Union(v.iter().map(|t| substitute_type_deep(t, subs)).collect()), + Type::Generic { base, type_args } => Type::Generic { + base: base.clone(), + type_args: type_args + .iter() + .map(|t| substitute_type_deep(t, subs)) + .collect(), + }, + Type::Function(f) => Type::Function(crate::types::FunctionType { + params: f + .params + .iter() + .map(|(n, t, opt)| (n.clone(), substitute_type_deep(t, subs), *opt)) + .collect(), + return_type: Box::new(substitute_type_deep(&f.return_type, subs)), + is_async: f.is_async, + is_generator: f.is_generator, + }), + _ => substitute_type(ty, subs), + } +} + +/// A resolved alias body may be substituted for its reference only when it +/// mentions no type variable and no alias the table knows but could not +/// resolve (a cycle or a non-closed instantiation). +fn type_is_closed(ty: &Type, table: &AliasTable) -> bool { + match ty { + Type::TypeVar(_) => false, + Type::Named(name) => !table.contains_key(name), + Type::Generic { base, type_args } => { + !table.contains_key(base) && type_args.iter().all(|t| type_is_closed(t, table)) + } + Type::Array(e) | Type::Promise(e) => type_is_closed(e, table), + Type::Tuple(v) | Type::Union(v) => v.iter().all(|t| type_is_closed(t, table)), + Type::Function(f) => { + f.params.iter().all(|(_, t, _)| type_is_closed(t, table)) + && type_is_closed(&f.return_type, table) + } + Type::Object(obj) => { + obj.properties + .values() + .all(|p| type_is_closed(&p.ty, table)) + && obj + .index_signature + .as_ref() + .is_none_or(|t| type_is_closed(t, table)) + } + _ => true, + } +} + +/// Rewrite every type position in `module` through [`resolve_type`]. +pub fn resolve_type_aliases_in_module(module: &mut Module, table: &AliasTable) { + if table.is_empty() { + return; + } + for func in &mut module.functions { + fix_function(func, table); + } + for class in &mut module.classes { + fix_class(class, table); + } + for global in &mut module.globals { + global.ty = resolve_type(&global.ty, table); + } + for iface in &mut module.interfaces { + for ext in &mut iface.extends { + *ext = resolve_type(ext, table); + } + for prop in &mut iface.properties { + prop.ty = resolve_type(&prop.ty, table); + } + for method in &mut iface.methods { + for (_, ty, _) in &mut method.params { + *ty = resolve_type(ty, table); + } + method.return_type = resolve_type(&method.return_type, table); + } + } + for alias in &mut module.type_aliases { + alias.ty = resolve_type(&alias.ty, table); + } + fix_stmts(&mut module.init, table); +} + +fn fix_function(func: &mut Function, table: &AliasTable) { + for param in &mut func.params { + param.ty = resolve_type(¶m.ty, table); + if let Some(default) = param.default.as_mut() { + fix_expr(default, table); + } + } + func.return_type = resolve_type(&func.return_type, table); + fix_stmts(&mut func.body, table); +} + +fn fix_field(field: &mut ClassField, table: &AliasTable) { + field.ty = resolve_type(&field.ty, table); + if let Some(init) = field.init.as_mut() { + fix_expr(init, table); + } + if let Some(key) = field.key_expr.as_mut() { + fix_expr(key, table); + } +} + +fn fix_class(class: &mut Class, table: &AliasTable) { + for field in &mut class.fields { + fix_field(field, table); + } + for field in &mut class.static_fields { + fix_field(field, table); + } + if let Some(ctor) = class.constructor.as_mut() { + fix_function(ctor, table); + } + for method in &mut class.methods { + fix_function(method, table); + } + for method in &mut class.static_methods { + fix_function(method, table); + } + for (_, getter) in &mut class.getters { + fix_function(getter, table); + } + for (_, setter) in &mut class.setters { + fix_function(setter, table); + } + for member in &mut class.computed_members { + fix_expr(&mut member.key_expr, table); + fix_function(&mut member.function, table); + } + if let Some(extends) = class.extends_expr.as_mut() { + fix_expr(extends, table); + } +} + +fn fix_stmts(stmts: &mut [Stmt], table: &AliasTable) { + for stmt in stmts.iter_mut() { + match stmt { + Stmt::Let { ty, init, .. } => { + *ty = resolve_type(ty, table); + if let Some(init) = init.as_mut() { + fix_expr(init, table); + } + } + Stmt::Expr(expr) | Stmt::Return(Some(expr)) | Stmt::Throw(expr) => { + fix_expr(expr, table); + } + Stmt::If { + condition, + then_branch, + else_branch, + } => { + fix_expr(condition, table); + fix_stmts(then_branch, table); + if let Some(else_branch) = else_branch.as_mut() { + fix_stmts(else_branch, table); + } + } + Stmt::While { condition, body } | Stmt::DoWhile { body, condition } => { + fix_expr(condition, table); + fix_stmts(body, table); + } + Stmt::Labeled { body, .. } => fix_stmts(std::slice::from_mut(&mut **body), table), + Stmt::For { + init, + condition, + update, + body, + } => { + if let Some(init) = init.as_mut() { + fix_stmts(std::slice::from_mut(&mut **init), table); + } + if let Some(condition) = condition.as_mut() { + fix_expr(condition, table); + } + if let Some(update) = update.as_mut() { + fix_expr(update, table); + } + fix_stmts(body, table); + } + Stmt::Switch { + discriminant, + cases, + } => { + fix_expr(discriminant, table); + for case in cases { + if let Some(test) = case.test.as_mut() { + fix_expr(test, table); + } + fix_stmts(&mut case.body, table); + } + } + Stmt::Try { + body, + catch, + finally, + } => { + fix_stmts(body, table); + if let Some(catch) = catch.as_mut() { + fix_stmts(&mut catch.body, table); + } + if let Some(finally) = finally.as_mut() { + fix_stmts(finally, table); + } + } + Stmt::Return(None) + | Stmt::Break + | Stmt::Continue + | Stmt::LabeledBreak(_) + | Stmt::LabeledContinue(_) + | Stmt::PreallocateBoxes(_) + | Stmt::PreallocateTdzBoxes(_) + | Stmt::ReleaseBoxes(_) => {} + } + } +} + +fn fix_expr(expr: &mut Expr, table: &AliasTable) { + match expr { + Expr::Closure { + params, + return_type, + body, + .. + } => { + for param in params.iter_mut() { + param.ty = resolve_type(¶m.ty, table); + } + *return_type = resolve_type(return_type, table); + fix_stmts(body, table); + } + Expr::ExternFuncRef { + param_types, + return_type, + .. + } => { + for ty in param_types.iter_mut() { + *ty = resolve_type(ty, table); + } + *return_type = resolve_type(return_type, table); + } + Expr::JsonParseTyped { ty, .. } + | Expr::PodLayoutSizeOf { ty } + | Expr::PodLayoutAlignOf { ty } + | Expr::PodLayoutOffsetOf { ty, .. } => { + *ty = resolve_type(ty, table); + } + _ => {} + } + // Direct children, including closure parameter defaults; the closure body + // (a statement list) was handled above. + walk_expr_children_mut(expr, &mut |child| fix_expr(child, table)); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn param(name: &str, default: Option) -> TypeParam { + TypeParam { + name: name.to_string(), + constraint: None, + default: default.map(Box::new), + } + } + + fn branded_table() -> AliasTable { + let mut table = AliasTable::new(); + // type EntityId = number & { … } → Number + table.insert( + "EntityId".into(), + AliasDef { + params: vec![ + param("T", Some(Type::Unknown)), + param("U", Some(Type::Unknown)), + ], + ty: Type::Number, + }, + ); + // type ComponentId = EntityId + table.insert( + "ComponentId".into(), + AliasDef { + params: vec![param("T", Some(Type::Void))], + ty: Type::Generic { + base: "EntityId".into(), + type_args: vec![ + Type::TypeVar("T".into()), + Type::StringLiteral("component".into()), + ], + }, + }, + ); + // type Box = { value: T } — open body + let mut props = HashMap::new(); + props.insert( + "value".to_string(), + PropertyInfo { + ty: Type::TypeVar("T".into()), + optional: false, + readonly: false, + }, + ); + table.insert( + "Box".into(), + AliasDef { + params: vec![param("T", None)], + ty: Type::Object(ObjectType { + name: None, + properties: props, + property_order: None, + index_signature: None, + }), + }, + ); + table + } + + #[test] + fn branded_generic_alias_resolves_to_its_primitive() { + let table = branded_table(); + assert_eq!( + resolve_type(&Type::Named("EntityId".into()), &table), + Type::Number + ); + assert_eq!( + resolve_type( + &Type::Generic { + base: "EntityId".into(), + type_args: vec![Type::Any], + }, + &table + ), + Type::Number + ); + // Alias of an alias, with the argument threaded through. + assert_eq!( + resolve_type( + &Type::Generic { + base: "ComponentId".into(), + type_args: vec![Type::String], + }, + &table + ), + Type::Number + ); + // Nested positions. + assert_eq!( + resolve_type( + &Type::Generic { + base: "Map".into(), + type_args: vec![ + Type::Named("EntityId".into()), + Type::Array(Box::new(Type::Named("ComponentId".into()))), + ], + }, + &table + ), + Type::Generic { + base: "Map".into(), + type_args: vec![Type::Number, Type::Array(Box::new(Type::Number))], + } + ); + } + + #[test] + fn open_instantiations_and_unknown_names_keep_their_spelling() { + let table = branded_table(); + // A consumer's own type variable stays a type variable, so the alias + // reference is left alone. + let open = Type::Generic { + base: "Box".into(), + type_args: vec![Type::TypeVar("Q".into())], + }; + assert_eq!(resolve_type(&open, &table), open); + // A closed instantiation of an object alias resolves. + let closed = Type::Generic { + base: "Box".into(), + type_args: vec![Type::Number], + }; + match resolve_type(&closed, &table) { + Type::Object(obj) => assert_eq!(obj.properties["value"].ty, Type::Number), + other => panic!("expected an object type, got {other:?}"), + } + // Names outside the table are untouched. + assert_eq!( + resolve_type(&Type::Named("Archetype".into()), &table), + Type::Named("Archetype".into()) + ); + } + + #[test] + fn cyclic_aliases_terminate_unresolved() { + let mut table = AliasTable::new(); + table.insert( + "A".into(), + AliasDef { + params: vec![], + ty: Type::Named("B".into()), + }, + ); + table.insert( + "B".into(), + AliasDef { + params: vec![], + ty: Type::Named("A".into()), + }, + ); + assert_eq!( + resolve_type(&Type::Named("A".into()), &table), + Type::Named("A".into()) + ); + } + + #[test] + fn module_pass_rewrites_params_locals_fields_and_closures() { + let table = branded_table(); + let id_ty = Type::Generic { + base: "EntityId".into(), + type_args: vec![Type::Any], + }; + let mut module = Module::new("m"); + module.functions.push(Function { + id: 0, + name: "f".into(), + type_params: vec![], + params: vec![Param { + id: 0, + name: "id".into(), + ty: id_ty.clone(), + default: None, + decorators: Vec::new(), + is_rest: false, + arguments_object: None, + }], + return_type: Type::Named("ComponentId".into()), + body: vec![Stmt::Let { + id: 1, + name: "x".into(), + ty: id_ty.clone(), + mutable: false, + init: Some(Expr::Closure { + func_id: 1, + params: vec![], + return_type: id_ty.clone(), + body: vec![Stmt::Let { + id: 2, + name: "y".into(), + ty: id_ty.clone(), + mutable: false, + init: None, + }], + captures: vec![], + mutable_captures: vec![], + captures_this: false, + captures_new_target: false, + enclosing_class: None, + is_arrow: true, + is_async: false, + is_generator: false, + is_strict: false, + }), + }], + is_async: false, + is_generator: false, + is_strict: false, + is_exported: false, + captures: vec![], + decorators: vec![], + was_plain_async: false, + was_unrolled: false, + }); + resolve_type_aliases_in_module(&mut module, &table); + let f = &module.functions[0]; + assert_eq!(f.params[0].ty, Type::Number); + assert_eq!(f.return_type, Type::Number); + let Stmt::Let { ty, init, .. } = &f.body[0] else { + panic!("expected let"); + }; + assert_eq!(*ty, Type::Number); + let Some(Expr::Closure { + return_type, body, .. + }) = init + else { + panic!("expected closure"); + }; + assert_eq!(*return_type, Type::Number); + let Stmt::Let { ty, .. } = &body[0] else { + panic!("expected inner let"); + }; + assert_eq!(*ty, Type::Number); + } +} diff --git a/crates/perry-runtime/src/array/indexing.rs b/crates/perry-runtime/src/array/indexing.rs index bf86f40666..725ff798a0 100644 --- a/crates/perry-runtime/src/array/indexing.rs +++ b/crates/perry-runtime/src/array/indexing.rs @@ -1254,12 +1254,156 @@ fn array_strict_index_write_guard_resolved(clean: *mut ArrayHeader, index: u32, } } +/// Fast lane for the dominant element-store shape: a plain number written +/// into an in-bounds slot of a live, unrestricted array whose GC layout is +/// either pointer-free or tag-scanned. +/// +/// The general path resolves the receiver through the tracked-header +/// classifier, probes two registries, roots both operands in a handle scope, +/// canonicalizes the value, and then funnels the slot write through the +/// layout note and the write barrier. For this shape every one of those steps +/// is provably a no-op, so it is answered here with a handful of header +/// tests: the receiver is validated the same way `clean_arr_ptr` starts +/// (tag strip, address band) and is then required to sit on a page the arena +/// owns (`classify_heap_generation`, the cached lookup the write barrier +/// itself relies on) before its header is read. Everything else — forwarded +/// stubs, descriptors, frozen/sealed/non-extensible arrays, side-mask or typed +/// or element-shape layouts, typed arrays and buffers, out-of-range indices, +/// tagged or NaN values, `Array.prototype` — returns `false` untouched and +/// takes the general path exactly as before. +/// +/// A plain double needs no numeric canonicalization (it is already the raw +/// `f64` the raw-f64 layout stores), cannot be a heap pointer (no barrier, no +/// pointer-mask update), and keeps a pointer-free or tag-scanned layout valid. +#[inline] +unsafe fn try_strict_dense_number_store(arr: *mut ArrayHeader, index: u32, value: f64) -> bool { + const PAYLOAD_MASK: u64 = 0x0000_FFFF_FFFF_FFFF; + let value_bits = value.to_bits(); + // A plain double or an INT32 box (`value_bits_to_number` already refuses + // the class-reference values that share that tag). NaN keeps the general + // path so its canonical encoding stays in one place. + let Some(number) = super::header::value_bits_to_number(value_bits) else { + return false; + }; + if number.is_nan() { + return false; + } + let bits = arr as u64; + let top16 = bits >> 48; + let raw = if top16 >= 0x7FF8 { + if top16 == 0x7FFC || bits & PAYLOAD_MASK == 0 { + return false; + } + (bits & PAYLOAD_MASK) as usize + } else { + bits as usize + }; + if raw < crate::gc::GC_HEADER_SIZE + || raw % std::mem::align_of::() != 0 + || !crate::value::addr_class::is_plausible_heap_addr(raw) + { + return false; + } + if matches!( + crate::arena::classify_heap_generation(raw), + crate::arena::HeapGeneration::Unknown + ) { + return false; + } + let mut raw = raw; + let mut header = (raw - crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader; + if (*header).obj_type != crate::gc::GC_TYPE_ARRAY { + return false; + } + if (*header).gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 { + // An alias that kept a growth stub (the resolver path-compresses + // longer chains to one edge): follow it once, re-proving the target + // exactly like the stub. Anything else stays on the full resolver. + let target = crate::gc::forwarding_address(header) as usize; + if target < crate::gc::GC_HEADER_SIZE + || target % std::mem::align_of::() != 0 + || !crate::value::addr_class::is_plausible_heap_addr(target) + || matches!( + crate::arena::classify_heap_generation(target), + crate::arena::HeapGeneration::Unknown + ) + { + return false; + } + let target_header = (target - crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader; + if (*target_header).obj_type != crate::gc::GC_TYPE_ARRAY + || (*target_header).gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 + { + return false; + } + raw = target; + header = target_header; + } + let flags = (*header)._reserved; + // Array header bits only: for `GC_TYPE_ARRAY` the 0x1000 bit is + // `GC_ARRAY_RAW_F64_HOLES`, not the object typed-layout flag. + const REJECT: u16 = crate::gc::OBJ_FLAG_FROZEN + | crate::gc::OBJ_FLAG_SEALED + | crate::gc::OBJ_FLAG_NO_EXTEND + | crate::gc::OBJ_FLAG_ARRAY_DESCRIPTORS + | crate::gc::GC_ARRAY_ELEMENT_SHAPE + | crate::gc::GC_LAYOUT_ALL_POINTERS; + if flags & REJECT != 0 { + return false; + } + let layout = flags & crate::gc::GC_LAYOUT_STATE_MASK; + if layout != crate::gc::GC_LAYOUT_POINTER_FREE && layout != 0 { + return false; + } + let arr = raw as *mut ArrayHeader; + if index >= (*arr).length || index >= (*arr).capacity { + return false; + } + if crate::buffer::is_registered_buffer(raw) + || crate::typedarray::lookup_typed_array_kind(raw).is_some() + || raw == array_prototype_addr() + { + return false; + } + // The raw-f64 layouts store the canonical double (what the general path's + // canonicalization and `note_array_numeric_index_write` produce); every + // other layout keeps the value's own encoding. + let store_bits = + if flags & (crate::gc::GC_ARRAY_RAW_F64_LAYOUT | crate::gc::GC_ARRAY_RAW_F64_HOLES) != 0 { + number.to_bits() + } else { + value_bits + }; + // GC_STORE_AUDIT(POINTER_FREE): a number never holds a heap pointer, and + // the receiver's layout was proved pointer-free or tag-scanned above. + ptr::write( + super::header::array_elements_ptr(arr).add(index as usize), + store_bits, + ); + true +} + +/// Exercised by the unit tests: `true` when the fast lane answered the store. +#[cfg(test)] +pub(crate) fn test_strict_dense_number_store( + arr: *mut ArrayHeader, + index: u32, + value: f64, +) -> bool { + unsafe { try_strict_dense_number_store(arr, index, value) } +} + #[no_mangle] pub extern "C" fn js_array_set_f64_extend_strict( arr: *mut ArrayHeader, index: u32, value: f64, ) -> *mut ArrayHeader { + // SAFETY: the lane validates the receiver before every dereference and + // stores only where the general path would store the same bits. + if unsafe { try_strict_dense_number_store(arr, index, value) } { + return clean_arr_ptr_mut(arr); + } let clean = clean_arr_ptr_mut(arr); if clean.is_null() || crate::buffer::is_registered_buffer(clean as usize) diff --git a/crates/perry-runtime/src/array/tests.rs b/crates/perry-runtime/src/array/tests.rs index e912b60872..a3b9512cc3 100644 --- a/crates/perry-runtime/src/array/tests.rs +++ b/crates/perry-runtime/src/array/tests.rs @@ -1991,3 +1991,66 @@ fn array_prototype_method_discriminator_separates_foreign_builtins() { "never obtained a GC-quiet view of Array.prototype / String.prototype" ); } + +/// The strict element-store fast lane (`try_strict_dense_number_store`) must +/// store exactly what the general path stores, for both the NaN-boxed +/// receiver codegen passes and a raw head, and must decline every shape it +/// cannot prove: out-of-range indices, tagged or NaN values. +#[test] +fn strict_dense_number_store_fast_lane_matches_the_general_path() { + use super::indexing::test_strict_dense_number_store as lane; + unsafe { + let mut arr = js_array_alloc(4); + for i in 0..3 { + arr = js_array_push_f64(arr, i as f64); + } + let boxed = crate::value::js_nanbox_pointer(arr as i64).to_bits() as *mut ArrayHeader; + + assert!( + lane(boxed, 1, 41.5), + "boxed receiver, plain number, in range" + ); + assert_eq!(js_array_get_f64(arr, 1), 41.5); + assert!(lane(arr, 2, -7.0), "raw receiver"); + assert_eq!(js_array_get_f64(arr, 2), -7.0); + + // An INT32 box stores its canonical double on this raw-f64 layout. + let boxed_int = f64::from_bits(crate::value::INT32_TAG | 12); + assert!(lane(boxed, 1, boxed_int), "INT32 box is a number"); + assert_eq!(js_array_get_f64(arr, 1).to_bits(), 12.0f64.to_bits()); + assert!(!lane(arr, 3, 1.0), "index == length is an extension"); + assert!( + !lane(arr, 0, f64::from_bits(crate::value::TAG_UNDEFINED)), + "tagged value" + ); + assert!( + !lane(arr, 0, f64::NAN), + "NaN keeps canonicalization on the general path" + ); + assert!(!lane(std::ptr::null_mut(), 0, 1.0), "null receiver"); + assert!( + !lane( + f64::from_bits(crate::value::TAG_UNDEFINED).to_bits() as *mut ArrayHeader, + 0, + 1.0 + ), + "non-pointer receiver" + ); + assert_eq!( + js_array_get_f64(arr, 0), + 0.0, + "declined stores leave the slot alone" + ); + assert_eq!((*arr).length, 3); + + // The public strict entry answers the same shape through the lane and + // still returns the live head. + let out = js_array_set_f64_extend_strict(boxed, 0, 9.0); + assert_eq!(out, arr); + assert_eq!(js_array_get_f64(arr, 0), 9.0); + // …and extension still goes through the general path. + let out = js_array_set_f64_extend_strict(boxed, 3, 3.0); + assert_eq!((*out).length, 4); + assert_eq!(js_array_get_f64(out, 3), 3.0); + } +} diff --git a/crates/perry-runtime/src/map.rs b/crates/perry-runtime/src/map.rs index 3b78bcda4f..3f2930040f 100644 --- a/crates/perry-runtime/src/map.rs +++ b/crates/perry-runtime/src/map.rs @@ -358,6 +358,9 @@ impl NumericIndex { if integer >= dense.base { let offset = integer as usize - dense.base as usize; if offset < dense.slots.len() { + // Authoritative for its span: a key inside the range was + // either copied here when the table was (re)built or + // inserted directly, so a miss is a definitive miss. let entry = dense.slots[offset]; return (entry != DENSE_NUMERIC_EMPTY).then_some(entry); } @@ -373,6 +376,23 @@ impl NumericIndex { fn insert(&mut self, key: NumericKey, entry_index: u32) { let integer = dense_integer_key(key); + // An integer inside the range table's span lives only there: the hash + // insert would be pure overhead on the sequential-id workloads the + // table exists for (`entityCommands.set(entityId, …)` per command). + // `rebuild_dense` carries these dense-only keys into a widened span + // and `remove` clears them here, so the hash index never needs them. + if let (Some(integer), Some(dense)) = (integer, self.dense.as_mut()) { + if integer >= dense.base { + let offset = integer as usize - dense.base as usize; + if offset < dense.slots.len() { + if dense.slots[offset] == DENSE_NUMERIC_EMPTY { + self.dense_key_count += 1; + } + dense.slots[offset] = entry_index; + return; + } + } + } let is_new = self.hashed.insert(key, entry_index).is_none(); if is_new && integer.is_some() { self.dense_key_count += 1; @@ -381,14 +401,7 @@ impl NumericIndex { let Some(integer) = integer else { return; }; - if let Some(dense) = self.dense.as_mut() { - if integer >= dense.base { - let offset = integer as usize - dense.base as usize; - if offset < dense.slots.len() { - dense.slots[offset] = entry_index; - return; - } - } + if self.dense.is_some() { self.maybe_expand_dense(integer); } else { self.maybe_initialize_dense(); @@ -396,26 +409,37 @@ impl NumericIndex { } fn remove(&mut self, key: &NumericKey) -> Option { - let removed = self.hashed.remove(key); - if removed.is_some() { - if let Some(integer) = dense_integer_key(*key) { - self.dense_key_count = self.dense_key_count.saturating_sub(1); - if let Some(dense) = self.dense.as_mut() { - if integer >= dense.base { - let offset = integer as usize - dense.base as usize; - if offset < dense.slots.len() { - dense.slots[offset] = DENSE_NUMERIC_EMPTY; - } + let integer = dense_integer_key(*key); + let mut removed = self.hashed.remove(key); + if let (Some(integer), Some(dense)) = (integer, self.dense.as_mut()) { + if integer >= dense.base { + let offset = integer as usize - dense.base as usize; + if offset < dense.slots.len() { + let entry = dense.slots[offset]; + if entry != DENSE_NUMERIC_EMPTY { + dense.slots[offset] = DENSE_NUMERIC_EMPTY; + // A key copied into the span at rebuild time is still + // in the hash index too; count it once either way. + removed = removed.or(Some(entry)); } } } } + if removed.is_some() && integer.is_some() { + self.dense_key_count = self.dense_key_count.saturating_sub(1); + } removed } fn clear(&mut self) { self.hashed.clear(); - self.dense = None; + // Keep the allocated span: `Map.clear()` followed by the same id + // population (a per-frame grouping map) would otherwise rebuild the + // table from scratch every cycle. The slots are reset, and the span + // still only widens through `maybe_expand_dense`'s density budget. + if let Some(dense) = self.dense.as_mut() { + dense.slots.fill(DENSE_NUMERIC_EMPTY); + } self.dense_key_count = 0; } @@ -504,10 +528,42 @@ impl NumericIndex { } } } + // Keys inserted straight into the previous span are not in the hash + // index; the new span always covers the old one, and the range table + // is authoritative, so its entries win over any stale hash copy. + if let Some(old) = self.dense.take() { + for (offset, &entry_index) in old.slots.iter().enumerate() { + if entry_index == DENSE_NUMERIC_EMPTY { + continue; + } + let integer = old.base as u64 + offset as u64; + if integer >= base as u64 { + let new_offset = (integer - base as u64) as usize; + if new_offset < slots.len() { + slots[new_offset] = entry_index; + continue; + } + } + // Outside the new span (cannot happen by construction, but a + // key must never be silently dropped): keep it in the hash. + self.hashed + .insert(NumericKey((integer as f64).to_bits()), entry_index); + } + } self.dense = Some(DenseNumericIndex { base, slots }); } } +/// `true` for an ordinary IEEE double that is neither NaN nor `±0`. Every +/// NaN-box tag shares the quiet-NaN prefix, so one mask separates a plain +/// number from every tagged value; the zero test removes the one pair of +/// distinct bit patterns (`+0`/`-0`) that SameValueZero identifies. +#[inline] +fn is_plain_nonzero_number_bits(bits: u64) -> bool { + const QNAN_PREFIX: u64 = 0x7FF8_0000_0000_0000; + (bits & QNAN_PREFIX) != QNAN_PREFIX && (bits & !(1u64 << 63)) != 0 +} + /// `true` if `bits` is a non-pointer JSValue (number, bool, undefined, /// null, or any NaN-tagged value that is NOT a string/heap pointer). /// We index only these in the side-table. @@ -1342,6 +1398,19 @@ pub(crate) unsafe fn find_key_index(map: *const MapHeader, key: f64) -> i32 { // Small maps: linear scan beats side-table dispatch. if size <= SIDE_TABLE_THRESHOLD { let entries = entries_ptr(map); + // A plain (untagged, non-NaN), non-zero number is SameValueZero-equal + // to an entry key exactly when the bits match: no tagged value can + // equal a number, and only `±0` / NaN break bit identity, so those + // (and every non-number) keep the general comparison below. + if is_plain_nonzero_number_bits(key_bits) { + for i in 0..size { + let entry_bits = ptr::read(entries.add((i as usize) * 2)).to_bits(); + if entry_bits == key_bits { + return i as i32; + } + } + return -1; + } for i in 0..size { let entry_key = ptr::read(entries.add((i as usize) * 2)); if jsvalue_eq(entry_key, key) { @@ -1838,13 +1907,69 @@ fn map_get_resolved(map: *const MapHeader, key: f64) -> f64 { if idx >= 0 { let entries = entries_ptr(map); - return ptr::read(entries.add((idx as usize) * 2 + 1)); + let value_slot = entries.add((idx as usize) * 2 + 1); + let value = ptr::read(value_slot); + return heal_forwarded_array_value(map, value_slot, value); } f64::from_bits(TAG_UNDEFINED) } } +/// Rewrite a Map value that still names an Array growth stub to the live head. +/// +/// `js_array_grow` preserves JavaScript identity by leaving a forwarding stub +/// at the old address, and codegen writes the grown head back only into the +/// binding it pushed through. A container that handed out the array — the +/// `componentData.get(type).push(v)` shape — keeps the stub, so every later +/// `get` returns it and every element access re-runs the tracked forwarding +/// resolver. That was the single largest runtime leaf on the ECS command +/// path. The stub and its target are both arrays the resolver validates, so +/// substituting the live head is unobservable (`===` already resolves +/// forwarding); the slot store goes through the ordinary external-slot +/// barrier. Non-pointers, non-arrays, and anything the cheap generation +/// classifier cannot place are returned untouched. +#[inline] +unsafe fn heal_forwarded_array_value( + map: *const MapHeader, + value_slot: *const f64, + value: f64, +) -> f64 { + let bits = value.to_bits(); + if bits & crate::value::TAG_MASK != crate::value::POINTER_TAG { + return value; + } + let raw = (bits & crate::value::POINTER_MASK) as usize; + if raw < crate::gc::GC_HEADER_SIZE + || raw % std::mem::align_of::() != 0 + || !crate::value::addr_class::is_plausible_heap_addr(raw) + || matches!( + crate::arena::classify_heap_generation(raw), + crate::arena::HeapGeneration::Unknown + ) + { + return value; + } + let header = (raw - crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader; + if (*header).obj_type != crate::gc::GC_TYPE_ARRAY + || (*header).gc_flags & crate::gc::GC_FLAG_FORWARDED == 0 + { + return value; + } + let live = crate::array::clean_arr_ptr(raw as *const crate::array::ArrayHeader); + if live.is_null() || live as usize == raw { + return value; + } + let live_value = crate::value::js_nanbox_pointer(live as i64); + // GC_STORE_AUDIT(EXTERNAL_BARRIERED): map value slot uses the shared external-slot helper. + crate::gc::runtime_store_external_jsvalue_slot( + map as usize, + value_slot as usize, + live_value.to_bits(), + ); + live_value +} + /// Fast `Map.get`/`ReadonlyMap.get` for a declared structural receiver. /// /// A TypeScript collection annotation does not prove Perry's native layout. @@ -2973,8 +3098,18 @@ mod tests { ); js_map_clear(map); - assert_eq!(test_map_dense_numeric_index_range(map), None); + // The span survives `clear()` (a per-frame grouping map repopulates + // the same ids), but every slot is reset: nothing is found and the + // next population starts from zero density. + assert_eq!(test_map_dense_numeric_index_range(map), Some((base, len))); assert_eq!(js_map_size(map), 0); + for key in 1_024..1_040 { + assert_eq!(js_map_has(map, key as f64), 0); + } + js_map_set(map, 1_030.0, 5.0); + assert_eq!(js_map_get(map, 1_030.0), 5.0); + assert_eq!(js_map_has(map, 1_031.0), 0); + assert_eq!(js_map_size(map), 1); } #[test] @@ -3094,4 +3229,38 @@ mod tests { pointer_keys[1].to_bits() ); } + + /// A Map value that names an Array growth stub is healed to the live head + /// on `get`, and the entry itself is rewritten so later reads are direct. + #[test] + fn map_get_heals_a_forwarded_array_value() { + unsafe { + let map = js_map_alloc(4); + let mut arr = crate::array::js_array_alloc(2); + let stub_value = crate::value::js_nanbox_pointer(arr as i64); + js_map_set(map, 7.0, stub_value); + // Grow past the initial capacity so the original head becomes a + // forwarding stub. + for i in 0..64 { + arr = crate::array::js_array_push_f64(arr, i as f64); + } + assert_ne!( + arr as usize, + stub_value.to_bits() as usize & 0xFFFF_FFFF_FFFF + ); + let got = js_map_get(map, 7.0); + assert_eq!( + got.to_bits() & 0x0000_FFFF_FFFF_FFFF, + arr as u64, + "get must answer the live head" + ); + let entries = entries_ptr(map); + assert_eq!( + ptr::read(entries.add(1)).to_bits(), + got.to_bits(), + "the entry slot is rewritten to the live head" + ); + assert_eq!(crate::array::js_array_get_f64(arr, 63), 63.0); + } + } } diff --git a/crates/perry/src/commands/compile/run_pipeline.rs b/crates/perry/src/commands/compile/run_pipeline.rs index e7078ff2e9..562cdb9fcc 100644 --- a/crates/perry/src/commands/compile/run_pipeline.rs +++ b/crates/perry/src/commands/compile/run_pipeline.rs @@ -1111,6 +1111,223 @@ pub fn run_with_parse_cache( } } + // Resolve imported and generic type-alias references in every module's + // type positions (`perry_hir::type_alias_resolve`). Per-module lowering + // only settles a non-generic, same-module alias; `import type { EntityId } + // from "../entity"` and `EntityId` stayed `Named`/`Generic` and erased + // the branded-primitive shape they spell. Exported aliases are keyed by + // (defining path, exported name) and followed through barrels exactly like + // the enum fix-up above, so a local binding resolves only to the alias its + // import actually names. Alias bodies are closed against their own + // module's scope before any consumer is rewritten. + { + let mut exported_aliases: BTreeMap<(String, String), perry_hir::AliasDef> = BTreeMap::new(); + for (path, hir_module) in &ctx.native_modules { + let path_str = path.to_string_lossy().to_string(); + for alias in &hir_module.type_aliases { + if alias.is_exported { + exported_aliases.insert( + (path_str.clone(), alias.name.clone()), + perry_hir::AliasDef { + params: alias.type_params.clone(), + ty: alias.ty.clone(), + }, + ); + } + } + } + loop { + let mut new_entries: Vec<((String, String), perry_hir::AliasDef)> = Vec::new(); + for (path, hir_module) in &ctx.native_modules { + let path_str = path.to_string_lossy().to_string(); + for export in &hir_module.exports { + let re_export = match export { + perry_hir::Export::ExportAll { source } => Some((source.as_str(), None)), + perry_hir::Export::ReExport { + source, + imported, + exported, + } => Some(( + source.as_str(), + Some((imported.as_str(), exported.as_str())), + )), + _ => None, + }; + let Some((source, names)) = re_export else { + continue; + }; + let Some((resolved_source, _)) = resolve_import( + source, + path, + &ctx.project_root, + &ctx.compile_packages, + &ctx.compile_package_dirs, + ) else { + continue; + }; + let source_path_str = resolved_source.to_string_lossy().to_string(); + for ((src_path, alias_name), def) in &exported_aliases { + if src_path != &source_path_str { + continue; + } + let (propagate, exported_name) = match names { + Some((imported, exported)) => { + (alias_name == imported, exported.to_string()) + } + None => (true, alias_name.clone()), + }; + if propagate { + let key = (path_str.clone(), exported_name); + if !exported_aliases.contains_key(&key) + && !new_entries.iter().any(|(k, _)| k == &key) + { + new_entries.push((key, def.clone())); + } + } + } + } + } + if new_entries.is_empty() { + break; + } + for (key, def) in new_entries { + exported_aliases.insert(key, def); + } + } + + // Per-module scope tables: own declarations plus imported bindings. + let mut tables: BTreeMap = BTreeMap::new(); + for (path, hir_module) in &ctx.native_modules { + let mut table = perry_hir::AliasTable::new(); + for alias in &hir_module.type_aliases { + table.insert( + alias.name.clone(), + perry_hir::AliasDef { + params: alias.type_params.clone(), + ty: alias.ty.clone(), + }, + ); + } + for import in &hir_module.imports { + if import.module_kind != perry_hir::ModuleKind::NativeCompiled { + continue; + } + let Some(resolved_path) = &import.resolved_path else { + continue; + }; + for spec in &import.specifiers { + let (local_name, exported_name) = match spec { + perry_hir::ImportSpecifier::Named { imported, local } => { + (local.clone(), imported.clone()) + } + perry_hir::ImportSpecifier::Default { local } => { + (local.clone(), local.clone()) + } + perry_hir::ImportSpecifier::Namespace { .. } => continue, + }; + if let Some(def) = exported_aliases.get(&(resolved_path.clone(), exported_name)) + { + table.entry(local_name).or_insert_with(|| def.clone()); + } + } + } + if !table.is_empty() { + tables.insert(path.clone(), table); + } + } + + // Close alias bodies in their defining scope (`ComponentId` spells + // `EntityId`, which spells `number`), then refresh the + // imported copies. Chains are short; the bound only guards a cycle. + for _ in 0..8 { + let mut changed = false; + let mut closed: BTreeMap<(String, String), perry_hir::AliasDef> = BTreeMap::new(); + for (path, hir_module) in &ctx.native_modules { + let Some(table) = tables.get(path) else { + continue; + }; + let path_str = path.to_string_lossy().to_string(); + for alias in &hir_module.type_aliases { + let Some(def) = table.get(&alias.name) else { + continue; + }; + let resolved = perry_hir::type_alias_resolve::resolve_type(&def.ty, table); + if resolved != def.ty { + changed = true; + } + closed.insert( + (path_str.clone(), alias.name.clone()), + perry_hir::AliasDef { + params: def.params.clone(), + ty: resolved, + }, + ); + } + } + if !changed { + break; + } + for (key, def) in &closed { + if let Some(existing) = exported_aliases.get_mut(key) { + *existing = def.clone(); + } + } + // Re-exported copies share the defining alias's body. + let mut by_body: Vec<((String, String), perry_hir::AliasDef)> = Vec::new(); + for (key, def) in &exported_aliases { + for (ckey, cdef) in &closed { + if ckey != key && def.params == cdef.params && def.ty == cdef.ty { + by_body.push((key.clone(), cdef.clone())); + } + } + } + for (key, def) in by_body { + exported_aliases.insert(key, def); + } + for (path, hir_module) in &ctx.native_modules { + let Some(table) = tables.get_mut(path) else { + continue; + }; + let path_str = path.to_string_lossy().to_string(); + for alias in &hir_module.type_aliases { + if let Some(def) = closed.get(&(path_str.clone(), alias.name.clone())) { + table.insert(alias.name.clone(), def.clone()); + } + } + for import in &hir_module.imports { + if import.module_kind != perry_hir::ModuleKind::NativeCompiled { + continue; + } + let Some(resolved_path) = &import.resolved_path else { + continue; + }; + for spec in &import.specifiers { + let (local_name, exported_name) = match spec { + perry_hir::ImportSpecifier::Named { imported, local } => { + (local.clone(), imported.clone()) + } + perry_hir::ImportSpecifier::Default { local } => { + (local.clone(), local.clone()) + } + perry_hir::ImportSpecifier::Namespace { .. } => continue, + }; + if let Some(def) = + exported_aliases.get(&(resolved_path.clone(), exported_name)) + { + table.insert(local_name, def.clone()); + } + } + } + } + } + + for (path, table) in &tables { + if let Some(hir_module) = ctx.native_modules.get_mut(path) { + perry_hir::resolve_type_aliases_in_module(hir_module, table); + } + } + } + // Collect all non-generic type aliases from all modules. // These are passed to each module's compiler so type_to_abi can resolve // Named("BlockTag") -> Union([...]) for correct ABI types in function signatures. From 3477802c0be3f67e8bb948ae4a389317b4477a61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 11:55:17 +0200 Subject: [PATCH 02/23] perf(runtime): clean_arr_ptr fast lane via the cached generation classifier; cheaper Map-get forwarding heal Claude-Session: https://claude.ai/code/session_01FUvFrRNZyc5qknBiJbYbby --- crates/perry-runtime/src/array/header.rs | 35 ++++++++++++++++++++++++ crates/perry-runtime/src/map.rs | 8 +++--- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/crates/perry-runtime/src/array/header.rs b/crates/perry-runtime/src/array/header.rs index 513dcb7ec6..a98901325f 100644 --- a/crates/perry-runtime/src/array/header.rs +++ b/crates/perry-runtime/src/array/header.rs @@ -638,6 +638,41 @@ pub(crate) fn clean_arr_ptr(arr: *const ArrayHeader) -> *const ArrayHeader { if !crate::value::addr_class::is_plausible_heap_addr(cleaned as usize) { return std::ptr::null(); } + // Fast lane for the overwhelmingly common receiver: a live plain array on + // a page the arena owns. The tracked-header classifier below re-derives + // page ownership, type-table membership, size and storage consistency on + // every call, and every runtime array entry point (push, pop, length, + // some, length assignment, …) funnels through here — it was the largest + // runtime leaf on the ECS command path. The cached generation classifier + // is the same page-ownership answer the write barrier relies on; with an + // in-band, aligned address on an owned page, the header's type, forwarding + // and arena bits settle the ordinary case. Forwarded stubs, lazy arrays, + // malloc-backed objects, registered buffers/typed arrays and any + // inconsistent header keep the full resolver. + { + let addr = cleaned as usize; + if addr >= crate::gc::GC_HEADER_SIZE + && addr % std::mem::align_of::() == 0 + && !matches!( + crate::arena::classify_heap_generation(addr), + crate::arena::HeapGeneration::Unknown + ) + { + // SAFETY: the address is on an arena page this process owns and + // is header-aligned; the header word precedes every arena block. + let header = (addr - crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader; + let (obj_type, gc_flags) = unsafe { ((*header).obj_type, (*header).gc_flags) }; + if obj_type == crate::gc::GC_TYPE_ARRAY + && gc_flags & crate::gc::GC_FLAG_FORWARDED == 0 + && gc_flags & crate::gc::GC_FLAG_ARENA != 0 + { + let hdr = unsafe { &*cleaned }; + if hdr.length <= hdr.capacity && hdr.length <= 100_000_000 { + return cleaned; + } + } + } + } // Issue #233: follow GC_FLAG_FORWARDED forwarding chains. When // an array grows (js_array_grow) we install a forwarding pointer // at the OLD location so any stale reference — e.g. an async diff --git a/crates/perry-runtime/src/map.rs b/crates/perry-runtime/src/map.rs index 3f2930040f..e05d5a8c3a 100644 --- a/crates/perry-runtime/src/map.rs +++ b/crates/perry-runtime/src/map.rs @@ -1943,13 +1943,13 @@ unsafe fn heal_forwarded_array_value( if raw < crate::gc::GC_HEADER_SIZE || raw % std::mem::align_of::() != 0 || !crate::value::addr_class::is_plausible_heap_addr(raw) - || matches!( - crate::arena::classify_heap_generation(raw), - crate::arena::HeapGeneration::Unknown - ) { return value; } + // A POINTER_TAG value in a Map entry was stored by the runtime and is kept + // alive by the entry itself, so its header can be read after the band + // check, exactly as codegen's inline array guards read it. Only the + // forwarding bit is decided here; the full resolver validates the target. let header = (raw - crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader; if (*header).obj_type != crate::gc::GC_TYPE_ARRAY || (*header).gc_flags & crate::gc::GC_FLAG_FORWARDED == 0 From 8b0a2b57c92dc9598d2658b6acdbe174504edb59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 12:19:23 +0200 Subject: [PATCH 03/23] perf: dedupe resolved alias unions, 16-way page-generation cache, single receiver resolution in Array.some Claude-Session: https://claude.ai/code/session_01FUvFrRNZyc5qknBiJbYbby --- crates/perry-hir/src/type_alias_resolve.rs | 45 ++++++++++++++++--- crates/perry-runtime/src/arena/page_meta.rs | 7 ++- crates/perry-runtime/src/array/indexing.rs | 15 +++++++ .../perry-runtime/src/array/iter_methods.rs | 8 +++- crates/perry-runtime/src/array/mod.rs | 11 ++--- 5 files changed, 73 insertions(+), 13 deletions(-) diff --git a/crates/perry-hir/src/type_alias_resolve.rs b/crates/perry-hir/src/type_alias_resolve.rs index e302c97b53..dd01dc4e56 100644 --- a/crates/perry-hir/src/type_alias_resolve.rs +++ b/crates/perry-hir/src/type_alias_resolve.rs @@ -98,12 +98,23 @@ fn resolve_type_inner(ty: &Type, table: &AliasTable, depth: usize) -> Type { .collect(), ), Type::Promise(inner) => Type::Promise(Box::new(resolve_type_inner(inner, table, depth))), - Type::Union(types) => Type::Union( - types - .iter() - .map(|t| resolve_type_inner(t, table, depth)) - .collect(), - ), + Type::Union(types) => { + // Two branded aliases of one primitive (`EntityId | ComponentId`) + // resolve to the same member; a union of identical members is + // that member, so the binding gets the primitive's lowering. + let mut resolved: Vec = Vec::with_capacity(types.len()); + for t in types { + let t = resolve_type_inner(t, table, depth); + if !resolved.contains(&t) { + resolved.push(t); + } + } + if resolved.len() == 1 { + resolved.pop().expect("one member") + } else { + Type::Union(resolved) + } + } Type::Function(f) => Type::Function(crate::types::FunctionType { params: f .params @@ -564,6 +575,28 @@ mod tests { ); } + #[test] + fn a_union_of_branded_aliases_of_one_primitive_is_that_primitive() { + let table = branded_table(); + assert_eq!( + resolve_type( + &Type::Union(vec![ + Type::Named("EntityId".into()), + Type::Named("ComponentId".into()), + ]), + &table + ), + Type::Number + ); + assert_eq!( + resolve_type( + &Type::Union(vec![Type::Named("EntityId".into()), Type::Any]), + &table + ), + Type::Union(vec![Type::Number, Type::Any]) + ); + } + #[test] fn open_instantiations_and_unknown_names_keep_their_spelling() { let table = branded_table(); diff --git a/crates/perry-runtime/src/arena/page_meta.rs b/crates/perry-runtime/src/arena/page_meta.rs index cf115fcfd7..4946dd7c51 100644 --- a/crates/perry-runtime/src/arena/page_meta.rs +++ b/crates/perry-runtime/src/arena/page_meta.rs @@ -132,7 +132,12 @@ impl PageGenerationCache { /// the barrier's working set (child, parent, array payload, header) with room /// to spare; it is still a fixed-size array probed linearly, so a hit is a few /// compares off one cache line. -const PAGE_GENERATION_CACHE_WAYS: usize = 4; +// 16 ways: the ECS command path touches a nursery page, several tenured +// 40 KB column arrays and the pooled command arrays per iteration; four ways +// thrashed (`classify_heap_generation_uncached` was 1.5% of samples) once the +// array-receiver fast lanes started asking this cache instead of the tracked +// classifier. The lookup is a short linear scan, so widening is nearly free. +const PAGE_GENERATION_CACHE_WAYS: usize = 16; /// Small direct-probed cache in front of [`PageGenerationMap`]. /// diff --git a/crates/perry-runtime/src/array/indexing.rs b/crates/perry-runtime/src/array/indexing.rs index 725ff798a0..cf98b83305 100644 --- a/crates/perry-runtime/src/array/indexing.rs +++ b/crates/perry-runtime/src/array/indexing.rs @@ -234,6 +234,21 @@ pub(crate) fn array_iteration_is_exotic(arr: *const ArrayHeader) -> bool { // precedes every operation that could allocate or safepoint. The // compatible header-less receivers exited above. let flags = unsafe { array_object_flags_resolved(arr) }; + unsafe { array_iteration_is_exotic_resolved(arr, flags) } +} + +/// [`array_iteration_is_exotic`] for a caller that already resolved the live +/// plain-array head, excluded Buffer/TypedArray receivers, and owns the header +/// word: the policy tests without a second receiver resolution and registry +/// probe (the iteration helpers call this once per invocation). +/// +/// # Safety +/// +/// `arr` and `flags` must satisfy [`array_object_flags_resolved`]'s contract. +pub(crate) unsafe fn array_iteration_is_exotic_resolved( + arr: *const ArrayHeader, + flags: u16, +) -> bool { if flags & crate::gc::OBJ_FLAG_ARRAY_DESCRIPTORS != 0 { return true; } diff --git a/crates/perry-runtime/src/array/iter_methods.rs b/crates/perry-runtime/src/array/iter_methods.rs index 4ab5e99859..ca0dc56c02 100644 --- a/crates/perry-runtime/src/array/iter_methods.rs +++ b/crates/perry-runtime/src/array/iter_methods.rs @@ -920,9 +920,15 @@ pub extern "C" fn js_array_some_captureless( unsafe { std::mem::transmute(callback_func) }; unsafe { let length = (*arr).length; + // SAFETY: `normalize_array_receiver` returned this live plain-array + // head and the registry exits above excluded Buffer/TypedArray + // receivers; nothing allocates before the flag read. + let exotic = crate::array::array_iteration_is_exotic_resolved( + arr, + crate::array::array_object_flags_resolved(arr), + ); let scope = crate::gc::RuntimeHandleScope::new(); let rooted = RootedIterArray::new(&scope, arr); - let exotic = crate::array::array_iteration_is_exotic(arr); for i in 0..length as usize { let element = if exotic { diff --git a/crates/perry-runtime/src/array/mod.rs b/crates/perry-runtime/src/array/mod.rs index 8eabcad213..01bb88733d 100644 --- a/crates/perry-runtime/src/array/mod.rs +++ b/crates/perry-runtime/src/array/mod.rs @@ -122,11 +122,12 @@ pub use self::immutable::{ #[cfg(test)] pub(crate) use self::indexing::test_keys_array_slot_fallbacks; pub(crate) use self::indexing::{ - array_has_own_index, array_iteration_is_exotic, array_proto_iterator_modified, - array_prototype_has_index_flag, array_spec_get, array_spec_has_index, array_spec_set, - invalidate_array_index_fast_path, keys_array_len_capped_to_capacity, keys_array_slot, - note_array_proto_iterator_write, note_object_prototype_index_write, - object_prototype_has_index_flag, PERRY_ARRAY_INDEX_FAST_PATH_INVALIDATED, + array_has_own_index, array_iteration_is_exotic, array_iteration_is_exotic_resolved, + array_proto_iterator_modified, array_prototype_has_index_flag, array_spec_get, + array_spec_has_index, array_spec_set, invalidate_array_index_fast_path, + keys_array_len_capped_to_capacity, keys_array_slot, note_array_proto_iterator_write, + note_object_prototype_index_write, object_prototype_has_index_flag, + PERRY_ARRAY_INDEX_FAST_PATH_INVALIDATED, }; pub use self::indexing::{ js_array_get_element, js_array_get_element_f64, js_array_get_f64, js_array_get_f64_unchecked, From 0588359167bfb7f4fea8711ace37a050202b3f36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 12:35:37 +0200 Subject: [PATCH 04/23] perf(arena): keep the 4-way page-generation cache (16 ways regressed 8.6%) Claude-Session: https://claude.ai/code/session_01FUvFrRNZyc5qknBiJbYbby --- crates/perry-runtime/src/arena/page_meta.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/crates/perry-runtime/src/arena/page_meta.rs b/crates/perry-runtime/src/arena/page_meta.rs index 4946dd7c51..ad344f7f82 100644 --- a/crates/perry-runtime/src/arena/page_meta.rs +++ b/crates/perry-runtime/src/arena/page_meta.rs @@ -132,12 +132,11 @@ impl PageGenerationCache { /// the barrier's working set (child, parent, array payload, header) with room /// to spare; it is still a fixed-size array probed linearly, so a hit is a few /// compares off one cache line. -// 16 ways: the ECS command path touches a nursery page, several tenured -// 40 KB column arrays and the pooled command arrays per iteration; four ways -// thrashed (`classify_heap_generation_uncached` was 1.5% of samples) once the -// array-receiver fast lanes started asking this cache instead of the tracked -// classifier. The lookup is a short linear scan, so widening is nearly free. -const PAGE_GENERATION_CACHE_WAYS: usize = 16; +// Four ways, measured: widening to 16 removed the 1.5% of misses the ECS +// command path took (`classify_heap_generation_uncached`) but the longer +// linear scan cost every barrier and array-receiver classification more than +// that — an 8.6% regression on the same row (0/7 pairs). Keep the scan short. +const PAGE_GENERATION_CACHE_WAYS: usize = 4; /// Small direct-probed cache in front of [`PageGenerationMap`]. /// From fa91aa432d01ccf54872411a97bc6342ab3927c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 13:05:01 +0200 Subject: [PATCH 05/23] =?UTF-8?q?perf(runtime):=20js=5Farray=5Flength=20pl?= =?UTF-8?q?ain-array=20fast=20lane;=20inline=20old=E2=86=92young=20slots?= =?UTF-8?q?=20skip=20the=20redundant=20page=20classification?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01FUvFrRNZyc5qknBiJbYbby --- crates/perry-runtime/src/array/indexing.rs | 41 ++++++++++++++++++++++ crates/perry-runtime/src/gc/barrier/mod.rs | 15 +++++++- 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/crates/perry-runtime/src/array/indexing.rs b/crates/perry-runtime/src/array/indexing.rs index cf98b83305..ca79f7a02b 100644 --- a/crates/perry-runtime/src/array/indexing.rs +++ b/crates/perry-runtime/src/array/indexing.rs @@ -647,6 +647,47 @@ static KEEP_ARRAY_LENGTH: extern "C" fn(*const ArrayHeader) -> u32 = js_array_le #[no_mangle] pub extern "C" fn js_array_length(arr: *const ArrayHeader) -> u32 { + // Fast lane: a live plain array on an arena page. Every dynamic `.length` + // read and every native push lowering (which re-reads the length for the + // result) lands here; the proxy, Set/Map, object and subclass arms below + // all begin with probes this receiver cannot satisfy. A proxy id sits in + // the handle band and a Set/Map/object header has another type, so the + // lane's own checks exclude them. + { + let bits = arr as u64; + let top16 = bits >> 48; + let raw = if top16 >= 0x7FF8 { + if top16 == (crate::value::POINTER_TAG >> 48) { + (bits & crate::value::POINTER_MASK) as usize + } else { + 0 + } + } else { + bits as usize + }; + if raw >= crate::gc::GC_HEADER_SIZE + && raw % std::mem::align_of::() == 0 + && crate::value::addr_class::is_plausible_heap_addr(raw) + && !matches!( + crate::arena::classify_heap_generation(raw), + crate::arena::HeapGeneration::Unknown + ) + { + // SAFETY: owned arena page, header-aligned; the header word + // precedes every arena block. + let header = (raw - crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader; + let (obj_type, gc_flags) = unsafe { ((*header).obj_type, (*header).gc_flags) }; + if obj_type == crate::gc::GC_TYPE_ARRAY + && gc_flags & crate::gc::GC_FLAG_FORWARDED == 0 + && gc_flags & crate::gc::GC_FLAG_ARENA != 0 + { + let hdr = unsafe { &*(raw as *const ArrayHeader) }; + if hdr.length <= hdr.capacity { + return hdr.length; + } + } + } + } // #5135: a Proxy typed (statically) as an array (immer drafts) reaches here // with the masked proxy id. Read `length` through the proxy `get` trap // rather than deref-ing the id as an `ArrayHeader`. diff --git a/crates/perry-runtime/src/gc/barrier/mod.rs b/crates/perry-runtime/src/gc/barrier/mod.rs index 005edd4091..f867b217a4 100644 --- a/crates/perry-runtime/src/gc/barrier/mod.rs +++ b/crates/perry-runtime/src/gc/barrier/mod.rs @@ -1325,7 +1325,7 @@ pub(super) fn write_barrier_decoded_parent( let inserted = if external_slot { remember_old_to_young_external_slot(parent_addr, slot_addr) } else { - remember_old_to_young_slot(parent_addr, slot_addr) + remember_old_to_young_inline_slot(parent_addr, slot_addr) }; if inserted { bump_write_barrier_trace_counter(BarrierTraceCounter::NewInserts); @@ -1639,6 +1639,19 @@ pub(super) fn decode_heap_addr(bits: u64) -> usize { } } +/// [`remember_old_to_young_slot`] for a slot INSIDE the parent's own block. +/// `barrier_parent_needs_remembering` has just classified the parent as Old, +/// and an inline slot lies in the same allocation, so its page is on the same +/// registered Old range: the slot's own classification would answer the +/// same thing and was one of three page lookups per old→young store. +#[inline] +pub(super) fn remember_old_to_young_inline_slot(parent_addr: usize, slot_addr: usize) -> bool { + if slot_addr != 0 && slot_addr >= parent_addr { + return mark_dirty_old_page(crate::arena::generation_page_for_addr(slot_addr)); + } + remember_old_to_young_slot(parent_addr, slot_addr) +} + pub(super) fn remember_old_to_young_slot(parent_addr: usize, slot_addr: usize) -> bool { if slot_addr != 0 && matches!( From 8b1caf5c056318d67cb59d7bc725460f761db669 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 13:07:35 +0200 Subject: [PATCH 06/23] perf(gc): build class key arrays under ImmortalLayoutScope so they never arm per-object layouts Claude-Session: https://claude.ai/code/session_01FUvFrRNZyc5qknBiJbYbby --- crates/perry-runtime/src/object/alloc.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/perry-runtime/src/object/alloc.rs b/crates/perry-runtime/src/object/alloc.rs index 482e3af632..e16b01a218 100644 --- a/crates/perry-runtime/src/object/alloc.rs +++ b/crates/perry-runtime/src/object/alloc.rs @@ -435,6 +435,15 @@ pub extern "C" fn js_build_class_keys_array( .filter(|s| !s.is_empty()) .collect(); let num_keys = keys.len(); + // This array is long-lived and never dies. Without the scope, the per-slot + // notes below mint a per-object pointer mask for any class with enough + // keys, which arms `PERRY_PER_OBJECT_LAYOUTS_ANY` and puts the address + // filter probe on EVERY later allocation in the program (measured as 3% + // of an allocation-heavy ECS row: `layout_forget_object` from each object + // literal). Under the scope the notes settle on the tag-checked scan, and + // `layout_init_all_pointer_slots` below records the final all-pointer + // layout anyway. + let _immortal = crate::gc::ImmortalLayoutScope::new(); // Issue #179: the keys_array and its string elements are shape-cache // resident for the program's lifetime (anchored by // `scan_shape_cache_roots`). Route them through the longlived arena From b3856ffff408dad1c4ddb7522c2d76d60571167f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 13:37:07 +0200 Subject: [PATCH 07/23] perf(transform): beta-reduce called arrow-literal locals and drop dead default guards after inlining Claude-Session: https://claude.ai/code/session_01FUvFrRNZyc5qknBiJbYbby --- .../src/closure_local_inline.rs | 780 ++++++++++++++++++ crates/perry-transform/src/lib.rs | 2 + 2 files changed, 782 insertions(+) create mode 100644 crates/perry-transform/src/closure_local_inline.rs diff --git a/crates/perry-transform/src/closure_local_inline.rs b/crates/perry-transform/src/closure_local_inline.rs new file mode 100644 index 0000000000..ee8a7d0eda --- /dev/null +++ b/crates/perry-transform/src/closure_local_inline.rs @@ -0,0 +1,780 @@ +//! Beta-reduce closure-literal locals that are only ever called. +//! +//! Inlining a helper that takes a callback leaves this shape behind: +//! +//! ```text +//! let exists = (id) => this.exists(id); // the caller's argument +//! if (exists === undefined) exists = () => true; // the callee's default +//! … +//! if (!exists(entityId)) throw … // the callee's only use +//! ``` +//! +//! Every call of `exists` still allocates the closure, dispatches through +//! `js_closure_call1`, and saves/restores the implicit `this` around it — on +//! the ECS `world.set` path that was three runtime calls per invocation for a +//! callback whose body is one expression. Nothing about the closure is +//! observable: it never escapes, is never compared, and is never reassigned +//! (the default guard cannot fire, because a closure literal is not +//! `undefined`). +//! +//! This pass runs after the inliner. For each statement list it: +//! +//! 1. removes `if (x === undefined) x = …` guards whose binding was +//! initialized with a literal that is never `undefined` and is written +//! nowhere else — the inliner's default-parameter expansion after a +//! supplied argument; +//! 2. finds `let f = ` where the arrow is synchronous, has +//! only plain parameters, no mutable captures, no `new.target` capture, +//! and a single `return ` body, and where every use of `f` in the +//! rest of the list is a direct call with exactly the declared number of +//! trivial arguments (locals, globals, literals — evaluated once either +//! way, so substitution preserves order and count of effects); +//! 3. replaces each such call with the arrow's return expression, parameters +//! substituted by the arguments, and deletes the `let`. +//! +//! Because the arrow is lexically scoped, `this`, captured locals and +//! `arguments` inside its body already denote the enclosing function's +//! bindings, so the substituted expression is valid exactly where the call +//! was. Any other use — a read that is not a callee, a write, a capture by a +//! nested closure, a self-reference — leaves the binding untouched. + +use std::collections::HashMap; + +use perry_hir::types::LocalId; +use perry_hir::walker::{walk_expr_children, walk_expr_children_mut}; +use perry_hir::{CompareOp, Expr, Function, Module, Stmt}; + +use crate::inline::substitute_locals; + +pub fn run(module: &mut Module) { + let mut next_local_id = crate::generator::compute_max_local_id(module).saturating_add(1); + for f in &mut module.functions { + run_function(f, &mut next_local_id); + } + for c in &mut module.classes { + if let Some(ctor) = &mut c.constructor { + run_function(ctor, &mut next_local_id); + } + for m in &mut c.methods { + run_function(m, &mut next_local_id); + } + for m in &mut c.static_methods { + run_function(m, &mut next_local_id); + } + for (_, g) in &mut c.getters { + run_function(g, &mut next_local_id); + } + for (_, s) in &mut c.setters { + run_function(s, &mut next_local_id); + } + } +} + +fn run_function(f: &mut Function, next_local_id: &mut LocalId) { + // The async/generator transforms rewrite these bodies into state machines + // whose locals are boxed cells; keep the shapes they expect. + if f.is_async || f.is_generator { + return; + } + process_stmts(&mut f.body, next_local_id); +} + +fn process_stmts(stmts: &mut Vec, next_local_id: &mut LocalId) { + // Inner lists and closure bodies first, so a nested helper is reduced in + // its own scope before the enclosing list is examined. + for s in stmts.iter_mut() { + for inner in nested_stmt_lists(s) { + process_stmts(inner, next_local_id); + } + for_each_expr_in_stmt_mut(s, &mut |e| process_closure_bodies(e, next_local_id)); + } + + remove_dead_default_guards(stmts); + + let mut i = 0; + while i < stmts.len() { + let candidate = match &stmts[i] { + Stmt::Let { + id, + init: Some(init), + .. + } => arrow_candidate(*id, init), + _ => None, + }; + let Some((id, params, body_expr)) = candidate else { + i += 1; + continue; + }; + let mut uses = Uses::default(); + for s in &stmts[i + 1..] { + collect_uses_in_stmt(s, id, &mut uses); + } + if uses.other || uses.calls == 0 || uses.bad_arity { + i += 1; + continue; + } + for s in &mut stmts[i + 1..] { + for_each_expr_in_stmt_mut(s, &mut |e| { + rewrite_calls(e, id, ¶ms, &body_expr, next_local_id) + }); + } + stmts.remove(i); + // Do not advance: the list shifted, and the statement now at `i` may + // itself be a candidate. + } +} + +fn process_closure_bodies(expr: &mut Expr, next_local_id: &mut LocalId) { + if let Expr::Closure { body, .. } = expr { + process_stmts(body, next_local_id); + } + walk_expr_children_mut(expr, &mut |child| { + process_closure_bodies(child, next_local_id) + }); +} + +/// `let f = (a, b) => ` with the admission rules from the module doc. +/// Returns the binding, its parameter ids, and the return expression. +fn arrow_candidate(id: LocalId, init: &Expr) -> Option<(LocalId, Vec, Expr)> { + let Expr::Closure { + params, + body, + captures, + mutable_captures, + captures_new_target, + is_arrow, + is_async, + is_generator, + .. + } = init + else { + return None; + }; + if !*is_arrow + || *is_async + || *is_generator + || *captures_new_target + || !mutable_captures.is_empty() + || captures.contains(&id) + || params + .iter() + .any(|p| p.default.is_some() || p.is_rest || p.arguments_object.is_some()) + { + return None; + } + let [Stmt::Return(Some(expr))] = body.as_slice() else { + return None; + }; + Some((id, params.iter().map(|p| p.id).collect(), expr.clone())) +} + +/// An argument that substitution may duplicate or drop without changing +/// the program's effects: a binding read or a literal. +fn is_trivial_expr(expr: &Expr) -> bool { + matches!( + expr, + Expr::Integer(_) + | Expr::Number(_) + | Expr::Bool(_) + | Expr::String(_) + | Expr::Null + | Expr::Undefined + | Expr::LocalGet(_) + | Expr::GlobalGet(_) + ) +} + +#[derive(Default)] +struct Uses { + calls: usize, + bad_arity: bool, + other: bool, +} + +fn collect_uses_in_stmt(stmt: &Stmt, id: LocalId, uses: &mut Uses) { + for_each_expr_in_stmt(stmt, &mut |e| collect_uses(e, id, uses)); + for inner in nested_stmt_lists_ref(stmt) { + for s in inner { + collect_uses_in_stmt(s, id, uses); + } + } +} + +fn collect_uses(expr: &Expr, id: LocalId, uses: &mut Uses) { + match expr { + Expr::Call { callee, args, .. } if matches!(callee.as_ref(), Expr::LocalGet(x) if *x == id) => + { + uses.calls += 1; + if !args.iter().all(is_trivial_expr) { + uses.bad_arity = true; + } + for a in args { + collect_uses(a, id, uses); + } + } + Expr::LocalGet(x) if *x == id => uses.other = true, + Expr::LocalSet(x, value) => { + if *x == id { + uses.other = true; + } + collect_uses(value, id, uses); + } + Expr::Closure { captures, body, .. } => { + if captures.contains(&id) { + uses.other = true; + } + for s in body { + collect_uses_in_stmt(s, id, uses); + } + walk_expr_children(expr, &mut |child| collect_uses(child, id, uses)); + } + _ => walk_expr_children(expr, &mut |child| collect_uses(child, id, uses)), + } +} + +fn rewrite_calls( + expr: &mut Expr, + id: LocalId, + params: &[LocalId], + body_expr: &Expr, + next_local_id: &mut LocalId, +) { + let is_target = matches!(expr, Expr::Call { callee, .. } if matches!(callee.as_ref(), Expr::LocalGet(x) if *x == id)); + if is_target { + let Expr::Call { args, .. } = expr else { + unreachable!() + }; + // Arity was checked during collection; a mismatch here means the + // count changed under us, which cannot happen, but stay conservative. + if args.len() != params.len() { + return; + } + let mut map: HashMap = HashMap::with_capacity(params.len()); + for (p, a) in params.iter().zip(args.iter()) { + map.insert(*p, a.clone()); + } + let mut replacement = body_expr.clone(); + substitute_locals(&mut replacement, &map, next_local_id); + *expr = replacement; + return; + } + if let Expr::Closure { body, .. } = expr { + for s in body.iter_mut() { + for_each_expr_in_stmt_mut(s, &mut |e| { + rewrite_calls(e, id, params, body_expr, next_local_id) + }); + for inner in nested_stmt_lists(s) { + for s2 in inner.iter_mut() { + for_each_expr_in_stmt_mut(s2, &mut |e| { + rewrite_calls(e, id, params, body_expr, next_local_id) + }); + } + } + } + } + walk_expr_children_mut(expr, &mut |child| { + rewrite_calls(child, id, params, body_expr, next_local_id) + }); +} + +/// Drop `if (x === undefined) x = …;` when `x` was declared in this list with +/// a literal initializer that is never `undefined` and is not written anywhere +/// else in the list (including the guard's own nested lists). +fn remove_dead_default_guards(stmts: &mut Vec) { + let mut literal_inits: Vec = Vec::new(); + for s in stmts.iter() { + if let Stmt::Let { + id, + init: Some(init), + .. + } = s + { + if init_is_never_undefined(init) { + literal_inits.push(*id); + } + } + } + if literal_inits.is_empty() { + return; + } + // Count writes per binding across the whole list. + let mut writes: HashMap = HashMap::new(); + for s in stmts.iter() { + count_writes_in_stmt(s, &mut writes); + } + let mut i = 0; + while i < stmts.len() { + let guarded = guarded_default_binding(&stmts[i]); + match guarded { + Some(id) + if literal_inits.contains(&id) && writes.get(&id).copied().unwrap_or(0) == 1 => + { + stmts.remove(i); + } + _ => i += 1, + } + } +} + +fn init_is_never_undefined(init: &Expr) -> bool { + matches!( + init, + Expr::Closure { .. } + | Expr::Integer(_) + | Expr::Number(_) + | Expr::Bool(_) + | Expr::String(_) + | Expr::Null + ) +} + +fn guarded_default_binding(stmt: &Stmt) -> Option { + let Stmt::If { + condition, + then_branch, + else_branch: None, + } = stmt + else { + return None; + }; + let Expr::Compare { + op: CompareOp::Eq, + left, + right, + } = condition + else { + return None; + }; + let Expr::LocalGet(id) = left.as_ref() else { + return None; + }; + if !matches!(right.as_ref(), Expr::Undefined) { + return None; + } + match then_branch.as_slice() { + [Stmt::Expr(Expr::LocalSet(x, _))] if x == id => Some(*id), + _ => None, + } +} + +fn count_writes_in_stmt(stmt: &Stmt, writes: &mut HashMap) { + for_each_expr_in_stmt(stmt, &mut |e| count_writes(e, writes)); + for inner in nested_stmt_lists_ref(stmt) { + for s in inner { + count_writes_in_stmt(s, writes); + } + } +} + +fn count_writes(expr: &Expr, writes: &mut HashMap) { + match expr { + Expr::LocalSet(x, value) => { + *writes.entry(*x).or_default() += 1; + count_writes(value, writes); + } + Expr::Update { id, .. } => { + *writes.entry(*id).or_default() += 1; + } + Expr::Closure { + body, + mutable_captures, + .. + } => { + // A write inside a nested closure counts against the binding too. + for id in mutable_captures { + *writes.entry(*id).or_default() += 1; + } + for s in body { + count_writes_in_stmt(s, writes); + } + walk_expr_children(expr, &mut |child| count_writes(child, writes)); + } + _ => walk_expr_children(expr, &mut |child| count_writes(child, writes)), + } +} + +// --------------------------------------------------------------------------- +// Statement plumbing +// --------------------------------------------------------------------------- + +fn for_each_expr_in_stmt(stmt: &Stmt, f: &mut dyn FnMut(&Expr)) { + match stmt { + Stmt::Let { init, .. } => { + if let Some(e) = init { + f(e); + } + } + Stmt::Expr(e) | Stmt::Throw(e) => f(e), + Stmt::Return(e) => { + if let Some(e) = e { + f(e); + } + } + Stmt::If { condition, .. } => f(condition), + Stmt::While { condition, .. } | Stmt::DoWhile { condition, .. } => f(condition), + Stmt::For { + init, + condition, + update, + .. + } => { + if let Some(init) = init { + for_each_expr_in_stmt(init, f); + } + if let Some(c) = condition { + f(c); + } + if let Some(u) = update { + f(u); + } + } + Stmt::Labeled { body, .. } => for_each_expr_in_stmt(body, f), + Stmt::Switch { + discriminant, + cases, + } => { + f(discriminant); + for c in cases { + if let Some(t) = &c.test { + f(t); + } + } + } + Stmt::Try { .. } + | Stmt::Break + | Stmt::Continue + | Stmt::LabeledBreak(_) + | Stmt::LabeledContinue(_) + | Stmt::PreallocateBoxes(_) + | Stmt::PreallocateTdzBoxes(_) + | Stmt::ReleaseBoxes(_) => {} + } +} + +fn for_each_expr_in_stmt_mut(stmt: &mut Stmt, f: &mut dyn FnMut(&mut Expr)) { + match stmt { + Stmt::Let { init, .. } => { + if let Some(e) = init { + f(e); + } + } + Stmt::Expr(e) | Stmt::Throw(e) => f(e), + Stmt::Return(e) => { + if let Some(e) = e { + f(e); + } + } + Stmt::If { condition, .. } => f(condition), + Stmt::While { condition, .. } | Stmt::DoWhile { condition, .. } => f(condition), + Stmt::For { + init, + condition, + update, + .. + } => { + if let Some(init) = init { + for_each_expr_in_stmt_mut(init, f); + } + if let Some(c) = condition { + f(c); + } + if let Some(u) = update { + f(u); + } + } + Stmt::Labeled { body, .. } => for_each_expr_in_stmt_mut(body, f), + Stmt::Switch { + discriminant, + cases, + } => { + f(discriminant); + for c in cases { + if let Some(t) = &mut c.test { + f(t); + } + } + } + Stmt::Try { .. } + | Stmt::Break + | Stmt::Continue + | Stmt::LabeledBreak(_) + | Stmt::LabeledContinue(_) + | Stmt::PreallocateBoxes(_) + | Stmt::PreallocateTdzBoxes(_) + | Stmt::ReleaseBoxes(_) => {} + } +} + +fn nested_stmt_lists(s: &mut Stmt) -> Vec<&mut Vec> { + match s { + Stmt::If { + then_branch, + else_branch, + .. + } => match else_branch { + Some(e) => vec![then_branch, e], + None => vec![then_branch], + }, + Stmt::While { body, .. } | Stmt::DoWhile { body, .. } => vec![body], + Stmt::For { body, .. } => vec![body], + Stmt::Labeled { body, .. } => nested_stmt_lists(body), + Stmt::Switch { cases, .. } => cases.iter_mut().map(|c| &mut c.body).collect(), + Stmt::Try { + body, + catch, + finally, + } => { + let mut v = vec![body]; + if let Some(c) = catch { + v.push(&mut c.body); + } + if let Some(f) = finally { + v.push(f); + } + v + } + _ => Vec::new(), + } +} + +fn nested_stmt_lists_ref(s: &Stmt) -> Vec<&Vec> { + match s { + Stmt::If { + then_branch, + else_branch, + .. + } => match else_branch { + Some(e) => vec![then_branch, e], + None => vec![then_branch], + }, + Stmt::While { body, .. } | Stmt::DoWhile { body, .. } => vec![body], + Stmt::For { body, .. } => vec![body], + Stmt::Labeled { body, .. } => nested_stmt_lists_ref(body), + Stmt::Switch { cases, .. } => cases.iter().map(|c| &c.body).collect(), + Stmt::Try { + body, + catch, + finally, + } => { + let mut v = vec![body]; + if let Some(c) = catch { + v.push(&c.body); + } + if let Some(f) = finally { + v.push(f); + } + v + } + _ => Vec::new(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use perry_hir::types::Type; + use perry_hir::Param; + + fn param(id: LocalId, name: &str) -> Param { + Param { + id, + name: name.to_string(), + ty: Type::Any, + default: None, + decorators: Vec::new(), + is_rest: false, + arguments_object: None, + } + } + + fn arrow(func_id: u32, params: Vec, ret: Expr, captures_this: bool) -> Expr { + Expr::Closure { + func_id, + params, + return_type: Type::Any, + body: vec![Stmt::Return(Some(ret))], + captures: Vec::new(), + mutable_captures: Vec::new(), + captures_this, + captures_new_target: false, + enclosing_class: Some("World".to_string()), + is_arrow: true, + is_async: false, + is_generator: false, + is_strict: true, + } + } + + fn call_local(id: LocalId, args: Vec) -> Expr { + Expr::Call { + callee: Box::new(Expr::LocalGet(id)), + args, + type_args: Vec::new(), + byte_offset: 0, + } + } + + fn this_exists(arg: Expr) -> Expr { + Expr::Call { + callee: Box::new(Expr::PropertyGet { + object: Box::new(Expr::This), + property: "exists".to_string(), + byte_offset: 0, + }), + args: vec![arg], + type_args: Vec::new(), + byte_offset: 0, + } + } + + fn guard(id: LocalId, default: Expr) -> Stmt { + Stmt::If { + condition: Expr::Compare { + op: CompareOp::Eq, + left: Box::new(Expr::LocalGet(id)), + right: Box::new(Expr::Undefined), + }, + then_branch: vec![Stmt::Expr(Expr::LocalSet(id, Box::new(default)))], + else_branch: None, + } + } + + const F: LocalId = 10; + const P: LocalId = 11; + const ENTITY: LocalId = 1; + + /// The `world.set` shape after inlining: callback local, dead default + /// guard, one direct call. + #[test] + fn inlines_a_called_arrow_local_and_drops_its_dead_default_guard() { + let mut stmts = vec![ + Stmt::Let { + id: F, + name: "exists".into(), + ty: Type::Any, + mutable: true, + init: Some(arrow( + 25, + vec![param(P, "id")], + this_exists(Expr::LocalGet(P)), + true, + )), + }, + guard(F, arrow(69, vec![], Expr::Bool(true), false)), + Stmt::If { + condition: Expr::Unary { + op: perry_hir::UnaryOp::Not, + operand: Box::new(call_local(F, vec![Expr::LocalGet(ENTITY)])), + }, + then_branch: vec![Stmt::Throw(Expr::String("missing".into()))], + else_branch: None, + }, + ]; + let mut next = 100; + process_stmts(&mut stmts, &mut next); + assert_eq!(stmts.len(), 1, "let and guard removed: {stmts:?}"); + let Stmt::If { condition, .. } = &stmts[0] else { + panic!("expected the if"); + }; + let Expr::Unary { operand, .. } = condition else { + panic!("expected the negation"); + }; + assert_eq!( + format!("{operand:?}"), + format!("{:?}", this_exists(Expr::LocalGet(ENTITY))) + ); + } + + #[test] + fn a_non_call_use_keeps_the_closure() { + let mut stmts = vec![ + Stmt::Let { + id: F, + name: "f".into(), + ty: Type::Any, + mutable: false, + init: Some(arrow(1, vec![param(P, "id")], Expr::LocalGet(P), false)), + }, + // `f` escapes as an argument. + Stmt::Expr(Expr::Call { + callee: Box::new(Expr::GlobalGet(0)), + args: vec![Expr::LocalGet(F)], + type_args: Vec::new(), + byte_offset: 0, + }), + Stmt::Expr(call_local(F, vec![Expr::Integer(1)])), + ]; + let before = stmts.clone(); + let mut next = 100; + process_stmts(&mut stmts, &mut next); + assert_eq!(format!("{stmts:?}"), format!("{before:?}")); + } + + #[test] + fn a_non_trivial_argument_or_a_capture_by_a_nested_closure_is_declined() { + // Non-trivial argument: the arrow would duplicate or reorder effects. + let mut stmts = vec![ + Stmt::Let { + id: F, + name: "f".into(), + ty: Type::Any, + mutable: false, + init: Some(arrow(1, vec![param(P, "id")], Expr::LocalGet(P), false)), + }, + Stmt::Expr(call_local(F, vec![this_exists(Expr::Integer(1))])), + ]; + let before = stmts.clone(); + let mut next = 100; + process_stmts(&mut stmts, &mut next); + assert_eq!(format!("{stmts:?}"), format!("{before:?}")); + + // Captured by a nested closure that calls it later. + let mut stmts = vec![ + Stmt::Let { + id: F, + name: "f".into(), + ty: Type::Any, + mutable: false, + init: Some(arrow(1, vec![], Expr::Integer(1), false)), + }, + Stmt::Return(Some(Expr::Closure { + func_id: 2, + params: vec![], + return_type: Type::Any, + body: vec![Stmt::Return(Some(call_local(F, vec![])))], + captures: vec![F], + mutable_captures: vec![], + captures_this: false, + captures_new_target: false, + enclosing_class: None, + is_arrow: true, + is_async: false, + is_generator: false, + is_strict: true, + })), + ]; + let before = stmts.clone(); + let mut next = 100; + process_stmts(&mut stmts, &mut next); + assert_eq!(format!("{stmts:?}"), format!("{before:?}")); + } + + #[test] + fn a_guard_on_a_reassigned_binding_stays() { + let mut stmts = vec![ + Stmt::Let { + id: F, + name: "n".into(), + ty: Type::Any, + mutable: true, + init: Some(Expr::Integer(3)), + }, + Stmt::Expr(Expr::LocalSet(F, Box::new(Expr::Undefined))), + guard(F, Expr::Integer(7)), + ]; + let before = stmts.clone(); + let mut next = 100; + process_stmts(&mut stmts, &mut next); + assert_eq!(format!("{stmts:?}"), format!("{before:?}")); + } +} diff --git a/crates/perry-transform/src/lib.rs b/crates/perry-transform/src/lib.rs index 5d64b6b518..81e844437e 100644 --- a/crates/perry-transform/src/lib.rs +++ b/crates/perry-transform/src/lib.rs @@ -9,6 +9,7 @@ mod aggregate_scalar; pub mod async_to_generator; pub mod closure; +mod closure_local_inline; pub mod deforest; pub mod finally_inline; pub mod generator; @@ -50,5 +51,6 @@ pub use unroll::unroll_static_loops; pub fn post_inline_cleanups(module: &mut perry_hir::Module) { unroll_static_loops(module); aggregate_scalar::run(module); + closure_local_inline::run(module); prop_cse::run(module); } From f76fa8a8bb61d29efd615c9349063e737da14302 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 13:54:56 +0200 Subject: [PATCH 08/23] perf(transform): follow copy aliases of a reduced callback local Claude-Session: https://claude.ai/code/session_01FUvFrRNZyc5qknBiJbYbby --- .../src/closure_local_inline.rs | 241 +++++++++++++++--- 1 file changed, 209 insertions(+), 32 deletions(-) diff --git a/crates/perry-transform/src/closure_local_inline.rs b/crates/perry-transform/src/closure_local_inline.rs index ee8a7d0eda..ce6715916f 100644 --- a/crates/perry-transform/src/closure_local_inline.rs +++ b/crates/perry-transform/src/closure_local_inline.rs @@ -105,20 +105,26 @@ fn process_stmts(stmts: &mut Vec, next_local_id: &mut LocalId) { i += 1; continue; }; + // An inlined callee binds its callback parameter as a copy local + // (`let exists' = exists`); follow such copies so the calls through + // them count as calls of the closure. + let set = collect_aliases(&stmts[i + 1..], id); let mut uses = Uses::default(); for s in &stmts[i + 1..] { - collect_uses_in_stmt(s, id, &mut uses); + collect_uses_in_stmt(s, &set, &mut uses); } if uses.other || uses.calls == 0 || uses.bad_arity { i += 1; continue; } - for s in &mut stmts[i + 1..] { - for_each_expr_in_stmt_mut(s, &mut |e| { - rewrite_calls(e, id, ¶ms, &body_expr, next_local_id) - }); + let mut tail: Vec = stmts.split_off(i + 1); + for s in tail.iter_mut() { + rewrite_calls_in_stmt(s, &set, ¶ms, &body_expr, next_local_id); } - stmts.remove(i); + remove_alias_lets(&mut tail, &set); + // Drop the closure's own `let` and splice the rewritten tail back. + stmts.truncate(i); + stmts.extend(tail); // Do not advance: the list shifted, and the statement now at `i` may // itself be a candidate. } @@ -184,6 +190,58 @@ fn is_trivial_expr(expr: &Expr) -> bool { ) } +/// `id` plus every local that is a plain copy of it (`let y = id;`, +/// transitively), searched through nested statement lists. +fn collect_aliases(stmts: &[Stmt], id: LocalId) -> Vec { + let mut set = vec![id]; + loop { + let before = set.len(); + for s in stmts { + collect_alias_lets_in_stmt(s, &mut set); + } + if set.len() == before { + return set; + } + } +} + +fn collect_alias_lets_in_stmt(stmt: &Stmt, set: &mut Vec) { + if let Stmt::Let { + id, + init: Some(Expr::LocalGet(src)), + .. + } = stmt + { + if set.contains(src) && !set.contains(id) { + set.push(*id); + } + } + for inner in nested_stmt_lists_ref(stmt) { + for s in inner { + collect_alias_lets_in_stmt(s, set); + } + } +} + +fn remove_alias_lets(stmts: &mut Vec, set: &[LocalId]) { + stmts.retain(|s| { + !matches!( + s, + Stmt::Let { id, init: Some(Expr::LocalGet(src)), .. } + if set.contains(id) && set.contains(src) + ) + }); + for s in stmts.iter_mut() { + remove_alias_lets_in_stmt(s, set); + } +} + +fn remove_alias_lets_in_stmt(stmt: &mut Stmt, set: &[LocalId]) { + for inner in nested_stmt_lists(stmt) { + remove_alias_lets(inner, set); + } +} + #[derive(Default)] struct Uses { calls: usize, @@ -191,55 +249,84 @@ struct Uses { other: bool, } -fn collect_uses_in_stmt(stmt: &Stmt, id: LocalId, uses: &mut Uses) { - for_each_expr_in_stmt(stmt, &mut |e| collect_uses(e, id, uses)); +fn collect_uses_in_stmt(stmt: &Stmt, set: &[LocalId], uses: &mut Uses) { + // The copy that defines an alias is not a use of the closure. + if let Stmt::Let { + id, + init: Some(Expr::LocalGet(src)), + .. + } = stmt + { + if set.contains(id) && set.contains(src) { + return; + } + } + for_each_expr_in_stmt(stmt, &mut |e| collect_uses(e, set, uses)); for inner in nested_stmt_lists_ref(stmt) { for s in inner { - collect_uses_in_stmt(s, id, uses); + collect_uses_in_stmt(s, set, uses); } } } -fn collect_uses(expr: &Expr, id: LocalId, uses: &mut Uses) { +fn collect_uses(expr: &Expr, set: &[LocalId], uses: &mut Uses) { match expr { - Expr::Call { callee, args, .. } if matches!(callee.as_ref(), Expr::LocalGet(x) if *x == id) => + Expr::Call { callee, args, .. } if matches!(callee.as_ref(), Expr::LocalGet(x) if set.contains(x)) => { uses.calls += 1; if !args.iter().all(is_trivial_expr) { uses.bad_arity = true; } for a in args { - collect_uses(a, id, uses); + collect_uses(a, set, uses); } } - Expr::LocalGet(x) if *x == id => uses.other = true, + Expr::LocalGet(x) if set.contains(x) => uses.other = true, Expr::LocalSet(x, value) => { - if *x == id { + if set.contains(x) { uses.other = true; } - collect_uses(value, id, uses); + collect_uses(value, set, uses); } Expr::Closure { captures, body, .. } => { - if captures.contains(&id) { + if captures.iter().any(|c| set.contains(c)) { uses.other = true; } for s in body { - collect_uses_in_stmt(s, id, uses); + collect_uses_in_stmt(s, set, uses); } - walk_expr_children(expr, &mut |child| collect_uses(child, id, uses)); + walk_expr_children(expr, &mut |child| collect_uses(child, set, uses)); + } + _ => walk_expr_children(expr, &mut |child| collect_uses(child, set, uses)), + } +} + +/// Rewrite every call of `id` in `stmt`, including its nested statement lists. +fn rewrite_calls_in_stmt( + stmt: &mut Stmt, + set: &[LocalId], + params: &[LocalId], + body_expr: &Expr, + next_local_id: &mut LocalId, +) { + for_each_expr_in_stmt_mut(stmt, &mut |e| { + rewrite_calls(e, set, params, body_expr, next_local_id) + }); + for inner in nested_stmt_lists(stmt) { + for s in inner.iter_mut() { + rewrite_calls_in_stmt(s, set, params, body_expr, next_local_id); } - _ => walk_expr_children(expr, &mut |child| collect_uses(child, id, uses)), } } fn rewrite_calls( expr: &mut Expr, - id: LocalId, + set: &[LocalId], params: &[LocalId], body_expr: &Expr, next_local_id: &mut LocalId, ) { - let is_target = matches!(expr, Expr::Call { callee, .. } if matches!(callee.as_ref(), Expr::LocalGet(x) if *x == id)); + let is_target = matches!(expr, Expr::Call { callee, .. } if matches!(callee.as_ref(), Expr::LocalGet(x) if set.contains(x))); if is_target { let Expr::Call { args, .. } = expr else { unreachable!() @@ -260,20 +347,11 @@ fn rewrite_calls( } if let Expr::Closure { body, .. } = expr { for s in body.iter_mut() { - for_each_expr_in_stmt_mut(s, &mut |e| { - rewrite_calls(e, id, params, body_expr, next_local_id) - }); - for inner in nested_stmt_lists(s) { - for s2 in inner.iter_mut() { - for_each_expr_in_stmt_mut(s2, &mut |e| { - rewrite_calls(e, id, params, body_expr, next_local_id) - }); - } - } + rewrite_calls_in_stmt(s, set, params, body_expr, next_local_id); } } walk_expr_children_mut(expr, &mut |child| { - rewrite_calls(child, id, params, body_expr, next_local_id) + rewrite_calls(child, set, params, body_expr, next_local_id) }); } @@ -685,6 +763,105 @@ mod tests { ); } + /// The inliner wraps a callee body in `do { … } while (false)`, so the + /// only call usually sits in a nested statement list. + #[test] + fn rewrites_calls_inside_nested_statement_lists() { + let mut stmts = vec![ + Stmt::Let { + id: F, + name: "exists".into(), + ty: Type::Any, + mutable: true, + init: Some(arrow( + 25, + vec![param(P, "id")], + this_exists(Expr::LocalGet(P)), + true, + )), + }, + Stmt::DoWhile { + body: vec![Stmt::If { + condition: Expr::Unary { + op: perry_hir::UnaryOp::Not, + operand: Box::new(call_local(F, vec![Expr::LocalGet(ENTITY)])), + }, + then_branch: vec![Stmt::Break], + else_branch: None, + }], + condition: Expr::Bool(false), + }, + ]; + let mut next = 100; + process_stmts(&mut stmts, &mut next); + assert_eq!(stmts.len(), 1, "{stmts:?}"); + let rendered = format!("{stmts:?}"); + assert!( + !rendered.contains("LocalGet(10)"), + "call not rewritten: {rendered}" + ); + assert!( + rendered.contains("\"exists\""), + "body not substituted: {rendered}" + ); + } + + /// The inliner binds the callee's callback parameter as a copy local + /// (`let exists' = exists`) inside its `do { … } while (false)` wrapper. + #[test] + fn follows_copy_aliases_of_the_callback_and_removes_them() { + const ALIAS: LocalId = 20; + let mut stmts = vec![ + Stmt::Let { + id: F, + name: "exists".into(), + ty: Type::Any, + mutable: true, + init: Some(arrow( + 25, + vec![param(P, "id")], + this_exists(Expr::LocalGet(P)), + true, + )), + }, + guard(F, arrow(69, vec![], Expr::Bool(true), false)), + Stmt::DoWhile { + body: vec![ + Stmt::Let { + id: ALIAS, + name: "exists".into(), + ty: Type::Any, + mutable: false, + init: Some(Expr::LocalGet(F)), + }, + Stmt::If { + condition: Expr::Unary { + op: perry_hir::UnaryOp::Not, + operand: Box::new(call_local(ALIAS, vec![Expr::LocalGet(ENTITY)])), + }, + then_branch: vec![Stmt::Break], + else_branch: None, + }, + ], + condition: Expr::Bool(false), + }, + ]; + let mut next = 100; + process_stmts(&mut stmts, &mut next); + let rendered = format!("{stmts:?}"); + assert_eq!(stmts.len(), 1, "{rendered}"); + assert!(!rendered.contains("LocalGet(10)"), "{rendered}"); + assert!( + !rendered.contains("LocalGet(20)"), + "alias let/read survived: {rendered}" + ); + assert!( + !rendered.contains("Closure"), + "closure survived: {rendered}" + ); + assert!(rendered.contains("\"exists\""), "{rendered}"); + } + #[test] fn a_non_call_use_keeps_the_closure() { let mut stmts = vec![ From e0078f52e302a1bf07bda176b2cd1d0e8dc1b4c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 14:23:46 +0200 Subject: [PATCH 09/23] perf: process-global address sketch ahead of the per-object layout probe; resolve the inline-arena state at the first executing allocation Claude-Session: https://claude.ai/code/session_01FUvFrRNZyc5qknBiJbYbby --- crates/perry-codegen/src/codegen/closure.rs | 1 + crates/perry-codegen/src/codegen/entry.rs | 2 ++ crates/perry-codegen/src/codegen/function.rs | 1 + crates/perry-codegen/src/codegen/method.rs | 2 ++ crates/perry-codegen/src/expr/mod.rs | 36 ++++++++++++++++++-- crates/perry-codegen/src/function.rs | 9 +++++ crates/perry-runtime/src/gc/layout_tables.rs | 28 +++++++++++++++ 7 files changed, 77 insertions(+), 2 deletions(-) diff --git a/crates/perry-codegen/src/codegen/closure.rs b/crates/perry-codegen/src/codegen/closure.rs index 19f5961df2..5984a82336 100644 --- a/crates/perry-codegen/src/codegen/closure.rs +++ b/crates/perry-codegen/src/codegen/closure.rs @@ -1109,6 +1109,7 @@ pub(super) fn compile_closure( declared_only_numeric_locals: std::collections::HashSet::new(), shadow_slot_clears_after_stmt, arena_state_slot: None, + arena_state_lazy: false, class_keys_slots: HashMap::new(), class_shape_slots: HashMap::new(), class_header_images: HashMap::new(), diff --git a/crates/perry-codegen/src/codegen/entry.rs b/crates/perry-codegen/src/codegen/entry.rs index 64c96ecb64..04f3c078bf 100644 --- a/crates/perry-codegen/src/codegen/entry.rs +++ b/crates/perry-codegen/src/codegen/entry.rs @@ -871,6 +871,7 @@ pub(super) fn compile_module_entry( declared_only_numeric_locals: std::collections::HashSet::new(), shadow_slot_clears_after_stmt: main_shadow_slot_clears_after_stmt, arena_state_slot: None, + arena_state_lazy: false, class_keys_slots: HashMap::new(), class_shape_slots: HashMap::new(), class_header_images: HashMap::new(), @@ -1583,6 +1584,7 @@ pub(super) fn compile_module_entry( declared_only_numeric_locals: std::collections::HashSet::new(), shadow_slot_clears_after_stmt: init_shadow_slot_clears_after_stmt, arena_state_slot: None, + arena_state_lazy: false, class_keys_slots: HashMap::new(), class_shape_slots: HashMap::new(), class_header_images: HashMap::new(), diff --git a/crates/perry-codegen/src/codegen/function.rs b/crates/perry-codegen/src/codegen/function.rs index d761abc0d6..ab85973cbc 100644 --- a/crates/perry-codegen/src/codegen/function.rs +++ b/crates/perry-codegen/src/codegen/function.rs @@ -1125,6 +1125,7 @@ pub(super) fn compile_function( shadow_slots_bound: bound_param_slots, temp_roots: crate::rooting::TempRootPool::default(), arena_state_slot, + arena_state_lazy: false, class_keys_slots: HashMap::new(), class_shape_slots: HashMap::new(), class_header_images: HashMap::new(), diff --git a/crates/perry-codegen/src/codegen/method.rs b/crates/perry-codegen/src/codegen/method.rs index 82b472e04d..df28af5eb3 100644 --- a/crates/perry-codegen/src/codegen/method.rs +++ b/crates/perry-codegen/src/codegen/method.rs @@ -481,6 +481,7 @@ pub(super) fn compile_method( declared_only_numeric_locals: std::collections::HashSet::new(), shadow_slot_clears_after_stmt, arena_state_slot: None, + arena_state_lazy: false, class_keys_slots: HashMap::new(), class_shape_slots: HashMap::new(), class_header_images: HashMap::new(), @@ -1779,6 +1780,7 @@ pub(super) fn compile_static_method( declared_only_numeric_locals: std::collections::HashSet::new(), shadow_slot_clears_after_stmt, arena_state_slot: None, + arena_state_lazy: false, class_keys_slots: HashMap::new(), class_shape_slots: HashMap::new(), class_header_images: HashMap::new(), diff --git a/crates/perry-codegen/src/expr/mod.rs b/crates/perry-codegen/src/expr/mod.rs index f816ad8d6a..657ce503ab 100644 --- a/crates/perry-codegen/src/expr/mod.rs +++ b/crates/perry-codegen/src/expr/mod.rs @@ -891,6 +891,9 @@ pub(crate) struct FnCtx<'a> { /// `None` until the first `new` lowers; thereafter `Some(slot_name)` /// (e.g. `"%r3"`). pub arena_state_slot: Option, + /// `arena_state_slot` is a lazily-resolved null-initialized slot minted by + /// `load_inline_arena_state` (as opposed to a seeded hidden parameter). + pub arena_state_lazy: bool, /// Per-class cached `keys_array` global slots. The /// `@perry_class_keys_` global is set once at module init, @@ -2145,14 +2148,43 @@ fn inline_cache_global_name_for_prefix(module_prefix: &str, site_id: u32) -> Str /// every other function lazily emits the ordinary entry accessor when its /// first inline allocation site is lowered. pub(crate) fn load_inline_arena_state(ctx: &mut FnCtx<'_>) -> String { + // The state is resolved on the first allocation that actually executes, + // not in the entry block: a function whose hot path never allocates + // (`exists`, the typed guard arms of `set`) used to pay the thread-local + // accessor on every call for an allocation on a cold branch. The slot is + // an entry alloca so the resolved pointer is shared by every later site, + // including sites inside loops; a seeded slot (#8591's hidden parameter) + // is simply never null. let arena_state_slot = if let Some(slot) = ctx.arena_state_slot.clone() { slot } else { - let slot = ctx.func.entry_init_call_ptr("js_inline_arena_state"); + let slot = ctx.func.alloca_entry_null_ptr(); ctx.arena_state_slot = Some(slot.clone()); + ctx.arena_state_lazy = true; slot }; - ctx.block().load(PTR, &arena_state_slot) + if !ctx.arena_state_lazy { + // Seeded by the recursive-allocator entry: never null. + return ctx.block().load(PTR, &arena_state_slot); + } + let cached = ctx.block().load(PTR, &arena_state_slot); + let is_null = ctx.block().icmp_eq(PTR, &cached, "null"); + let init_idx = ctx.new_block("arena_state.init"); + let done_idx = ctx.new_block("arena_state.ready"); + let init_label = ctx.block_label(init_idx); + let done_label = ctx.block_label(done_idx); + let cached_pred = ctx.block().label.clone(); + ctx.block().cond_br(&is_null, &init_label, &done_label); + + ctx.current_block = init_idx; + let fresh = ctx.block().call(PTR, "js_inline_arena_state", &[]); + ctx.block().store(PTR, &fresh, &arena_state_slot); + let init_pred = ctx.block().label.clone(); + ctx.block().br(&done_label); + + ctx.current_block = done_idx; + ctx.block() + .phi(PTR, &[(&cached, &cached_pred), (&fresh, &init_pred)]) } #[cfg(test)] diff --git a/crates/perry-codegen/src/function.rs b/crates/perry-codegen/src/function.rs index 635b3625c9..06d26d1845 100644 --- a/crates/perry-codegen/src/function.rs +++ b/crates/perry-codegen/src/function.rs @@ -586,6 +586,15 @@ impl LlFunction { /// user code in the entry block, dominating every reachable use. /// The slot pointer is returned for the caller to load from at /// each subsequent allocation site. + /// An entry-block `ptr` slot initialized to `null`, for a value that is + /// resolved lazily at its first use (see `load_inline_arena_state`). + pub fn alloca_entry_null_ptr(&mut self) -> String { + let slot = self.alloca_entry(crate::types::PTR); + self.entry_allocas + .push(format!(" store ptr null, ptr {}", slot)); + slot + } + pub fn entry_init_call_ptr(&mut self, func_name: &str) -> String { let slot = self.alloca_entry(crate::types::PTR); let result_reg = format!("%r{}", self.reg_counter.next()); diff --git a/crates/perry-runtime/src/gc/layout_tables.rs b/crates/perry-runtime/src/gc/layout_tables.rs index b2465deafb..843d6a2e62 100644 --- a/crates/perry-runtime/src/gc/layout_tables.rs +++ b/crates/perry-runtime/src/gc/layout_tables.rs @@ -104,6 +104,28 @@ impl Drop for PerObjectLayoutHint { /// thread. const LAYOUT_ADDR_FILTER_BITS: usize = 4096; const LAYOUT_ADDR_FILTER_WORDS: usize = LAYOUT_ADDR_FILTER_BITS / 64; + +/// Process-global, monotone union of every thread's address filter. +/// +/// `layout_forget_object` runs on every allocation once any thread holds a +/// per-object record, and its first real step used to be resolving this +/// thread's hint through `_tlv_get_addr` just to consult the filter. One +/// long-lived masked object (a test harness's callback closure, a registered +/// listener) therefore taxed every later allocation in the program with a +/// thread-local access — 3.4% of an allocation-heavy ECS row. Bits are set +/// alongside the thread-local filter and never cleared: a stale bit is only a +/// false positive that falls through to the thread-local check, whereas +/// clearing while another thread still holds records would be a false +/// negative and leave a stale mask on a recycled address. The filter is a +/// 4,096-bit sketch, so saturation degrades to exactly the previous cost. +static GLOBAL_LAYOUT_ADDR_FILTER: [std::sync::atomic::AtomicU64; LAYOUT_ADDR_FILTER_WORDS] = + [const { std::sync::atomic::AtomicU64::new(0) }; LAYOUT_ADDR_FILTER_WORDS]; + +#[inline(always)] +pub(in crate::gc) fn global_layout_addr_filter_may_hold(user_ptr: usize) -> bool { + let (word, bit) = layout_addr_filter_slot(user_ptr); + GLOBAL_LAYOUT_ADDR_FILTER[word].load(std::sync::atomic::Ordering::Relaxed) & bit != 0 +} /// Rebuild the filter from the live keys once this many bits have been set /// since the last rebuild. Without it a workload that churns per-object /// records would saturate the filter and never recover; with it the false @@ -188,6 +210,7 @@ pub(in crate::gc) fn layout_addr_filter_note(user_ptr: usize) { unsafe { (*hint.filter.get())[word] |= bit; } + GLOBAL_LAYOUT_ADDR_FILTER[word].fetch_or(bit, std::sync::atomic::Ordering::Relaxed); hint.sets.set(hint.sets.get().saturating_add(1)); } @@ -643,6 +666,11 @@ pub(in crate::gc) fn layout_forget_object(user_ptr: usize) { if PERRY_PER_OBJECT_LAYOUTS_ANY.load(std::sync::atomic::Ordering::SeqCst) == 0 { return; } + // Process-global sketch before any thread-local access: an address no + // thread ever recorded needs nothing removed. + if !global_layout_addr_filter_may_hold(user_ptr) { + return; + } // ONE hot-slot resolution for both halves of the guard: the flag (cheap, // and false for the overwhelming majority of workloads) and then the // address filter (what rescues a workload with an immortal record). From 313c39c40815b0c1b444d805a1fda07210ba81cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 14:41:24 +0200 Subject: [PATCH 10/23] perf: inline the per-object layout address sketch in the construction gate; one rooted resolution per element in Array.some Claude-Session: https://claude.ai/code/session_01FUvFrRNZyc5qknBiJbYbby --- .../src/lower_call/typed_shape_init.rs | 23 ++++++++++++- .../src/runtime_decls/objects.rs | 6 ++++ .../perry-runtime/src/array/iter_methods.rs | 32 ++++++++++++++++--- crates/perry-runtime/src/gc/layout_tables.rs | 11 +++++-- 4 files changed, 63 insertions(+), 9 deletions(-) diff --git a/crates/perry-codegen/src/lower_call/typed_shape_init.rs b/crates/perry-codegen/src/lower_call/typed_shape_init.rs index 5f89e7c172..c3f15e0838 100644 --- a/crates/perry-codegen/src/lower_call/typed_shape_init.rs +++ b/crates/perry-codegen/src/lower_call/typed_shape_init.rs @@ -191,15 +191,36 @@ pub(super) fn emit_typed_shape_layout_declare( /// takes the call. A never-taken branch per allocation is the entire /// steady-state cost. fn emit_gated_forget_object_layout(ctx: &mut FnCtx<'_>, obj_handle: &str) { + let sketch_idx = ctx.new_block("layout_forget.sketch"); let call_idx = ctx.new_block("layout_forget.armed"); let done_idx = ctx.new_block("layout_forget.done"); + let sketch_label = ctx.block_label(sketch_idx); let call_label = ctx.block_label(call_idx); let done_label = ctx.block_label(done_idx); { let blk = ctx.block(); let any = blk.load_atomic_monotonic(crate::types::I32, "@PERRY_PER_OBJECT_LAYOUTS_ANY", 4); let armed = blk.icmp_ne(crate::types::I32, &any, "0"); - blk.cond_br(&armed, &call_label, &done_label); + blk.cond_br(&armed, &sketch_label, &done_label); + } + // Armed: some thread holds a per-object record. Test the process-global + // address sketch (`layout_tables::layout_addr_filter_slot`: Fibonacci + // hash, top 12 bits index 4,096 bits) before paying the runtime call — + // one long-lived masked object (a harness closure, a registered listener) + // otherwise taxes every later allocation with a thread-local probe. + ctx.current_block = sketch_idx; + { + let blk = ctx.block(); + let hashed = blk.mul(I64, obj_handle, "-7046029254386353131"); + let index = blk.lshr(I64, &hashed, "52"); + let word = blk.lshr(I64, &index, "6"); + let bit_index = blk.and(I64, &index, "63"); + let bit = blk.shl(I64, "1", &bit_index); + let word_ptr = blk.gep(I64, "@PERRY_LAYOUT_ADDR_FILTER", &[(I64, &word)]); + let word_bits = blk.load_atomic_monotonic(I64, &word_ptr, 8); + let masked = blk.and(I64, &word_bits, &bit); + let may_hold = blk.icmp_ne(I64, &masked, "0"); + blk.cond_br(&may_hold, &call_label, &done_label); } ctx.current_block = call_idx; ctx.block() diff --git a/crates/perry-codegen/src/runtime_decls/objects.rs b/crates/perry-codegen/src/runtime_decls/objects.rs index 2bd7a89eed..f8f520a3d8 100644 --- a/crates/perry-codegen/src/runtime_decls/objects.rs +++ b/crates/perry-codegen/src/runtime_decls/objects.rs @@ -44,6 +44,12 @@ pub fn declare_phase_b_objects(module: &mut LlModule) { // construction site can skip `js_gc_forget_object_layout` outright. // perry-runtime: `gc::layout_tables::PERRY_PER_OBJECT_LAYOUTS_ANY`. module.add_external_global("PERRY_PER_OBJECT_LAYOUTS_ANY", I32); + // Process-global 4,096-bit address sketch of every per-object layout + // record ever installed (`gc::layout_tables::PERRY_LAYOUT_ADDR_FILTER`). + // An armed count only says SOME thread holds records; a clear sketch bit + // for this address proves none of them is about this object, so the + // construction site skips the call without a thread-local access. + module.add_external_global("PERRY_LAYOUT_ADDR_FILTER", "[64 x i64]"); // Sticky summary of indexed Array/Object prototype pollution and custom // Array [[Prototype]] installation. Normal compiled programs read this // byte directly in the inline plain-array index guard. diff --git a/crates/perry-runtime/src/array/iter_methods.rs b/crates/perry-runtime/src/array/iter_methods.rs index ca0dc56c02..a3dc190a04 100644 --- a/crates/perry-runtime/src/array/iter_methods.rs +++ b/crates/perry-runtime/src/array/iter_methods.rs @@ -931,19 +931,41 @@ pub extern "C" fn js_array_some_captureless( let rooted = RootedIterArray::new(&scope, arr); for i in 0..length as usize { + // One rooted resolution per element serves the presence test, the + // slot read and the receiver argument; the callback may move the + // array, so the next iteration resolves again. + let arr = rooted.arr(); let element = if exotic { - let arr = rooted.arr(); if !crate::array::array_spec_has_index(arr, i as u32) { continue; } crate::array::array_spec_get(arr, i as u32) } else { - match rooted.present(i) { - Some(element) => element, - None => continue, + if i >= (*arr).length as usize { + continue; } + let bits = *(array_elements_ptr(arr) as *const u64).add(i); + if bits == crate::value::TAG_HOLE { + continue; + } + f64::from_bits(bits) }; - let result = callback(std::ptr::null(), element, i as f64, rooted.receiver()); + let result = callback( + std::ptr::null(), + element, + i as f64, + array_receiver_value(arr), + ); + // A predicate callback answers with a boolean box almost always; + // decide those two bit patterns here and keep the runtime + // predicate for everything else. + let result_bits = result.to_bits(); + if result_bits == TAG_TRUE { + return f64::from_bits(TAG_TRUE); + } + if result_bits == TAG_FALSE { + continue; + } if crate::value::js_is_truthy(result) != 0 { return f64::from_bits(TAG_TRUE); } diff --git a/crates/perry-runtime/src/gc/layout_tables.rs b/crates/perry-runtime/src/gc/layout_tables.rs index 843d6a2e62..3ac2e1c2ca 100644 --- a/crates/perry-runtime/src/gc/layout_tables.rs +++ b/crates/perry-runtime/src/gc/layout_tables.rs @@ -118,13 +118,18 @@ const LAYOUT_ADDR_FILTER_WORDS: usize = LAYOUT_ADDR_FILTER_BITS / 64; /// clearing while another thread still holds records would be a false /// negative and leave a stale mask on a recycled address. The filter is a /// 4,096-bit sketch, so saturation degrades to exactly the previous cost. -static GLOBAL_LAYOUT_ADDR_FILTER: [std::sync::atomic::AtomicU64; LAYOUT_ADDR_FILTER_WORDS] = +/// Exported (`#[no_mangle]`) because generated code tests the sketch inline, +/// right after `PERRY_PER_OBJECT_LAYOUTS_ANY`, before calling +/// `js_gc_forget_object_layout` — the hash and geometry are mirrored in +/// `perry-codegen`'s `emit_gated_forget_object_layout`. +#[no_mangle] +pub static PERRY_LAYOUT_ADDR_FILTER: [std::sync::atomic::AtomicU64; LAYOUT_ADDR_FILTER_WORDS] = [const { std::sync::atomic::AtomicU64::new(0) }; LAYOUT_ADDR_FILTER_WORDS]; #[inline(always)] pub(in crate::gc) fn global_layout_addr_filter_may_hold(user_ptr: usize) -> bool { let (word, bit) = layout_addr_filter_slot(user_ptr); - GLOBAL_LAYOUT_ADDR_FILTER[word].load(std::sync::atomic::Ordering::Relaxed) & bit != 0 + PERRY_LAYOUT_ADDR_FILTER[word].load(std::sync::atomic::Ordering::Relaxed) & bit != 0 } /// Rebuild the filter from the live keys once this many bits have been set /// since the last rebuild. Without it a workload that churns per-object @@ -210,7 +215,7 @@ pub(in crate::gc) fn layout_addr_filter_note(user_ptr: usize) { unsafe { (*hint.filter.get())[word] |= bit; } - GLOBAL_LAYOUT_ADDR_FILTER[word].fetch_or(bit, std::sync::atomic::Ordering::Relaxed); + PERRY_LAYOUT_ADDR_FILTER[word].fetch_or(bit, std::sync::atomic::Ordering::Relaxed); hint.sets.set(hint.sets.get().saturating_add(1)); } From 0d7d77e37c75e8a8c716cc8630491f01f2ab189f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 15:13:54 +0200 Subject: [PATCH 11/23] perf(gc): objects and closures below eight payload slots take the tag scan instead of a per-object mask Claude-Session: https://claude.ai/code/session_01FUvFrRNZyc5qknBiJbYbby --- crates/perry-runtime/src/gc/layout_tables.rs | 40 ++++++++++++++++++- crates/perry-runtime/src/gc/tests/copying.rs | 8 ++-- .../src/gc/tests/layout_trace.rs | 8 ++-- .../layout_trace/object_closure_slots.rs | 16 ++++---- .../tests/layout_trace/per_object_tables.rs | 18 ++++----- crates/perry-runtime/src/gc/tests/oldgen.rs | 12 +++--- .../gc/tests/typed_layout_intact_residual.rs | 6 ++- 7 files changed, 74 insertions(+), 34 deletions(-) diff --git a/crates/perry-runtime/src/gc/layout_tables.rs b/crates/perry-runtime/src/gc/layout_tables.rs index 3ac2e1c2ca..b9e9bb5d15 100644 --- a/crates/perry-runtime/src/gc/layout_tables.rs +++ b/crates/perry-runtime/src/gc/layout_tables.rs @@ -404,6 +404,39 @@ pub(in crate::gc) fn layout_mask_min_slots() -> usize { /// bounds the extra trace work and changes the fewest layout preconditions. pub(in crate::gc) const DEFAULT_MASK_MIN_SLOTS: usize = 4; +/// Objects and closures take the scan below EIGHT slots (2026-08-27). +/// +/// The array threshold above was tuned on long-lived arrays. Small records +/// are different: a four-to-seven-slot object literal or iterator backing +/// with one pointer field — the shape of every command record and every +/// `for…of` iterator on the `codehz/ecs` sync path — minted and dropped a +/// per-object mask on EVERY allocation and death (35k side-table inserts per +/// frame), which saturated the address filters and kept +/// `layout_forget_object` on the thread-local slow path for every later +/// allocation in the program. `PERRY_LAYOUT_MASK_MIN_SLOTS=8` measured +5.9% +/// (5/5 pairs) and `=16` +6.1% on that row; a mask on a record that small can +/// skip at most a handful of tag checks per scan, which never repays a hash +/// insert and remove per object lifetime. +/// `PERRY_LAYOUT_OBJECT_MASK_MIN_SLOTS` overrides it for bisection. +pub(in crate::gc) const DEFAULT_OBJECT_MASK_MIN_SLOTS: usize = 8; + +#[inline(always)] +pub(in crate::gc) fn layout_object_mask_min_slots() -> usize { + use std::sync::atomic::{AtomicUsize, Ordering}; + static N: AtomicUsize = AtomicUsize::new(usize::MAX); + match N.load(Ordering::Relaxed) { + usize::MAX => { + let v = std::env::var("PERRY_LAYOUT_OBJECT_MASK_MIN_SLOTS") + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(DEFAULT_OBJECT_MASK_MIN_SLOTS); + N.store(v, Ordering::Relaxed); + v + } + v => v, + } +} + /// True when either per-object side table may hold an entry. `false` is a /// proof of emptiness (see [`PER_OBJECT_LAYOUTS_NONEMPTY`]); `true` is only a /// hint, so every caller still has to handle a miss. @@ -770,5 +803,10 @@ pub(in crate::gc) unsafe fn layout_prefers_scan_over_mask( user_ptr: usize, slot_index: usize, ) -> bool { - layout_payload_slot_count(header, user_ptr, slot_index) < layout_mask_min_slots() + let min_slots = if (*header).obj_type == GC_TYPE_ARRAY { + layout_mask_min_slots() + } else { + layout_object_mask_min_slots() + }; + layout_payload_slot_count(header, user_ptr, slot_index) < min_slots } diff --git a/crates/perry-runtime/src/gc/tests/copying.rs b/crates/perry-runtime/src/gc/tests/copying.rs index 7873c85b39..289829b854 100644 --- a/crates/perry-runtime/src/gc/tests/copying.rs +++ b/crates/perry-runtime/src/gc/tests/copying.rs @@ -478,11 +478,11 @@ fn test_copying_minor_rewrites_exact_object_pointer_slot_only() { let _trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); let child = young_leaf(); - let obj = crate::object::js_object_alloc(0, 3); + let obj = crate::object::js_object_alloc(0, 8); crate::object::js_object_set_field(obj, 0, crate::value::JSValue::number(11.0)); crate::object::js_object_set_field(obj, 1, crate::value::JSValue::from_bits(ptr_bits(child))); crate::object::js_object_set_field(obj, 2, crate::value::JSValue::number(33.0)); - assert_eq!(test_layout_pointer_slot_count(obj as usize, 3), Some(1)); + assert_eq!(test_layout_pointer_slot_count(obj as usize, 8), Some(1)); js_shadow_slot_set(0, ptr_bits(obj as usize)); let trace = collect_minor_trace(GcTriggerKind::Direct); @@ -517,11 +517,11 @@ fn test_copying_minor_rewrites_exact_closure_pointer_capture_only() { let _trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); let child = young_leaf(); - let closure = crate::closure::js_closure_alloc(test_captured_singleton_func as *const u8, 3); + let closure = crate::closure::js_closure_alloc(test_captured_singleton_func as *const u8, 8); crate::closure::js_closure_set_capture_f64(closure, 0, 10.0); crate::closure::js_closure_set_capture_f64(closure, 1, f64::from_bits(ptr_bits(child))); crate::closure::js_closure_set_capture_f64(closure, 2, 30.0); - assert_eq!(test_layout_pointer_slot_count(closure as usize, 3), Some(1)); + assert_eq!(test_layout_pointer_slot_count(closure as usize, 8), Some(1)); js_shadow_slot_set(0, ptr_bits(closure as usize)); let trace = collect_minor_trace(GcTriggerKind::Direct); diff --git a/crates/perry-runtime/src/gc/tests/layout_trace.rs b/crates/perry-runtime/src/gc/tests/layout_trace.rs index c52cd2436a..0ebb286208 100644 --- a/crates/perry-runtime/src/gc/tests/layout_trace.rs +++ b/crates/perry-runtime/src/gc/tests/layout_trace.rs @@ -386,7 +386,7 @@ fn test_layout_mask_object_and_closure_slots() { let object_child = crate::string::js_string_from_bytes(b"object-child".as_ptr(), 12) as *mut u8; let object_child_header = unsafe { header_from_user_ptr(object_child) }; - let obj = crate::object::js_object_alloc(0, 3); + let obj = crate::object::js_object_alloc(0, 8); crate::object::js_object_set_field(obj, 0, crate::value::JSValue::number(1.0)); crate::object::js_object_set_field( obj, @@ -395,7 +395,7 @@ fn test_layout_mask_object_and_closure_slots() { ); crate::object::js_object_set_field(obj, 2, crate::value::JSValue::number(3.0)); - assert_eq!(test_layout_pointer_slot_count(obj as usize, 3), Some(1)); + assert_eq!(test_layout_pointer_slot_count(obj as usize, 8), Some(1)); let valid_ptrs = build_valid_pointer_set(); let mut worklist = Vec::new(); test_reset_trace_slot_reads(); @@ -408,7 +408,7 @@ fn test_layout_mask_object_and_closure_slots() { let closure_child = crate::string::js_string_from_bytes(b"closure-child".as_ptr(), 13) as *mut u8; let closure_child_header = unsafe { header_from_user_ptr(closure_child) }; - let closure = crate::closure::js_closure_alloc(std::ptr::null(), 3); + let closure = crate::closure::js_closure_alloc(std::ptr::null(), 8); crate::closure::js_closure_set_capture_f64(closure, 0, 10.0); crate::closure::js_closure_set_capture_f64( closure, @@ -417,7 +417,7 @@ fn test_layout_mask_object_and_closure_slots() { ); crate::closure::js_closure_set_capture_f64(closure, 2, 30.0); - assert_eq!(test_layout_pointer_slot_count(closure as usize, 3), Some(1)); + assert_eq!(test_layout_pointer_slot_count(closure as usize, 8), Some(1)); let valid_ptrs = build_valid_pointer_set(); let mut worklist = Vec::new(); test_reset_trace_slot_reads(); diff --git a/crates/perry-runtime/src/gc/tests/layout_trace/object_closure_slots.rs b/crates/perry-runtime/src/gc/tests/layout_trace/object_closure_slots.rs index 27a6639c8e..bbfb2fd0d4 100644 --- a/crates/perry-runtime/src/gc/tests/layout_trace/object_closure_slots.rs +++ b/crates/perry-runtime/src/gc/tests/layout_trace/object_closure_slots.rs @@ -5,11 +5,11 @@ fn test_trace_object_uses_pointer_layout_mask() { clear_marks(); clear_mark_seeds(); - let numeric = crate::object::js_object_alloc(0, 3); + let numeric = crate::object::js_object_alloc(0, 8); crate::object::js_object_set_field(numeric, 0, crate::value::JSValue::number(1.0)); crate::object::js_object_set_field(numeric, 1, crate::value::JSValue::number(2.0)); crate::object::js_object_set_field(numeric, 2, crate::value::JSValue::bool(false)); - assert_eq!(test_layout_pointer_slot_count(numeric as usize, 3), Some(0)); + assert_eq!(test_layout_pointer_slot_count(numeric as usize, 8), Some(0)); assert_eq!(test_heap_child_slot_count(numeric as *mut u8), 0); let valid_ptrs = build_valid_pointer_set(); @@ -25,11 +25,11 @@ fn test_trace_object_uses_pointer_layout_mask() { let child = crate::string::js_string_from_bytes(b"object-child".as_ptr(), 12); let child_header = unsafe { header_from_user_ptr(child as *mut u8) }; - let mixed = crate::object::js_object_alloc(0, 3); + let mixed = crate::object::js_object_alloc(0, 8); crate::object::js_object_set_field(mixed, 0, crate::value::JSValue::number(1.0)); crate::object::js_object_set_field(mixed, 1, crate::value::JSValue::string_ptr(child)); crate::object::js_object_set_field(mixed, 2, crate::value::JSValue::number(3.0)); - assert_eq!(test_layout_pointer_slot_count(mixed as usize, 3), Some(1)); + assert_eq!(test_layout_pointer_slot_count(mixed as usize, 8), Some(1)); let valid_ptrs = build_valid_pointer_set(); assert!(try_mark_value( @@ -134,11 +134,11 @@ fn test_trace_closure_uses_pointer_layout_mask() { clear_marks(); clear_mark_seeds(); - let numeric = crate::closure::js_closure_alloc(layout_mask_test_closure as *const u8, 3); + let numeric = crate::closure::js_closure_alloc(layout_mask_test_closure as *const u8, 8); crate::closure::js_closure_set_capture_f64(numeric, 0, 1.0); crate::closure::js_closure_set_capture_f64(numeric, 1, 2.0); crate::closure::js_closure_set_capture_ptr(numeric, 2, 7); - assert_eq!(test_layout_pointer_slot_count(numeric as usize, 3), Some(0)); + assert_eq!(test_layout_pointer_slot_count(numeric as usize, 8), Some(0)); assert_eq!(test_heap_child_slot_count(numeric as *mut u8), 0); let valid_ptrs = build_valid_pointer_set(); @@ -154,7 +154,7 @@ fn test_trace_closure_uses_pointer_layout_mask() { let child = crate::string::js_string_from_bytes(b"closure-child".as_ptr(), 13) as *mut u8; let child_header = unsafe { header_from_user_ptr(child) }; - let mixed = crate::closure::js_closure_alloc(layout_mask_test_closure as *const u8, 3); + let mixed = crate::closure::js_closure_alloc(layout_mask_test_closure as *const u8, 8); crate::closure::js_closure_set_capture_f64(mixed, 0, 1.0); crate::closure::js_closure_set_capture_f64( mixed, @@ -162,7 +162,7 @@ fn test_trace_closure_uses_pointer_layout_mask() { f64::from_bits(STRING_TAG | (child as u64 & POINTER_MASK)), ); crate::closure::js_closure_set_capture_ptr(mixed, 2, 7); - assert_eq!(test_layout_pointer_slot_count(mixed as usize, 3), Some(1)); + assert_eq!(test_layout_pointer_slot_count(mixed as usize, 8), Some(1)); let valid_ptrs = build_valid_pointer_set(); assert!(try_mark_value( diff --git a/crates/perry-runtime/src/gc/tests/layout_trace/per_object_tables.rs b/crates/perry-runtime/src/gc/tests/layout_trace/per_object_tables.rs index 28d5f33407..8303c7685d 100644 --- a/crates/perry-runtime/src/gc/tests/layout_trace/per_object_tables.rs +++ b/crates/perry-runtime/src/gc/tests/layout_trace/per_object_tables.rs @@ -161,7 +161,7 @@ fn test_global_gate_exposes_a_recycled_address_record_to_forget() { clear_marks(); clear_mark_seeds(); - let previous_tenant = crate::object::js_object_alloc(0, 2); + let previous_tenant = crate::object::js_object_alloc(0, 8); let child = crate::object::js_object_alloc(0, 0); crate::gc::layout_note_slot( previous_tenant as usize, @@ -169,7 +169,7 @@ fn test_global_gate_exposes_a_recycled_address_record_to_forget() { POINTER_TAG | (child as u64 & POINTER_MASK), ); assert_eq!( - test_layout_pointer_slot_count(previous_tenant as usize, 2), + test_layout_pointer_slot_count(previous_tenant as usize, 8), Some(1), "test premise: the previous tenant must leave an address-keyed mask" ); @@ -285,7 +285,7 @@ fn test_per_object_tables_flag_arms_on_a_pointer_store_into_a_pointer_free_objec clear_marks(); clear_mark_seeds(); - let obj = crate::object::js_object_alloc(0, 2); + let obj = crate::object::js_object_alloc(0, 8); crate::object::js_object_set_field(obj, 0, crate::value::JSValue::number(1.0)); crate::object::js_object_set_field(obj, 1, crate::value::JSValue::number(2.0)); crate::gc::layout_clear_for_ptr(obj as usize); @@ -302,7 +302,7 @@ fn test_per_object_tables_flag_arms_on_a_pointer_store_into_a_pointer_free_objec flag(), "the mask grown in place by `layout_note_slot` must arm the flag too" ); - assert_eq!(test_layout_pointer_slot_count(obj as usize, 2), Some(1)); + assert_eq!(test_layout_pointer_slot_count(obj as usize, 8), Some(1)); clear_marks(); clear_mark_seeds(); @@ -377,7 +377,7 @@ fn test_pointer_store_outside_an_immortal_scope_still_mints_a_mask() { assert_flag_sound("before store"); assert!(test_per_object_tables_are_empty()); - let obj = crate::object::js_object_alloc(0, 2); + let obj = crate::object::js_object_alloc(0, 8); let child = crate::object::js_object_alloc(0, 0); crate::gc::layout_note_slot(obj as usize, 1, POINTER_TAG | (child as u64 & POINTER_MASK)); @@ -387,7 +387,7 @@ fn test_pointer_store_outside_an_immortal_scope_still_mints_a_mask() { if it no longer does, the scoped test below proves nothing" ); assert!(!test_per_object_tables_are_empty()); - assert_eq!(test_layout_pointer_slot_count(obj as usize, 2), Some(1)); + assert_eq!(test_layout_pointer_slot_count(obj as usize, 8), Some(1)); crate::gc::layout_clear_for_ptr(obj as usize); assert!(test_per_object_tables_are_empty()); @@ -500,7 +500,7 @@ fn test_addr_filter_never_hides_a_live_record_across_a_rebuild() { // the rebuild path is exercised rather than merely reachable. let mut objs = Vec::new(); for _ in 0..6000 { - let obj = crate::object::js_object_alloc(0, 2); + let obj = crate::object::js_object_alloc(0, 8); let child = crate::object::js_object_alloc(0, 0); crate::gc::layout_note_slot(obj as usize, 1, POINTER_TAG | (child as u64 & POINTER_MASK)); objs.push(obj); @@ -509,7 +509,7 @@ fn test_addr_filter_never_hides_a_live_record_across_a_rebuild() { for (i, obj) in objs.iter().enumerate() { assert_eq!( - test_layout_pointer_slot_count(*obj as usize, 2), + test_layout_pointer_slot_count(*obj as usize, 8), Some(1), "record {i} became invisible — the filter proved absence for an \ address that has a live entry" @@ -544,7 +544,7 @@ fn test_addr_filter_proves_absence_while_the_global_flag_is_armed() { clear_marks(); clear_mark_seeds(); - let live = crate::object::js_object_alloc(0, 2); + let live = crate::object::js_object_alloc(0, 8); let child = crate::object::js_object_alloc(0, 0); crate::gc::layout_note_slot( live as usize, diff --git a/crates/perry-runtime/src/gc/tests/oldgen.rs b/crates/perry-runtime/src/gc/tests/oldgen.rs index 25d947095e..589adc3132 100644 --- a/crates/perry-runtime/src/gc/tests/oldgen.rs +++ b/crates/perry-runtime/src/gc/tests/oldgen.rs @@ -1481,7 +1481,7 @@ fn test_minor_sweep_keeps_unmarked_old_object_layout_mask() { // A live old-gen object with one pointer slot, plus the slot-layout mask // the collector reads to find that pointer. - let old_obj = crate::arena::arena_alloc_gc_old(4 * 8, 8, GC_TYPE_OBJECT) as usize; + let old_obj = crate::arena::arena_alloc_gc_old(8 * 8, 8, GC_TYPE_OBJECT) as usize; unsafe { std::ptr::write_bytes(old_obj as *mut u8, 0, 4 * 8); // A freshly allocated payload starts pointer-free; the first pointer @@ -1491,7 +1491,7 @@ fn test_minor_sweep_keeps_unmarked_old_object_layout_mask() { let child = crate::arena::arena_alloc_gc_old(16, 8, GC_TYPE_STRING) as usize; layout_note_slot(old_obj, 0, string_bits(child)); assert_eq!( - test_layout_pointer_slot_count(old_obj, 4), + test_layout_pointer_slot_count(old_obj, 8), Some(1), "precondition: the old object starts with a one-pointer slot mask" ); @@ -1502,7 +1502,7 @@ fn test_minor_sweep_keeps_unmarked_old_object_layout_mask() { let _ = sweep.finish_unbounded(); assert_eq!( - test_layout_pointer_slot_count(old_obj, 4), + test_layout_pointer_slot_count(old_obj, 8), Some(1), "#6892: minor sweep wiped the slot-layout mask of a live old-gen object" ); @@ -1523,7 +1523,7 @@ fn test_full_sweep_still_finalizes_unmarked_old_object() { clear_mark_seeds(); crate::arena::old_pages_begin_gc_cycle(); - let old_obj = crate::arena::arena_alloc_gc_old(4 * 8, 8, GC_TYPE_OBJECT) as usize; + let old_obj = crate::arena::arena_alloc_gc_old(8 * 8, 8, GC_TYPE_OBJECT) as usize; unsafe { std::ptr::write_bytes(old_obj as *mut u8, 0, 4 * 8); // A freshly allocated payload starts pointer-free; the first pointer @@ -1532,14 +1532,14 @@ fn test_full_sweep_still_finalizes_unmarked_old_object() { } let child = crate::arena::arena_alloc_gc_old(16, 8, GC_TYPE_STRING) as usize; layout_note_slot(old_obj, 0, string_bits(child)); - assert_eq!(test_layout_pointer_slot_count(old_obj, 4), Some(1)); + assert_eq!(test_layout_pointer_slot_count(old_obj, 8), Some(1)); // Full trace (`minor_sweep = false`): unmarked is provably dead. let mut sweep = IncrementalSweepState::new(false, true, None, false, false); let _ = sweep.finish_unbounded(); assert_eq!( - test_layout_pointer_slot_count(old_obj, 4), + test_layout_pointer_slot_count(old_obj, 8), None, "a full sweep must still finalize genuinely dead old-gen objects" ); diff --git a/crates/perry-runtime/src/gc/tests/typed_layout_intact_residual.rs b/crates/perry-runtime/src/gc/tests/typed_layout_intact_residual.rs index 486b8d7fc7..000ef4ee53 100644 --- a/crates/perry-runtime/src/gc/tests/typed_layout_intact_residual.rs +++ b/crates/perry-runtime/src/gc/tests/typed_layout_intact_residual.rs @@ -111,9 +111,11 @@ unsafe fn slot_bits(obj: *mut crate::ObjectHeader, slot: usize) -> u64 { /// `layout_note_slot` decline the mask and take `GC_LAYOUT_UNKNOWN` instead; /// this test wants the `SIDE_MASK` arm. unsafe fn plant_baked_instance(shape_id: u32, packed_keys: &[u8]) -> *mut crate::ObjectHeader { + // Eight fields: a payload below eight slots takes the tag scan, and this + // fixture's premise is the generic branch minting a per-object mask. let obj = crate::object::js_object_alloc_with_shape( shape_id, - 4, + 8, packed_keys.as_ptr(), packed_keys.len() as u32, ); @@ -174,7 +176,7 @@ unsafe fn plant_descriptor_backed_instance( #[test] fn a_descriptorless_bake_drops_its_intact_claim_on_the_generic_downgrade() { unsafe { - let obj = plant_baked_instance(0x8115_0001, b"x\0y\0z\0pad\0"); + let obj = plant_baked_instance(0x8115_0001, b"x\0y\0z\0pad\0p4\0p5\0p6\0p7\0"); let child = string_bits(young_leaf()); crate::object::store_object_field_slot(obj, 0, child); From f2c7c1a10ea77f2a5f8c377568322b95303ac63c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 15:36:53 +0200 Subject: [PATCH 12/23] perf(gc): count an in-flight array append as a live slot in the per-object mask policy Claude-Session: https://claude.ai/code/session_01FUvFrRNZyc5qknBiJbYbby --- crates/perry-runtime/src/gc/layout_tables.rs | 16 ++++++++++++++-- .../src/gc/tests/layout_trace/array_layout.rs | 10 ++++++---- crates/perry-runtime/src/json/mod.rs | 7 +++++-- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/crates/perry-runtime/src/gc/layout_tables.rs b/crates/perry-runtime/src/gc/layout_tables.rs index b9e9bb5d15..267b4be087 100644 --- a/crates/perry-runtime/src/gc/layout_tables.rs +++ b/crates/perry-runtime/src/gc/layout_tables.rs @@ -778,10 +778,22 @@ pub(in crate::gc) unsafe fn layout_payload_slot_count( let arr = user_ptr as *const crate::array::ArrayHeader; let length = (*arr).length as usize; let capacity = (*arr).capacity as usize; - if length > capacity || length > 16_000_000 || slot_index >= length { + if length > capacity || length > 16_000_000 || slot_index >= capacity { usize::MAX - } else { + } else if slot_index < length { length + } else { + // An append in flight: `push` notes the slot before publishing + // the new length, so the live prefix is `slot_index + 1`. + // Reporting "unknown" here minted a per-object mask for the + // first pointer pushed into every small pooled array (10k + // side-table inserts per ECS frame) that the size policy would + // have sent to the tag scan. A large backing store is expected + // to fill, and the layout state is sticky once it settles on + // the scan, so a capacity of eight or more counts as the size + // the array will reach. + let expected = if capacity >= 8 { capacity } else { 0 }; + (slot_index + 1).max(expected) } } GC_TYPE_OBJECT => { diff --git a/crates/perry-runtime/src/gc/tests/layout_trace/array_layout.rs b/crates/perry-runtime/src/gc/tests/layout_trace/array_layout.rs index 0bb8c7ee3e..0eccbeaa3c 100644 --- a/crates/perry-runtime/src/gc/tests/layout_trace/array_layout.rs +++ b/crates/perry-runtime/src/gc/tests/layout_trace/array_layout.rs @@ -278,7 +278,8 @@ fn test_numeric_array_push_heap_value_transitions_and_traces() { let mut arr = crate::array::js_array_alloc(4); arr = crate::array::js_array_push_f64(arr, 1.0); arr = crate::array::js_array_push_f64(arr, 2.0); - assert_eq!(test_layout_pointer_slot_count(arr as usize, 2), Some(0)); + arr = crate::array::js_array_push_f64(arr, 3.0); + assert_eq!(test_layout_pointer_slot_count(arr as usize, 3), Some(0)); let child = crate::string::js_string_from_bytes(b"pushed-child".as_ptr(), 12) as *mut u8; let child_header = unsafe { header_from_user_ptr(child) }; @@ -287,7 +288,7 @@ fn test_numeric_array_push_heap_value_transitions_and_traces() { assert_eq!(pushed, arr, "fixture should exercise the no-grow push path"); assert_eq!( - test_layout_pointer_slot_count(pushed as usize, 3), + test_layout_pointer_slot_count(pushed as usize, 4), Some(1), "heap writes into a numeric array must transition to a pointer-bearing layout" ); @@ -316,9 +317,10 @@ fn test_numeric_array_layout_metadata_matches_gc_scan_state() { let mut arr = crate::array::js_array_alloc(4); arr = crate::array::js_array_push_f64(arr, 1.0); arr = crate::array::js_array_push_f64(arr, 2.0); + arr = crate::array::js_array_push_f64(arr, 3.0); assert_eq!(crate::array::js_array_is_numeric_f64_layout(arr), 1); - assert_numeric_array_trace_free(arr, 2); + assert_numeric_array_trace_free(arr, 3); let child = crate::string::js_string_from_bytes(b"layout-child".as_ptr(), 12) as *mut u8; let child_header = unsafe { header_from_user_ptr(child) }; @@ -326,7 +328,7 @@ fn test_numeric_array_layout_metadata_matches_gc_scan_state() { arr = crate::array::js_array_push_f64(arr, child_box); assert_eq!(crate::array::js_array_is_numeric_f64_layout(arr), 0); - assert_eq!(test_layout_pointer_slot_count(arr as usize, 3), Some(1)); + assert_eq!(test_layout_pointer_slot_count(arr as usize, 4), Some(1)); clear_marks(); clear_mark_seeds(); diff --git a/crates/perry-runtime/src/json/mod.rs b/crates/perry-runtime/src/json/mod.rs index 85421de6fd..b8868ed0be 100644 --- a/crates/perry-runtime/src/json/mod.rs +++ b/crates/perry-runtime/src/json/mod.rs @@ -1178,14 +1178,17 @@ mod tests { Some(0) ); - let mixed_input = br#"[1,"longer",3]"#; + // Eight elements: the pointer lands while the live prefix is two, and + // only a backing store of eight or more slots is expected to fill into + // a mask-worthy payload; this assertion is about the pointer mask. + let mixed_input = br#"[1,"longer",3,4,5,6,7,8]"#; let mixed_text = js_string_from_bytes(mixed_input.as_ptr(), mixed_input.len() as u32); let mixed_value = unsafe { js_json_parse(mixed_text) }; let mixed_arr = (mixed_value.bits() & POINTER_MASK) as *mut crate::ArrayHeader; assert_eq!(crate::array::js_array_is_numeric_f64_layout(mixed_arr), 0); assert_eq!( - crate::gc::test_layout_pointer_slot_count(mixed_arr as usize, 3), + crate::gc::test_layout_pointer_slot_count(mixed_arr as usize, 8), Some(1) ); } From 96ad0ba277f4a31948299ed6eb1bf70d984edf9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 16:15:03 +0200 Subject: [PATCH 13/23] perf(gc): prune dead per-object layout records at every collection and gate the inline forget probe on live young records MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-object layout tables had no death prune: a dead owner's record lingered until its address was recycled and the allocation-site js_gc_forget_object_layout cleared it. That is why every inline allocation had to probe while PERRY_PER_OBJECT_LAYOUTS_ANY was armed — and one long-lived masked object (a harness closure) keeps it armed forever. The monotone address sketch cannot help against a nursery that recycles the same addresses each cycle: a 5k-entity ECS round still paid ~14k forget calls with zero live nursery-keyed records. - Register LAYOUT_SLOT_MASKS + TYPED_LAYOUTS in DEAD_KEY_PRUNES so both cycle kinds drop dead keys while headers are intact, then rebuild the thread's address filter from the survivors. - PERRY_YOUNG_LAYOUT_RECORDS: process-global count of records keyed by an address the bump allocator could hand out again (arena object not on a Longlived/Old page). Conservative between collections (every new such insert bumps it), exact after each prune, zeroed when the tables empty, released on thread exit. - Codegen: the inline-alloc gate reads the armed count, then this count, and only then hashes the address into the sketch. Tests: copied-minor prune drops a dead nursery owner's record and the young count returns to its baseline; a gc_malloc owner keeps the armed count non-zero without counting as young; IR census pins the gate order. Claude-Session: https://claude.ai/code/session_01FUvFrRNZyc5qknBiJbYbby --- .../src/lower_call/typed_shape_bake_tests.rs | 16 ++ .../src/lower_call/typed_shape_init.rs | 17 +- .../src/runtime_decls/objects.rs | 3 + crates/perry-runtime/src/gc/dead_owner.rs | 9 + crates/perry-runtime/src/gc/layout_tables.rs | 162 +++++++++++++++++- .../src/gc/tests/dead_owner_side_tables.rs | 94 ++++++++++ 6 files changed, 296 insertions(+), 5 deletions(-) diff --git a/crates/perry-codegen/src/lower_call/typed_shape_bake_tests.rs b/crates/perry-codegen/src/lower_call/typed_shape_bake_tests.rs index 085b43639f..ee2b5254a8 100644 --- a/crates/perry-codegen/src/lower_call/typed_shape_bake_tests.rs +++ b/crates/perry-codegen/src/lower_call/typed_shape_bake_tests.rs @@ -51,6 +51,13 @@ const FORGET_CALL: &str = "call void @js_gc_forget_object_layout("; const ANY_GLOBAL: &str = "@PERRY_PER_OBJECT_LAYOUTS_ANY"; const ANY_ATOMIC_LOAD: &str = "load atomic i32, ptr @PERRY_PER_OBJECT_LAYOUTS_ANY monotonic, align 4"; +/// The second gate: records keyed by an address this allocator could have +/// recycled. Read only once the armed count is non-zero, and before the +/// address sketch — a long-lived masked object on an old page keeps the +/// armed count non-zero forever while this stays at zero. +const YOUNG_ATOMIC_LOAD: &str = + "load atomic i32, ptr @PERRY_YOUNG_LAYOUT_RECORDS monotonic, align 4"; +const SKETCH_WORD_GEP: &str = "getelementptr i64, ptr @PERRY_LAYOUT_ADDR_FILTER"; /// The packed `GcHeader` word the inline bump writes for a two-`number`-field /// class: @@ -405,6 +412,15 @@ fn a_pointer_free_shape_bakes_its_layout_into_the_header_constant() { per-object mask, and `layout_note_slot` would then OR the new \ object's pointer bits into it:\n{ir}" ); + let any_at = ir.find(ANY_ATOMIC_LOAD).expect("armed-count load"); + let young_at = ir.find(YOUNG_ATOMIC_LOAD).expect("young-record load"); + let sketch_at = ir.find(SKETCH_WORD_GEP).expect("address sketch probe"); + assert!( + any_at < young_at && young_at < sketch_at, + "the gate must read the armed count, then the young-record count, and \ + only then hash the address into the sketch — each load is the cheap \ + proof that skips everything after it:\n{ir}" + ); } /// `class Link { a: number; b: Link | null }` — one declared type differs; diff --git a/crates/perry-codegen/src/lower_call/typed_shape_init.rs b/crates/perry-codegen/src/lower_call/typed_shape_init.rs index c3f15e0838..3cd4460eed 100644 --- a/crates/perry-codegen/src/lower_call/typed_shape_init.rs +++ b/crates/perry-codegen/src/lower_call/typed_shape_init.rs @@ -191,9 +191,11 @@ pub(super) fn emit_typed_shape_layout_declare( /// takes the call. A never-taken branch per allocation is the entire /// steady-state cost. fn emit_gated_forget_object_layout(ctx: &mut FnCtx<'_>, obj_handle: &str) { + let young_idx = ctx.new_block("layout_forget.young"); let sketch_idx = ctx.new_block("layout_forget.sketch"); let call_idx = ctx.new_block("layout_forget.armed"); let done_idx = ctx.new_block("layout_forget.done"); + let young_label = ctx.block_label(young_idx); let sketch_label = ctx.block_label(sketch_idx); let call_label = ctx.block_label(call_idx); let done_label = ctx.block_label(done_idx); @@ -201,7 +203,20 @@ fn emit_gated_forget_object_layout(ctx: &mut FnCtx<'_>, obj_handle: &str) { let blk = ctx.block(); let any = blk.load_atomic_monotonic(crate::types::I32, "@PERRY_PER_OBJECT_LAYOUTS_ANY", 4); let armed = blk.icmp_ne(crate::types::I32, &any, "0"); - blk.cond_br(&armed, &sketch_label, &done_label); + blk.cond_br(&armed, &young_label, &done_label); + } + // Armed, but is any record keyed by an address THIS allocator could have + // just recycled? `layout_tables::PERRY_YOUNG_LAYOUT_RECORDS` counts the + // nursery-keyed records and is exact after every collection's death + // prune; a long-lived masked object on an old page keeps the flag armed + // without keeping this non-zero. + ctx.current_block = young_idx; + { + let blk = ctx.block(); + let young = + blk.load_atomic_monotonic(crate::types::I32, "@PERRY_YOUNG_LAYOUT_RECORDS", 4); + let any_young = blk.icmp_ne(crate::types::I32, &young, "0"); + blk.cond_br(&any_young, &sketch_label, &done_label); } // Armed: some thread holds a per-object record. Test the process-global // address sketch (`layout_tables::layout_addr_filter_slot`: Fibonacci diff --git a/crates/perry-codegen/src/runtime_decls/objects.rs b/crates/perry-codegen/src/runtime_decls/objects.rs index f8f520a3d8..c2bc565bba 100644 --- a/crates/perry-codegen/src/runtime_decls/objects.rs +++ b/crates/perry-codegen/src/runtime_decls/objects.rs @@ -50,6 +50,9 @@ pub fn declare_phase_b_objects(module: &mut LlModule) { // for this address proves none of them is about this object, so the // construction site skips the call without a thread-local access. module.add_external_global("PERRY_LAYOUT_ADDR_FILTER", "[64 x i64]"); + // perry-runtime: `gc::layout_tables::PERRY_YOUNG_LAYOUT_RECORDS` — records + // keyed by an address the inline bump allocator could recycle. + module.add_external_global("PERRY_YOUNG_LAYOUT_RECORDS", I32); // Sticky summary of indexed Array/Object prototype pollution and custom // Array [[Prototype]] installation. Normal compiled programs read this // byte directly in the inline plain-array index guard. diff --git a/crates/perry-runtime/src/gc/dead_owner.rs b/crates/perry-runtime/src/gc/dead_owner.rs index 194ddfda72..c060f62d9e 100644 --- a/crates/perry-runtime/src/gc/dead_owner.rs +++ b/crates/perry-runtime/src/gc/dead_owner.rs @@ -291,6 +291,15 @@ pub(super) const DEAD_KEY_PRUNES: &[DeadKeyPrune] = &[ owner: DeadKeyOwner::Any, prune: crate::array::prune_dead_array_named_property_owners, }, + // Re-keyed by the per-object move hook (`transfer_per_object_slot_mask` / + // `transfer_per_object_descriptor`), not by a metadata visitor. Dropping + // dead keys here is what lets `PERRY_YOUNG_LAYOUT_RECORDS` reach zero, so + // the inline allocator stops probing for a previous tenant's record. + DeadKeyPrune { + table: "LAYOUT_SLOT_MASKS + TYPED_LAYOUTS", + owner: DeadKeyOwner::Any, + prune: crate::gc::layout_tables::prune_dead_per_object_layout_owners, + }, // Re-keyed by the per-object move hook, not by a metadata visitor. DeadKeyPrune { table: "ELEMENT_SHAPES", diff --git a/crates/perry-runtime/src/gc/layout_tables.rs b/crates/perry-runtime/src/gc/layout_tables.rs index 267b4be087..1c8e25e021 100644 --- a/crates/perry-runtime/src/gc/layout_tables.rs +++ b/crates/perry-runtime/src/gc/layout_tables.rs @@ -31,7 +31,7 @@ use super::hot_tls::{hot_layout_slot_masks, hot_per_object_layout_hint, hot_typed_layouts}; use super::layout::{LayoutSlotMask, TypedLayoutDescriptor}; -use super::types::{GcHeader, GC_HEADER_SIZE, GC_TYPE_ARRAY, GC_TYPE_OBJECT}; +use super::types::{GcHeader, GC_FLAG_ARENA, GC_HEADER_SIZE, GC_TYPE_ARRAY, GC_TYPE_OBJECT}; use std::cell::{Cell, RefCell}; thread_local! { @@ -70,6 +70,12 @@ pub(in crate::gc) struct PerObjectLayoutHint { pub(in crate::gc) sets: Cell, /// Which addresses may have an entry — see [`layout_addr_filter_may_hold`]. pub(in crate::gc) filter: std::cell::UnsafeCell<[u64; LAYOUT_ADDR_FILTER_WORDS]>, + /// This thread's contribution to [`PERRY_YOUNG_LAYOUT_RECORDS`]: how many + /// of its per-object records are keyed by an address the inline bump + /// allocator could hand out again (nursery, or not yet classified). Bumped + /// on every new nursery-keyed insert; made exact again by + /// [`recount_young_layout_records`] after each collection's death prune. + pub(in crate::gc) young_records: Cell, } impl PerObjectLayoutHint { @@ -78,6 +84,7 @@ impl PerObjectLayoutHint { nonempty: Cell::new(false), sets: Cell::new(0), filter: std::cell::UnsafeCell::new([0u64; LAYOUT_ADDR_FILTER_WORDS]), + young_records: Cell::new(0), } } } @@ -91,9 +98,145 @@ impl Drop for PerObjectLayoutHint { if self.nonempty.get() { per_object_layouts_global_disarm(); } + let young = self.young_records.replace(0); + if young != 0 { + PERRY_YOUNG_LAYOUT_RECORDS.fetch_sub(young, std::sync::atomic::Ordering::SeqCst); + } + } +} + +/// Process-global count of per-object layout records keyed by an address the +/// inline bump allocator could hand out again — a nursery address, or one the +/// page classifier cannot place yet — summed over every thread. +/// +/// Why it exists: [`PERRY_PER_OBJECT_LAYOUTS_ANY`] stays armed for the life of +/// ONE long-lived masked object (a harness closure, a registered listener), +/// and once it is armed every inline allocation has to ask whether the +/// recycled address still carries a previous tenant's record. The process +/// address sketch (`PERRY_LAYOUT_ADDR_FILTER`) is monotone, so a nursery that +/// recycles the same addresses every cycle saturates it: a 5k-entity ECS +/// round still paid ~14k `js_gc_forget_object_layout` calls with ZERO live +/// nursery records. This count answers the question the allocator is really +/// asking. It is kept conservative between collections (every new +/// nursery-keyed insert bumps it, nothing decrements it) and exact at each +/// collection's death prune (`prune_dead_per_object_layout_owners`), which +/// is also the moment a stale from-space key is dropped — so a zero load +/// proves no inline allocation can inherit a record. +/// +/// Cross-thread staleness is harmless for the same reason it is for the +/// armed-thread count: a thread's own records are program-ordered with its +/// own loads, and another thread's nursery cannot hand out this thread's +/// addresses. +#[no_mangle] +pub static PERRY_YOUNG_LAYOUT_RECORDS: std::sync::atomic::AtomicU32 = + std::sync::atomic::AtomicU32::new(0); + +/// Could the inline bump allocator ever produce `addr` again? A `gc_malloc` +/// block (no `GC_FLAG_ARENA`; system-allocated, unregistered in the page map) +/// never; an arena object on a `Longlived`/`Old` page never; anything else — +/// eden, either survivor space, or a page the classifier cannot place — is +/// counted, not assumed away. +#[inline] +fn layout_key_may_be_nursery(addr: usize) -> bool { + use crate::arena::HeapSpace; + // The tracked probe refuses anything outside a registered arena range or + // the malloc registry, so an untracked key (a test fixture's synthetic + // address) is simply counted. + let Some(header) = (unsafe { crate::value::addr_class::try_read_tracked_gc_header(addr) }) + else { + return true; + }; + if unsafe { header.as_ref() }.gc_flags & GC_FLAG_ARENA == 0 { + return false; + } + !matches!( + crate::arena::classify_heap_space(addr), + HeapSpace::Longlived | HeapSpace::Old + ) +} + +/// A NEW per-object record was keyed by `user_ptr`. +#[inline] +fn note_new_layout_record(user_ptr: usize) { + if !layout_key_may_be_nursery(user_ptr) { + return; + } + let hint = hot_per_object_layout_hint(); + if let Some(next) = hint.young_records.get().checked_add(1) { + hint.young_records.set(next); + PERRY_YOUNG_LAYOUT_RECORDS.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + } +} + +/// Re-derive this thread's young-record count from the live keys and publish +/// the delta. Runs after every death prune (all cycle kinds) and whenever the +/// tables empty, so promotion (a key moving to an old page) and death both +/// bring the count back down. +fn recount_young_layout_records() { + let live = { + let masks = hot_layout_slot_masks().borrow(); + let typed = hot_typed_layouts().borrow(); + masks + .keys() + .chain(typed.keys()) + .filter(|key| layout_key_may_be_nursery(**key)) + .count() + }; + let live = u32::try_from(live).unwrap_or(u32::MAX); + let prev = hot_per_object_layout_hint().young_records.replace(live); + use std::sync::atomic::Ordering::SeqCst; + if live > prev { + PERRY_YOUNG_LAYOUT_RECORDS.fetch_add(live - prev, SeqCst); + } else if prev > live { + PERRY_YOUNG_LAYOUT_RECORDS.fetch_sub(prev - live, SeqCst); } } +/// Death prune for both per-object layout tables (`DEAD_KEY_PRUNES` entry). +/// +/// Before this, a dead owner's record lingered until its address was +/// recycled and `layout_forget_object` cleared it — which is exactly why +/// every allocation had to probe. Dropping dead keys here (headers are still +/// intact at every prune site) and recounting leaves the young-record count +/// at zero whenever the surviving records all live on old pages, and the +/// inline allocator's gate reads that instead of probing. +pub(in crate::gc) fn prune_dead_per_object_layout_owners(is_dead_owner: &dyn Fn(usize) -> bool) { + if !per_object_layouts_maybe_nonempty() { + return; + } + let masks_emptied = { + let mut masks = hot_layout_slot_masks().borrow_mut(); + let had = !masks.is_empty(); + masks.retain(|key, _| !is_dead_owner(*key)); + had && masks.is_empty() + }; + let typed_emptied = { + let mut typed = hot_typed_layouts().borrow_mut(); + let had = !typed.is_empty(); + typed.retain(|key, _| !is_dead_owner(*key)); + had && typed.is_empty() + }; + refresh_per_object_layouts_flag(masks_emptied || typed_emptied); + if per_object_layouts_maybe_nonempty() { + // Stale filter bits are what the pruned keys leave behind; rebuilding + // from the survivors keeps the runtime probes as selective as the + // tables really are. + layout_addr_filter_rebuild(); + recount_young_layout_records(); + } +} + +#[cfg(test)] +pub(in crate::gc) fn test_per_object_layout_present(user_ptr: usize) -> bool { + hot_layout_slot_masks().borrow().contains_key(&user_ptr) + || hot_typed_layouts().borrow().contains_key(&user_ptr) +} + +#[cfg(test)] +pub(in crate::gc) fn test_young_layout_records() -> u32 { + PERRY_YOUNG_LAYOUT_RECORDS.load(std::sync::atomic::Ordering::SeqCst) +} + /// Bits in the per-object address filter (see [`layout_addr_filter_may_hold`]). /// 4096 bits is 512 B of thread-local storage, held INLINE in /// [`PerObjectLayoutHint`] so the flag and the filter share one hot slot. One @@ -557,6 +700,10 @@ pub(in crate::gc) fn refresh_per_object_layouts_flag(touched_map_emptied: bool) if hot_per_object_layout_hint().nonempty.replace(false) { per_object_layouts_global_disarm(); } + let young = hot_per_object_layout_hint().young_records.replace(0); + if young != 0 { + PERRY_YOUNG_LAYOUT_RECORDS.fetch_sub(young, std::sync::atomic::Ordering::SeqCst); + } // Both maps are empty, so every bit is now stale. Clearing here is what // makes the filter's occupancy track LIVE entries rather than every // entry the program has ever created. @@ -569,9 +716,13 @@ pub(in crate::gc) fn refresh_per_object_layouts_flag(touched_map_emptied: bool) pub(in crate::gc) fn typed_layouts_insert(user_ptr: usize, descriptor: TypedLayoutDescriptor) { mark_per_object_layouts_nonempty(); layout_addr_filter_add(user_ptr); - hot_typed_layouts() + let fresh = hot_typed_layouts() .borrow_mut() - .insert(user_ptr, descriptor); + .insert(user_ptr, descriptor) + .is_none(); + if fresh { + note_new_layout_record(user_ptr); + } } /// The one way to add a per-object pointer mask. @@ -579,7 +730,10 @@ pub(in crate::gc) fn typed_layouts_insert(user_ptr: usize, descriptor: TypedLayo pub(in crate::gc) fn slot_masks_insert(user_ptr: usize, mask: LayoutSlotMask) { mark_per_object_layouts_nonempty(); layout_addr_filter_add(user_ptr); - hot_layout_slot_masks().borrow_mut().insert(user_ptr, mask); + let fresh = hot_layout_slot_masks().borrow_mut().insert(user_ptr, mask).is_none(); + if fresh { + note_new_layout_record(user_ptr); + } } /// Drop `user_ptr`'s per-object typed descriptor (only). diff --git a/crates/perry-runtime/src/gc/tests/dead_owner_side_tables.rs b/crates/perry-runtime/src/gc/tests/dead_owner_side_tables.rs index 3abf60f889..0d658e488c 100644 --- a/crates/perry-runtime/src/gc/tests/dead_owner_side_tables.rs +++ b/crates/perry-runtime/src/gc/tests/dead_owner_side_tables.rs @@ -1692,3 +1692,97 @@ fn test_live_symbol_accessor_owner_survives_full_gc() { ); js_shadow_slot_set(0, 0); } + +// --- per-object layout tables (LAYOUT_SLOT_MASKS + TYPED_LAYOUTS) ----------- +// +// The inline allocator's forget probe is gated on +// `PERRY_YOUNG_LAYOUT_RECORDS`: the count of records keyed by an address the +// nursery could hand out again. These pin the two halves of that contract — +// a dead nursery owner's record is pruned by the copied-minor pass (so it can +// never be inherited), and the count returns to zero once no nursery-keyed +// record remains, while a live old-page record keeps the armed flag without +// re-opening the probe. + +fn young_layout_records() -> u32 { + crate::gc::layout_tables::test_young_layout_records() +} + +unsafe fn install_typed_record(addr: usize) { + let pointer_mask = [0b10u64]; + crate::gc::js_gc_init_typed_shape_layout( + addr as u64, + 2, + std::ptr::null(), + 0, + pointer_mask.as_ptr(), + pointer_mask.len() as u32, + ); +} + +#[test] +fn test_dead_nursery_owner_layout_record_pruned_on_copied_minor_and_young_count_drops() { + let _guard = CopyingNurseryTestGuard::new(1); + let before = young_layout_records(); + let (obj, _) = unsafe { alloc_nursery_test_object(2) }; + let addr = obj as usize; + unsafe { install_typed_record(addr) }; + assert!( + crate::gc::layout_tables::test_per_object_layout_present(addr), + "premise: the nursery owner carries a typed record" + ); + assert!( + young_layout_records() > before, + "a fresh nursery-keyed record must count as young until a collection proves otherwise" + ); + js_shadow_slot_set(0, 0); + + let _ = gc_collect_minor(); + + assert!( + !crate::gc::layout_tables::test_per_object_layout_present(addr), + "dead from-space owner's per-object record must be pruned by the copied-minor pass" + ); + assert_eq!( + young_layout_records(), + before, + "after the prune no nursery-keyed record remains, so the inline allocator's gate must read zero" + ); +} + +#[test] +fn test_live_old_owner_layout_record_keeps_flag_armed_but_not_young() { + let _guard = CopyingNurseryTestGuard::new(1); + let before = young_layout_records(); + let addr = unsafe { + let shape_id = crate::object::shapes::shape_descriptor_ensure(std::ptr::null(), 0, 2) + .expect("shape id range exhausted in a test fixture"); + let obj = gc_malloc( + std::mem::size_of::() + 2 * 8, + GC_TYPE_OBJECT, + ) as *mut crate::object::ObjectHeader; + (*obj).class_id = 0; + (*obj).parent_class_id = shape_id; + (*obj).meta = std::ptr::null_mut(); + let fields = + (obj as *mut u8).add(std::mem::size_of::()) as *mut u64; + *fields = 0; + *fields.add(1) = 0; + obj as usize + }; + unsafe { install_typed_record(addr) }; + assert!( + crate::gc::layout_tables::test_per_object_layout_present(addr), + "premise: the malloc'd owner carries a typed record" + ); + assert_ne!( + crate::gc::layout_tables::test_per_object_layout_armed_threads(), + 0, + "a live record keeps the armed-thread count non-zero" + ); + assert_eq!( + young_layout_records(), + before, + "a record on a gc_malloc page is not one the bump allocator can recycle" + ); + crate::gc::layout_clear_for_ptr(addr); +} From 411bcc7b295f5969629dbe9e5d9bda0fc7f47a9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 16:32:12 +0200 Subject: [PATCH 14/23] perf(runtime): strict array store lane skips the registry probes and returns its resolved head; dirty-page cache moves to hot TLS - try_strict_dense_number_store: a GC_TYPE_ARRAY header is never a Buffer, %TypedArray% or native view (each registration carries its own object type), so the two registry probes per store answered nothing the obj_type test had not. Only the Array.prototype address compare stays. The lane now hands back the head it resolved instead of letting the entry point classify the receiver a second time. - dirty_page_cache::LAST_DIRTY_OLD_PAGE was a plain thread_local! on the hit path of every remembered old->young store; _tlv_get_addr under write_barrier_decoded_parent was ~1% of a 5k-entity ECS frame. Claude-Session: https://claude.ai/code/session_01FUvFrRNZyc5qknBiJbYbby --- crates/perry-runtime/src/array/indexing.rs | 52 +++++++++++-------- .../perry-runtime/src/gc/dirty_page_cache.rs | 6 ++- 2 files changed, 36 insertions(+), 22 deletions(-) diff --git a/crates/perry-runtime/src/array/indexing.rs b/crates/perry-runtime/src/array/indexing.rs index ca79f7a02b..b6c93c08a3 100644 --- a/crates/perry-runtime/src/array/indexing.rs +++ b/crates/perry-runtime/src/array/indexing.rs @@ -1332,23 +1332,27 @@ fn array_strict_index_write_guard_resolved(clean: *mut ArrayHeader, index: u32, /// `f64` the raw-f64 layout stores), cannot be a heap pointer (no barrier, no /// pointer-mask update), and keeps a pointer-free or tag-scanned layout valid. #[inline] -unsafe fn try_strict_dense_number_store(arr: *mut ArrayHeader, index: u32, value: f64) -> bool { +unsafe fn try_strict_dense_number_store( + arr: *mut ArrayHeader, + index: u32, + value: f64, +) -> Option<*mut ArrayHeader> { const PAYLOAD_MASK: u64 = 0x0000_FFFF_FFFF_FFFF; let value_bits = value.to_bits(); // A plain double or an INT32 box (`value_bits_to_number` already refuses // the class-reference values that share that tag). NaN keeps the general // path so its canonical encoding stays in one place. let Some(number) = super::header::value_bits_to_number(value_bits) else { - return false; + return None; }; if number.is_nan() { - return false; + return None; } let bits = arr as u64; let top16 = bits >> 48; let raw = if top16 >= 0x7FF8 { if top16 == 0x7FFC || bits & PAYLOAD_MASK == 0 { - return false; + return None; } (bits & PAYLOAD_MASK) as usize } else { @@ -1358,18 +1362,18 @@ unsafe fn try_strict_dense_number_store(arr: *mut ArrayHeader, index: u32, value || raw % std::mem::align_of::() != 0 || !crate::value::addr_class::is_plausible_heap_addr(raw) { - return false; + return None; } if matches!( crate::arena::classify_heap_generation(raw), crate::arena::HeapGeneration::Unknown ) { - return false; + return None; } let mut raw = raw; let mut header = (raw - crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader; if (*header).obj_type != crate::gc::GC_TYPE_ARRAY { - return false; + return None; } if (*header).gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 { // An alias that kept a growth stub (the resolver path-compresses @@ -1384,13 +1388,13 @@ unsafe fn try_strict_dense_number_store(arr: *mut ArrayHeader, index: u32, value crate::arena::HeapGeneration::Unknown ) { - return false; + return None; } let target_header = (target - crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader; if (*target_header).obj_type != crate::gc::GC_TYPE_ARRAY || (*target_header).gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 { - return false; + return None; } raw = target; header = target_header; @@ -1405,21 +1409,24 @@ unsafe fn try_strict_dense_number_store(arr: *mut ArrayHeader, index: u32, value | crate::gc::GC_ARRAY_ELEMENT_SHAPE | crate::gc::GC_LAYOUT_ALL_POINTERS; if flags & REJECT != 0 { - return false; + return None; } let layout = flags & crate::gc::GC_LAYOUT_STATE_MASK; if layout != crate::gc::GC_LAYOUT_POINTER_FREE && layout != 0 { - return false; + return None; } let arr = raw as *mut ArrayHeader; if index >= (*arr).length || index >= (*arr).capacity { - return false; + return None; } - if crate::buffer::is_registered_buffer(raw) - || crate::typedarray::lookup_typed_array_kind(raw).is_some() - || raw == array_prototype_addr() - { - return false; + // No registry probes: a `GC_TYPE_ARRAY` header is never a Buffer + // (`GC_TYPE_BUFFER`), a %TypedArray% (`GC_TYPE_TYPED_ARRAY`) or a native + // view (`GC_TYPE_NATIVE_TYPED_VIEW`) — every registration carries its own + // object type — so the obj_type test above already answered both. Only + // `Array.prototype` itself still needs the address compare: an index + // write there must flip `ARRAY_PROTO_HAS_INDEX` on the slow path. + if raw == array_prototype_addr() { + return None; } // The raw-f64 layouts store the canonical double (what the general path's // canonicalization and `note_array_numeric_index_write` produce); every @@ -1436,7 +1443,7 @@ unsafe fn try_strict_dense_number_store(arr: *mut ArrayHeader, index: u32, value super::header::array_elements_ptr(arr).add(index as usize), store_bits, ); - true + Some(arr) } /// Exercised by the unit tests: `true` when the fast lane answered the store. @@ -1446,7 +1453,7 @@ pub(crate) fn test_strict_dense_number_store( index: u32, value: f64, ) -> bool { - unsafe { try_strict_dense_number_store(arr, index, value) } + unsafe { try_strict_dense_number_store(arr, index, value) }.is_some() } #[no_mangle] @@ -1457,8 +1464,11 @@ pub extern "C" fn js_array_set_f64_extend_strict( ) -> *mut ArrayHeader { // SAFETY: the lane validates the receiver before every dereference and // stores only where the general path would store the same bits. - if unsafe { try_strict_dense_number_store(arr, index, value) } { - return clean_arr_ptr_mut(arr); + if let Some(resolved) = unsafe { try_strict_dense_number_store(arr, index, value) } { + // The lane already resolved the head it stored into (one forwarding + // edge at most); handing that back saves the second classification + // `clean_arr_ptr_mut` would repeat. + return resolved; } let clean = clean_arr_ptr_mut(arr); if clean.is_null() diff --git a/crates/perry-runtime/src/gc/dirty_page_cache.rs b/crates/perry-runtime/src/gc/dirty_page_cache.rs index 74bd44b8f0..da33d1a6d2 100644 --- a/crates/perry-runtime/src/gc/dirty_page_cache.rs +++ b/crates/perry-runtime/src/gc/dirty_page_cache.rs @@ -85,7 +85,11 @@ use std::cell::Cell; /// `usize::MAX` would need a 76-bit address. const NO_PAGE: usize = usize::MAX; -thread_local! { +// Hot TLS, not `std::thread_local!`: this is the HIT path of every old→young +// store the barrier remembers (an old bucket taking a young command each +// push), and the `_tlv_get_addr` resolution a plain thread-local pays per +// probe was ~1% of a 5k-entity ECS frame by itself. +crate::perry_thread_local! { static LAST_DIRTY_OLD_PAGE: Cell = const { Cell::new(NO_PAGE) }; } From e746d7f011709129909df694d9b7fac7ea272a2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 16:46:00 +0200 Subject: [PATCH 15/23] perf(runtime): resolve array header flags once in pop/set_length; latch the named-property table; length-0 rebuild shortcut An object pool's acquire (pool.pop(); pooled.length = 0) classified the same resolved head five more times: array_is_frozen, guard_writable_length and array_iteration_is_exotic in pop; array_object_flags and array_has_named_properties in set_length, each re-running clean_arr_ptr. set_length then rebuilt the empty layout twice (layout_rebuild_from_slots, then refresh_array_numeric_layout -> rebuild_array_numeric_raw_f64 -> layout_init_pointer_free with its own forget probe) and rooted the head in a handle scope for a branch that cannot allocate. - resolved_plain_array_flags reads the header once for a GC_TYPE_ARRAY head; non-array receivers keep the generic registry-probing helpers. - ARRAY_NAMED_PROPS_EVER: monotone latch so the shrink path skips the thread-local probe until some array has taken a named property. - rebuild_array_layout: for length 0 the slots rebuild already left the head POINTER_FREE with its records dropped; only the vacuous raw-f64 claim is left to set. - js_array_set_length roots the head only on the branches that can run user code (descriptor deletes, growth). Claude-Session: https://claude.ai/code/session_01FUvFrRNZyc5qknBiJbYbby --- crates/perry-runtime/src/array/header.rs | 36 ++++++++++- .../src/array/header_gc_slots.rs | 13 +++- crates/perry-runtime/src/array/mod.rs | 12 ++-- crates/perry-runtime/src/array/push_pop.rs | 62 ++++++++++++++++--- 4 files changed, 108 insertions(+), 15 deletions(-) diff --git a/crates/perry-runtime/src/array/header.rs b/crates/perry-runtime/src/array/header.rs index a98901325f..370922697c 100644 --- a/crates/perry-runtime/src/array/header.rs +++ b/crates/perry-runtime/src/array/header.rs @@ -336,6 +336,7 @@ fn merge_array_named_props( owner: usize, owner_props: Vec, ) { + note_array_named_props_ever(); let entry = props.entry(owner).or_default(); for prop in owner_props { if let Some(existing) = entry.iter_mut().find(|existing| existing.name == prop.name) { @@ -396,6 +397,20 @@ unsafe fn string_header_as_str<'a>(key: *const crate::StringHeader) -> Option<&' std::str::from_utf8(bytes).ok() } +/// Has ANY array on this process ever taken a named (non-index) property? +/// +/// `array_has_named_properties` is on the `length` shrink path of every +/// `pooled.length = 0` an object pool performs; without the latch each of +/// those paid a thread-local hash probe to learn that the table has always +/// been empty. Monotone (never cleared), so a false answer is always safe. +static ARRAY_NAMED_PROPS_EVER: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + +#[inline] +fn note_array_named_props_ever() { + ARRAY_NAMED_PROPS_EVER.store(true, std::sync::atomic::Ordering::Release); +} + pub(crate) unsafe fn array_named_property_set( arr: *mut ArrayHeader, key: *const crate::StringHeader, @@ -409,6 +424,7 @@ pub(crate) unsafe fn array_named_property_set( return; }; let owner = arr as usize; + note_array_named_props_ever(); ARRAY_NAMED_PROPS.with(|m| { let mut map = m.borrow_mut(); let props = map.entry(owner).or_default(); @@ -485,10 +501,22 @@ pub(crate) unsafe fn array_named_property_get_by_name( /// instead of leaving that second representation observable. #[inline] pub(crate) unsafe fn array_has_named_properties(arr: *const ArrayHeader) -> bool { + if !ARRAY_NAMED_PROPS_EVER.load(std::sync::atomic::Ordering::Acquire) { + return false; + } let arr = clean_arr_ptr(arr); if arr.is_null() { return false; } + array_has_named_properties_resolved(arr) +} + +/// [`array_has_named_properties`] for a head the caller already resolved. +#[inline] +pub(crate) unsafe fn array_has_named_properties_resolved(arr: *const ArrayHeader) -> bool { + if !ARRAY_NAMED_PROPS_EVER.load(std::sync::atomic::Ordering::Acquire) { + return false; + } ARRAY_NAMED_PROPS.with(|m| { m.borrow() .get(&(arr as usize)) @@ -1168,7 +1196,7 @@ pub(super) unsafe fn array_has_raw_f64_layout_flag(arr: *const ArrayHeader) -> b } #[inline] -unsafe fn set_array_raw_f64_layout_flag(arr: *const ArrayHeader) { +pub(super) unsafe fn set_array_raw_f64_layout_flag(arr: *const ArrayHeader) { if let Some(header) = array_gc_header(arr) { (*header)._reserved |= crate::gc::GC_ARRAY_RAW_F64_LAYOUT; } @@ -1665,6 +1693,12 @@ pub(crate) unsafe fn refresh_array_numeric_layout(arr: *mut ArrayHeader) { if arr.is_null() { return; } + refresh_array_numeric_layout_resolved(arr); +} + +/// [`refresh_array_numeric_layout`] for a head the caller already resolved. +#[inline] +pub(crate) unsafe fn refresh_array_numeric_layout_resolved(arr: *mut ArrayHeader) { if array_slots_are_numeric(arr) { rebuild_array_numeric_raw_f64(arr); } else { diff --git a/crates/perry-runtime/src/array/header_gc_slots.rs b/crates/perry-runtime/src/array/header_gc_slots.rs index 1e46cbd7de..77b88eedd4 100644 --- a/crates/perry-runtime/src/array/header_gc_slots.rs +++ b/crates/perry-runtime/src/array/header_gc_slots.rs @@ -138,7 +138,18 @@ pub(crate) unsafe fn rebuild_array_layout(arr: *mut ArrayHeader) { return; } crate::gc::layout_rebuild_from_slots(arr as *mut u8, array_elements_ptr(arr), length); - refresh_array_numeric_layout(arr); + if length == 0 { + // `layout_rebuild_from_slots` just left the head POINTER_FREE with its + // per-object records dropped and the typed-intact bit cleared, which + // is everything `refresh_array_numeric_layout` would redo for zero + // slots via `rebuild_array_numeric_raw_f64` -> `layout_init_pointer_free` + // (a second header resolution, a second forget probe). All that is + // left of that path is the raw-f64 claim an empty array holds + // vacuously; there are no slots for the old-gen barrier replay either. + super::header::set_array_raw_f64_layout_flag(arr); + return; + } + super::header::refresh_array_numeric_layout_resolved(arr); if crate::arena::pointer_in_old_gen(arr as usize) { let slots = array_elements_ptr(arr); for i in 0..length { diff --git a/crates/perry-runtime/src/array/mod.rs b/crates/perry-runtime/src/array/mod.rs index 01bb88733d..63cc4a7c41 100644 --- a/crates/perry-runtime/src/array/mod.rs +++ b/crates/perry-runtime/src/array/mod.rs @@ -216,8 +216,9 @@ pub(crate) use self::alloc::array_length_from_property_value_or_throw; pub(crate) use self::alloc::{js_array_from_arraylike, js_array_from_string_codepoints}; pub(crate) use self::flat_clone::{dense_spread_copy, dense_spread_source, flattenable_array_ptr}; pub(crate) use self::header::{ - array_byte_size, array_has_named_properties, array_is_frozen, array_is_sealed_or_no_extend, - array_named_property_delete, array_named_property_delete_by_name, array_named_property_get, + array_byte_size, array_has_named_properties, array_has_named_properties_resolved, + array_is_frozen, array_is_sealed_or_no_extend, array_named_property_delete, + array_named_property_delete_by_name, array_named_property_get, array_named_property_get_by_name, array_named_property_has, array_named_property_names, array_named_property_set, array_numeric_raw_f64_get, array_numeric_raw_f64_push_inbounds, array_numeric_raw_f64_set_inbounds, array_object_flags, array_object_flags_from_tag, @@ -226,9 +227,10 @@ pub(crate) use self::header::{ clear_array_numeric_layout, clear_array_numeric_layout_ptr, gc_element_slot_range, mark_array_layout_unknown, mark_array_raw_f64_holes_fresh, normalize_array_receiver, note_array_slot, note_array_slot_layout_only, rebuild_array_layout, rebuild_array_layout_exact, - refresh_array_numeric_layout, replay_array_growth_write_barriers, set_array_numeric_layout, - store_array_slot, store_array_slot_resolved, transfer_array_numeric_layout, - typed_array_receiver, value_bits_to_number, NumericArrayLayout, MIN_ARRAY_CAPACITY, + refresh_array_numeric_layout, refresh_array_numeric_layout_resolved, + replay_array_growth_write_barriers, set_array_numeric_layout, store_array_slot, + store_array_slot_resolved, transfer_array_numeric_layout, typed_array_receiver, + value_bits_to_number, NumericArrayLayout, MIN_ARRAY_CAPACITY, }; // Sole caller is the regex-engine-gated `regex::exec_array`, so the helper and diff --git a/crates/perry-runtime/src/array/push_pop.rs b/crates/perry-runtime/src/array/push_pop.rs index ce0ff02c63..2ffb8cc9e5 100644 --- a/crates/perry-runtime/src/array/push_pop.rs +++ b/crates/perry-runtime/src/array/push_pop.rs @@ -80,6 +80,20 @@ pub(crate) fn guard_writable_length(arr: *const ArrayHeader) { } } +/// The `_reserved` flags of `arr`'s header when that header is a plain +/// `GC_TYPE_ARRAY`, read without re-classifying an already-resolved head. +/// `None` for anything else `clean_arr_ptr` can hand back unchanged (a typed +/// array, a Buffer), which keeps the registry-probing generic helpers in +/// charge of those. +#[inline] +unsafe fn resolved_plain_array_flags(arr: *const ArrayHeader) -> Option { + if (arr as usize) < crate::gc::GC_HEADER_SIZE + 0x1000 { + return None; + } + let gc_header = (arr as *const u8).sub(crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader; + ((*gc_header).obj_type == crate::gc::GC_TYPE_ARRAY).then(|| (*gc_header)._reserved) +} + #[inline] fn guard_writable_length_with_flags(arr: *const ArrayHeader, flags: u16) { if array_length_is_non_writable_with_flags(arr, flags) { @@ -915,10 +929,25 @@ pub extern "C" fn js_array_pop_f64(arr: *mut ArrayHeader) -> f64 { if arr.is_null() { return TAG_UNDEFINED_F64; } - if array_is_frozen(arr) { - throw_frozen_array_mutation(); + // Resolve the header flags ONCE. `array_is_frozen`, `guard_writable_length` + // and `array_iteration_is_exotic` each re-ran `clean_arr_ptr` on the head + // this function had just resolved — three classifications per pop on an + // object pool's `pool.pop()`. + let plain_flags = unsafe { resolved_plain_array_flags(arr) }; + match plain_flags { + Some(flags) => { + if flags & crate::gc::OBJ_FLAG_FROZEN != 0 { + throw_frozen_array_mutation(); + } + guard_writable_length_with_flags(arr, flags); + } + None => { + if array_is_frozen(arr) { + throw_frozen_array_mutation(); + } + guard_writable_length(arr); + } } - guard_writable_length(arr); unsafe { let length = (*arr).length; if length == 0 { @@ -926,7 +955,11 @@ pub extern "C" fn js_array_pop_f64(arr: *mut ArrayHeader) -> f64 { } let new_length = length - 1; - if !crate::array::array_iteration_is_exotic(arr) { + let exotic = match plain_flags { + Some(flags) => crate::array::array_iteration_is_exotic_resolved(arr, flags), + None => crate::array::array_iteration_is_exotic(arr), + }; + if !exotic { let elements_ptr = (arr as *mut u8).add(std::mem::size_of::()) as *mut f64; let value = *elements_ptr.add(new_length as usize); (*arr).length = new_length; @@ -1020,11 +1053,15 @@ pub extern "C" fn js_array_set_length(arr: *mut ArrayHeader, new_length: f64) { return; } let n = array_length_from_property_value_or_throw(new_length); - let scope = crate::gc::RuntimeHandleScope::new(); - let _arr_handle = scope.root_raw_mut_ptr(arr); unsafe { let cur = (*arr).length; - let flags = array_object_flags(arr); + // The head was resolved a line ago; read its flags directly when the + // header really is an array (the common case) instead of classifying + // it a second time through `array_object_flags`. + let flags = match resolved_plain_array_flags(arr) { + Some(flags) => flags, + None => array_object_flags(arr), + }; if flags & crate::gc::OBJ_FLAG_FROZEN != 0 { return; } @@ -1064,8 +1101,13 @@ pub extern "C" fn js_array_set_length(arr: *mut ArrayHeader, new_length: f64) { // three descriptor/expando probes for every removed element. if flags & crate::gc::OBJ_FLAG_ARRAY_DESCRIPTORS == 0 && cur <= capacity - && !array_has_named_properties(arr) + && !array_has_named_properties_resolved(arr) { + // Plain shrink: nothing below can run user code or allocate + // on the GC heap (hole stores, a length write, a layout + // rebuild from the surviving slots), so the head needs no + // handle scope. `pooled.length = 0` in an object pool is this + // branch every time. let elements = (arr as *mut u8).add(std::mem::size_of::()) as *mut u64; for i in n..cur { // GC_STORE_AUDIT(BARRIERED): the suffix becomes unreachable @@ -1077,6 +1119,8 @@ pub extern "C" fn js_array_set_length(arr: *mut ArrayHeader, new_length: f64) { rebuild_array_layout(arr); return; } + let scope = crate::gc::RuntimeHandleScope::new(); + let _arr_handle = scope.root_raw_mut_ptr(arr); if cur > capacity { let mut sparse_indices: Vec = array_named_property_names(arr, false) .into_iter() @@ -1109,6 +1153,8 @@ pub extern "C" fn js_array_set_length(arr: *mut ArrayHeader, new_length: f64) { (*arr).length = n; refresh_array_numeric_layout(arr); } else if n > cur { + let scope = crate::gc::RuntimeHandleScope::new(); + let _arr_handle = scope.root_raw_mut_ptr(arr); // Growing `length` creates holes conceptually; it must not allocate // a dense backing store proportional to the requested length. // Test262's descriptor probe writes 2^32-1 here. Keep large sparse From e33a93bbd420ae9ed783080370e673a8633e27c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 16:46:00 +0200 Subject: [PATCH 16/23] perf(codegen): hoist the STRING_TAG test of js_string_addref_if_heap_string into IR at the local-copy demote sites The demote is deliberately applied to every local-sourced copy (#7846), so on a numeric-heavy path (the ids and records world.set copies around) every one of those calls returned after its first compare. The compare is now four inline instructions; the call is taken only for a STRING_TAG value, so the aliasing contract (#7846, #8432) is unchanged. Register-numbered IR assertions in shadow_inline tests updated for the extra instructions. Claude-Session: https://claude.ai/code/session_01FUvFrRNZyc5qknBiJbYbby --- .../src/codegen/declared_string_add_tests.rs | 14 +++++++++ crates/perry-codegen/src/expr/helpers.rs | 30 +++++++++++++++++++ .../perry-codegen/src/expr/literals_vars.rs | 6 ++-- crates/perry-codegen/src/expr/mod.rs | 9 +++--- crates/perry-codegen/src/expr/property_set.rs | 8 ++--- .../perry-codegen/src/expr/shadow_inline.rs | 8 ++--- crates/perry-codegen/src/stmt/let_stmt.rs | 9 ++---- 7 files changed, 60 insertions(+), 24 deletions(-) diff --git a/crates/perry-codegen/src/codegen/declared_string_add_tests.rs b/crates/perry-codegen/src/codegen/declared_string_add_tests.rs index 98dc3b1ed6..7d2f4640d1 100644 --- a/crates/perry-codegen/src/codegen/declared_string_add_tests.rs +++ b/crates/perry-codegen/src/codegen/declared_string_add_tests.rs @@ -628,6 +628,20 @@ fn assigning_one_local_to_another_demotes_a_possible_string_alias() { ir.contains("call void @js_string_addref_if_heap_string("), "assignment aliases need the same demote as declaration aliases:\n{ir}" ); + // The helper's tag test is hoisted into IR: the call sits behind an + // inline `STRING_TAG` compare, so a numeric copy never leaves the + // function. Pin the compare AND the call — the demote must still reach + // the runtime for a real heap string. + let tag_at = ir + .find("icmp ne i64 %") + .expect("inline STRING_TAG compare before the demote call"); + let call_at = ir + .find("call void @js_string_addref_if_heap_string(") + .expect("demote call"); + assert!( + ir.contains(", 9223090561878065152") && tag_at < call_at, + "the demote call must be guarded by an inline 0x7FFF_0000_0000_0000 tag compare:\n{ir}" + ); } // ------------------------------------------------------- untouched tiers diff --git a/crates/perry-codegen/src/expr/helpers.rs b/crates/perry-codegen/src/expr/helpers.rs index a328c99c3f..ee56239dd1 100644 --- a/crates/perry-codegen/src/expr/helpers.rs +++ b/crates/perry-codegen/src/expr/helpers.rs @@ -296,6 +296,36 @@ pub(crate) fn class_field_store_layout_note_is_conforming( /// there would leave a refcount==1 string aliased from the heap for a later /// in-place `+=` to rewrite underneath the stored slot — silent corruption /// with no crash to trace it back from. +/// `js_string_addref_if_heap_string(v)` with the helper's own tag test hoisted +/// into IR: the call is taken only when `v` carries `STRING_TAG`. +/// +/// The helper is deliberately applied to every local-sourced copy (#7846: a +/// declared numeric/object type is not proof the value is not a string), so +/// on a numeric-heavy path — the ids and records an ECS `world.set` copies +/// around — every one of those calls returned after its first compare. The +/// compare is four instructions inline; the call stays exactly as it was for +/// a real heap string, so the aliasing contract (#7846, #8432) is unchanged. +pub(crate) fn emit_string_addref_if_heap_string(ctx: &mut FnCtx<'_>, value: &str) { + const TAG_MASK_I64: &str = "-281474976710656"; // 0xFFFF_0000_0000_0000 + const STRING_TAG_I64: &str = "9223090561878065152"; // 0x7FFF_0000_0000_0000 + let call_idx = ctx.new_block("str_addref.call"); + let done_idx = ctx.new_block("str_addref.done"); + let call_label = ctx.block_label(call_idx); + let done_label = ctx.block_label(done_idx); + { + let blk = ctx.block(); + let bits = blk.bitcast_double_to_i64(value); + let tag = blk.and(I64, &bits, TAG_MASK_I64); + let not_string = blk.icmp_ne(I64, &tag, STRING_TAG_I64); + blk.cond_br(¬_string, &done_label, &call_label); + } + ctx.current_block = call_idx; + ctx.block() + .call_void("js_string_addref_if_heap_string", &[(DOUBLE, value)]); + ctx.block().br(&done_label); + ctx.current_block = done_idx; +} + pub(crate) fn class_field_store_needs_string_addref(ctx: &FnCtx<'_>, value: &Expr) -> bool { store_needs_string_addref(ctx, value) } diff --git a/crates/perry-codegen/src/expr/literals_vars.rs b/crates/perry-codegen/src/expr/literals_vars.rs index 023b526505..966c51cbe8 100644 --- a/crates/perry-codegen/src/expr/literals_vars.rs +++ b/crates/perry-codegen/src/expr/literals_vars.rs @@ -72,8 +72,7 @@ fn demote_extracted_string_binding(ctx: &mut FnCtx<'_>, id: u32, value: &str) { || (ctx.boxed_vars.contains(&id) && !ctx.module_globals.contains_key(&id)) || ctx.module_globals.contains_key(&id); if persistent_binding && matches!(ctx.local_type_hint(&id), Some(HirType::String)) { - ctx.block() - .call_void("js_string_addref_if_heap_string", &[(DOUBLE, value)]); + super::helpers::emit_string_addref_if_heap_string(ctx, value); } } @@ -734,8 +733,7 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // variables as LocalSet, making that gap observable as the saved // string growing in place with its boxed accumulator (#8432). if matches!(value.as_ref(), Expr::LocalGet(source_id) if source_id != id) { - ctx.block() - .call_void("js_string_addref_if_heap_string", &[(DOUBLE, &v)]); + super::helpers::emit_string_addref_if_heap_string(ctx, &v); } // Closure captures first (write through the runtime), then // locals, then module globals. diff --git a/crates/perry-codegen/src/expr/mod.rs b/crates/perry-codegen/src/expr/mod.rs index 657ce503ab..03971896ec 100644 --- a/crates/perry-codegen/src/expr/mod.rs +++ b/crates/perry-codegen/src/expr/mod.rs @@ -80,10 +80,11 @@ pub(crate) use helpers::{ array_store_needs_layout_note, array_store_needs_write_barrier, buffer_alias_metadata_suffix, class_field_store_layout_note_is_conforming, class_field_store_needs_layout_note, class_field_store_needs_string_addref, emit_all_pointer_array_declaration, - expr_has_numeric_pointer_free_array_layout, expr_produces_fresh_heap_allocation, - expr_produces_non_pointer_bits_by_construction, is_global_this_builtin_function_name, - is_global_this_builtin_name, lower_expr_with_expected_type, lower_js_args_array, - store_needs_string_addref, unbox_str_handle, unbox_to_i64, + emit_string_addref_if_heap_string, expr_has_numeric_pointer_free_array_layout, + expr_produces_fresh_heap_allocation, expr_produces_non_pointer_bits_by_construction, + is_global_this_builtin_function_name, is_global_this_builtin_name, + lower_expr_with_expected_type, lower_js_args_array, store_needs_string_addref, + unbox_str_handle, unbox_to_i64, }; pub(crate) use i32_fast_path::{ can_lower_expr_as_i32, can_lower_expr_as_i32_in_current_region, diff --git a/crates/perry-codegen/src/expr/property_set.rs b/crates/perry-codegen/src/expr/property_set.rs index 2344152b67..2fa5c9ef78 100644 --- a/crates/perry-codegen/src/expr/property_set.rs +++ b/crates/perry-codegen/src/expr/property_set.rs @@ -761,8 +761,7 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // every local source: an erased non-string annotation is // not proof that the current value cannot be a string. if matches!(&**value, Expr::LocalGet(_)) { - ctx.block() - .call_void("js_string_addref_if_heap_string", &[(DOUBLE, &val_double)]); + super::helpers::emit_string_addref_if_heap_string(ctx, &val_double); } let lowered_js = LoweredValue { semantic: SemanticKind::JsValue, @@ -840,10 +839,7 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // self-append doesn't mutate it in-place and corrupt the // field. if matches!(&**value, Expr::LocalGet(_)) { - ctx.block().call_void( - "js_string_addref_if_heap_string", - &[(DOUBLE, &val_double)], - ); + super::helpers::emit_string_addref_if_heap_string(ctx, &val_double); } let lowered_js = LoweredValue { semantic: SemanticKind::JsValue, diff --git a/crates/perry-codegen/src/expr/shadow_inline.rs b/crates/perry-codegen/src/expr/shadow_inline.rs index f98f3b8e97..a3afea6b2a 100644 --- a/crates/perry-codegen/src/expr/shadow_inline.rs +++ b/crates/perry-codegen/src/expr/shadow_inline.rs @@ -504,18 +504,18 @@ mod tests { "inline store must keep both guards; body:\n{body}" ); assert!( - body.contains("icmp eq i64 %r13, -1"), + body.contains("icmp eq i64 %r16, -1"), "frame_top must be tested against the usize::MAX no-frame sentinel; \ body:\n{body}" ); assert!( - body.contains("icmp ult i64 %r15, %r17"), + body.contains("icmp ult i64 %r18, %r20"), "slot index must be bounds-checked against ShadowStackState::len; \ body:\n{body}" ); assert!( body.contains(&format!( - "getelementptr inbounds i8, ptr %r10, i64 {}", + "getelementptr inbounds i8, ptr %r13, i64 {}", SHADOW_STATE_LEN_OFFSET )), "the bounds check must read len at offset {SHADOW_STATE_LEN_OFFSET}; \ @@ -543,7 +543,7 @@ mod tests { body:\n{body}" ); assert!( - body.contains("call void @js_write_barrier_root_nanbox(i64 %r23)"), + body.contains("call void @js_write_barrier_root_nanbox(i64 %r26)"), "inline bind must shade the value it just stored when a cycle is in \ flight; body:\n{body}" ); diff --git a/crates/perry-codegen/src/stmt/let_stmt.rs b/crates/perry-codegen/src/stmt/let_stmt.rs index 3a2d17ff92..be02c302ae 100644 --- a/crates/perry-codegen/src/stmt/let_stmt.rs +++ b/crates/perry-codegen/src/stmt/let_stmt.rs @@ -749,8 +749,7 @@ pub(crate) fn lower_let( let needs_string_demote = matches!(elem, perry_hir::Expr::LocalGet(_)) && !expr_produces_non_pointer_bits_by_construction(ctx, elem); if needs_string_demote { - ctx.block() - .call_void("js_string_addref_if_heap_string", &[(DOUBLE, &v)]); + crate::expr::emit_string_addref_if_heap_string(ctx, &v); } let lowered = LoweredValue { semantic: SemanticKind::JsValue, @@ -1826,8 +1825,7 @@ pub(crate) fn lower_let( // still hold a string at runtime, and the old type gate // then left this alias invisible to self-append (#7846). if matches!(init_expr, perry_hir::Expr::LocalGet(_)) { - ctx.block() - .call_void("js_string_addref_if_heap_string", &[(DOUBLE, &v)]); + crate::expr::emit_string_addref_if_heap_string(ctx, &v); } ctx.block().store(DOUBLE, &v, &slot); v @@ -1850,8 +1848,7 @@ pub(crate) fn lower_let( // started returning `start-try-finally` instead of // `start-try`. if matches!(init_expr, perry_hir::Expr::LocalGet(_)) { - ctx.block() - .call_void("js_string_addref_if_heap_string", &[(DOUBLE, &v)]); + crate::expr::emit_string_addref_if_heap_string(ctx, &v); } ctx.block().store(DOUBLE, &v, &slot); v From 57319a8eeac2edca8f5478e0d8287dd7c0bcc4f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 17:01:27 +0200 Subject: [PATCH 17/23] perf(runtime): typeof result-string cache on hot TLS; drop the unresolved named-property probe The eight cached typeof strings were eight std::thread_local!s, so every typeof on a branchy fast path (typeof merge per command in an ECS apply loop) paid a _tlv_get_addr resolution to reach a cached pointer. One perry_thread_local! array with slot constants keeps the names, the root scanner and the test helpers. Also removes array_has_named_properties (unresolved form) and a stray re-export left unused by the previous commit's resolved variants. Claude-Session: https://claude.ai/code/session_01FUvFrRNZyc5qknBiJbYbby --- crates/perry-runtime/src/array/header.rs | 15 +-- crates/perry-runtime/src/array/mod.rs | 29 ++-- .../perry-runtime/src/builtins/arithmetic.rs | 127 +++++++++--------- 3 files changed, 77 insertions(+), 94 deletions(-) diff --git a/crates/perry-runtime/src/array/header.rs b/crates/perry-runtime/src/array/header.rs index 370922697c..eb8c796ff5 100644 --- a/crates/perry-runtime/src/array/header.rs +++ b/crates/perry-runtime/src/array/header.rs @@ -500,18 +500,9 @@ pub(crate) unsafe fn array_named_property_get_by_name( /// Bulk element operations use this predicate to decline a dense-only path /// instead of leaving that second representation observable. #[inline] -pub(crate) unsafe fn array_has_named_properties(arr: *const ArrayHeader) -> bool { - if !ARRAY_NAMED_PROPS_EVER.load(std::sync::atomic::Ordering::Acquire) { - return false; - } - let arr = clean_arr_ptr(arr); - if arr.is_null() { - return false; - } - array_has_named_properties_resolved(arr) -} - -/// [`array_has_named_properties`] for a head the caller already resolved. +/// Does this (already resolved) array head carry named properties in the +/// side table? Answered by the monotone latch first: until some array has +/// taken a named property, the table has always been empty. #[inline] pub(crate) unsafe fn array_has_named_properties_resolved(arr: *const ArrayHeader) -> bool { if !ARRAY_NAMED_PROPS_EVER.load(std::sync::atomic::Ordering::Acquire) { diff --git a/crates/perry-runtime/src/array/mod.rs b/crates/perry-runtime/src/array/mod.rs index 63cc4a7c41..58c1cf4b0c 100644 --- a/crates/perry-runtime/src/array/mod.rs +++ b/crates/perry-runtime/src/array/mod.rs @@ -216,21 +216,20 @@ pub(crate) use self::alloc::array_length_from_property_value_or_throw; pub(crate) use self::alloc::{js_array_from_arraylike, js_array_from_string_codepoints}; pub(crate) use self::flat_clone::{dense_spread_copy, dense_spread_source, flattenable_array_ptr}; pub(crate) use self::header::{ - array_byte_size, array_has_named_properties, array_has_named_properties_resolved, - array_is_frozen, array_is_sealed_or_no_extend, array_named_property_delete, - array_named_property_delete_by_name, array_named_property_get, - array_named_property_get_by_name, array_named_property_has, array_named_property_names, - array_named_property_set, array_numeric_raw_f64_get, array_numeric_raw_f64_push_inbounds, - array_numeric_raw_f64_set_inbounds, array_object_flags, array_object_flags_from_tag, - array_object_flags_resolved, array_ptr_as_proxy, array_receiver_addr, array_receiver_gc_tag, - buffer_receiver_as_uint8_typed_array, clean_arr_ptr, clean_arr_ptr_mut, - clear_array_numeric_layout, clear_array_numeric_layout_ptr, gc_element_slot_range, - mark_array_layout_unknown, mark_array_raw_f64_holes_fresh, normalize_array_receiver, - note_array_slot, note_array_slot_layout_only, rebuild_array_layout, rebuild_array_layout_exact, - refresh_array_numeric_layout, refresh_array_numeric_layout_resolved, - replay_array_growth_write_barriers, set_array_numeric_layout, store_array_slot, - store_array_slot_resolved, transfer_array_numeric_layout, typed_array_receiver, - value_bits_to_number, NumericArrayLayout, MIN_ARRAY_CAPACITY, + array_byte_size, array_has_named_properties_resolved, array_is_frozen, + array_is_sealed_or_no_extend, array_named_property_delete, array_named_property_delete_by_name, + array_named_property_get, array_named_property_get_by_name, array_named_property_has, + array_named_property_names, array_named_property_set, array_numeric_raw_f64_get, + array_numeric_raw_f64_push_inbounds, array_numeric_raw_f64_set_inbounds, array_object_flags, + array_object_flags_from_tag, array_object_flags_resolved, array_ptr_as_proxy, + array_receiver_addr, array_receiver_gc_tag, buffer_receiver_as_uint8_typed_array, + clean_arr_ptr, clean_arr_ptr_mut, clear_array_numeric_layout, clear_array_numeric_layout_ptr, + gc_element_slot_range, mark_array_layout_unknown, mark_array_raw_f64_holes_fresh, + normalize_array_receiver, note_array_slot, note_array_slot_layout_only, rebuild_array_layout, + rebuild_array_layout_exact, refresh_array_numeric_layout, replay_array_growth_write_barriers, + set_array_numeric_layout, store_array_slot, store_array_slot_resolved, + transfer_array_numeric_layout, typed_array_receiver, value_bits_to_number, NumericArrayLayout, + MIN_ARRAY_CAPACITY, }; // Sole caller is the regex-engine-gated `regex::exec_array`, so the helper and diff --git a/crates/perry-runtime/src/builtins/arithmetic.rs b/crates/perry-runtime/src/builtins/arithmetic.rs index 810ce20e5b..7fa461ea0e 100644 --- a/crates/perry-runtime/src/builtins/arithmetic.rs +++ b/crates/perry-runtime/src/builtins/arithmetic.rs @@ -551,23 +551,30 @@ pub extern "C" fn js_ge(a: JSValue, b: JSValue) -> JSValue { // tool reads emitted LLVM IR, and this is a runtime-side table. The static // checker could never have found it, which is why the runtime instruments had // to be pointed at the registry first. -thread_local! { - static TYPEOF_UNDEFINED: std::cell::Cell<*mut StringHeader> = const { std::cell::Cell::new(std::ptr::null_mut()) }; - static TYPEOF_OBJECT: std::cell::Cell<*mut StringHeader> = const { std::cell::Cell::new(std::ptr::null_mut()) }; - static TYPEOF_BOOLEAN: std::cell::Cell<*mut StringHeader> = const { std::cell::Cell::new(std::ptr::null_mut()) }; - static TYPEOF_NUMBER: std::cell::Cell<*mut StringHeader> = const { std::cell::Cell::new(std::ptr::null_mut()) }; - static TYPEOF_STRING: std::cell::Cell<*mut StringHeader> = const { std::cell::Cell::new(std::ptr::null_mut()) }; - static TYPEOF_FUNCTION: std::cell::Cell<*mut StringHeader> = const { std::cell::Cell::new(std::ptr::null_mut()) }; - static TYPEOF_BIGINT: std::cell::Cell<*mut StringHeader> = const { std::cell::Cell::new(std::ptr::null_mut()) }; - static TYPEOF_SYMBOL: std::cell::Cell<*mut StringHeader> = const { std::cell::Cell::new(std::ptr::null_mut()) }; +// +// One hot-TLS array rather than eight `std::thread_local!`s: `typeof` on a +// branchy fast path (`typeof merge !== "undefined"` per command in an ECS +// apply loop) paid a `_tlv_get_addr` resolution per call just to reach the +// cached pointer. The slot constants keep the eight names. +const TYPEOF_UNDEFINED: usize = 0; +const TYPEOF_OBJECT: usize = 1; +const TYPEOF_BOOLEAN: usize = 2; +const TYPEOF_NUMBER: usize = 3; +const TYPEOF_STRING: usize = 4; +const TYPEOF_FUNCTION: usize = 5; +const TYPEOF_BIGINT: usize = 6; +const TYPEOF_SYMBOL: usize = 7; +const TYPEOF_CACHE_SLOTS: usize = 8; + +crate::perry_thread_local! { + static TYPEOF_CACHE: [std::cell::Cell<*mut StringHeader>; TYPEOF_CACHE_SLOTS] = + const { [const { std::cell::Cell::new(std::ptr::null_mut()) }; TYPEOF_CACHE_SLOTS] }; } /// Get or initialize a cached `typeof` string. -fn get_cached( - cache: &'static std::thread::LocalKey>, - s: &str, -) -> *mut StringHeader { - cache.with(|cell| { +fn get_cached(slot: usize, s: &str) -> *mut StringHeader { + TYPEOF_CACHE.with(|cells| { + let cell = &cells[slot]; let ptr = cell.get(); if !ptr.is_null() { return ptr; @@ -590,28 +597,17 @@ fn get_cached( /// `StringHeader`s, matching `json::scan_parse_roots_mut`'s interned-key /// treatment. pub fn scan_typeof_string_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { - fn visit( - cache: &'static std::thread::LocalKey>, - visitor: &mut crate::gc::RuntimeRootVisitor<'_>, - ) { - cache.with(|cell| { + TYPEOF_CACHE.with(|cells| { + for cell in cells { let mut ptr = cell.get() as *const StringHeader; if ptr.is_null() { - return; + continue; } if visitor.visit_tagged_raw_const_ptr_slot(&mut ptr, crate::value::STRING_TAG) { cell.set(ptr as *mut StringHeader); } - }); - } - visit(&TYPEOF_UNDEFINED, visitor); - visit(&TYPEOF_OBJECT, visitor); - visit(&TYPEOF_BOOLEAN, visitor); - visit(&TYPEOF_NUMBER, visitor); - visit(&TYPEOF_STRING, visitor); - visit(&TYPEOF_FUNCTION, visitor); - visit(&TYPEOF_BIGINT, visitor); - visit(&TYPEOF_SYMBOL, visitor); + } + }); } /// The eight cells and their payloads, in `scan_typeof_string_roots_mut` @@ -619,19 +615,16 @@ pub fn scan_typeof_string_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor< /// cell: that scanner is eight hand-written `visit(...)` calls, and a dropped /// line is invisible to any test that exercises only some of them. #[cfg(test)] -type TypeofCacheCell = &'static std::thread::LocalKey>; - -#[cfg(test)] -fn typeof_cache_entries_for_test() -> [(TypeofCacheCell, &'static str); 8] { +fn typeof_cache_entries_for_test() -> [(usize, &'static str); 8] { [ - (&TYPEOF_UNDEFINED, "undefined"), - (&TYPEOF_OBJECT, "object"), - (&TYPEOF_BOOLEAN, "boolean"), - (&TYPEOF_NUMBER, "number"), - (&TYPEOF_STRING, "string"), - (&TYPEOF_FUNCTION, "function"), - (&TYPEOF_BIGINT, "bigint"), - (&TYPEOF_SYMBOL, "symbol"), + (TYPEOF_UNDEFINED, "undefined"), + (TYPEOF_OBJECT, "object"), + (TYPEOF_BOOLEAN, "boolean"), + (TYPEOF_NUMBER, "number"), + (TYPEOF_STRING, "string"), + (TYPEOF_FUNCTION, "function"), + (TYPEOF_BIGINT, "bigint"), + (TYPEOF_SYMBOL, "symbol"), ] } @@ -645,8 +638,8 @@ fn typeof_cache_entries_for_test() -> [(TypeofCacheCell, &'static str); 8] { // nothing adopts it, delete it rather than letting it rot behind this attribute. #[allow(dead_code)] pub(crate) fn reset_typeof_string_cache_for_test() { - for (cache, _) in typeof_cache_entries_for_test() { - cache.with(|cell| cell.set(std::ptr::null_mut())); + for (slot, _) in typeof_cache_entries_for_test() { + TYPEOF_CACHE.with(|cells| cells[slot].set(std::ptr::null_mut())); } } @@ -655,15 +648,15 @@ pub(crate) fn reset_typeof_string_cache_for_test() { /// from Rust otherwise means building a BigInt and a registered Symbol. #[cfg(test)] pub(crate) fn populate_typeof_string_cache_for_test() { - for (cache, text) in typeof_cache_entries_for_test() { - get_cached(cache, text); + for (slot, text) in typeof_cache_entries_for_test() { + get_cached(slot, text); } } /// Read the eight cells without populating them. Test-only. #[cfg(test)] pub(crate) fn typeof_string_cache_cells_for_test() -> [*mut StringHeader; 8] { - typeof_cache_entries_for_test().map(|(cache, _)| cache.with(|cell| cell.get())) + typeof_cache_entries_for_test().map(|(slot, _)| TYPEOF_CACHE.with(|cells| cells[slot].get())) } /// Return the typeof a value as a string @@ -678,26 +671,26 @@ pub extern "C" fn js_value_typeof(value: f64) -> *mut StringHeader { let jsval = JSValue::from_bits(value.to_bits()); if jsval.is_undefined() { - get_cached(&TYPEOF_UNDEFINED, "undefined") + get_cached(TYPEOF_UNDEFINED, "undefined") } else if jsval.is_null() { // typeof null === "object" in JavaScript - get_cached(&TYPEOF_OBJECT, "object") + get_cached(TYPEOF_OBJECT, "object") } else if jsval.is_bool() { - get_cached(&TYPEOF_BOOLEAN, "boolean") + get_cached(TYPEOF_BOOLEAN, "boolean") } else if jsval.is_any_string() { // String pointer (STRING_TAG) OR inline SSO (SHORT_STRING_TAG). // `typeof` doesn't distinguish between representations — both // are observed as "string" from user code. - get_cached(&TYPEOF_STRING, "string") + get_cached(TYPEOF_STRING, "string") } else if crate::value::is_js_handle(value) { // JS handle from V8 runtime — ask V8 whether it's a callable, otherwise default // to "object". Issue #258: pre-fix this always returned "object" even for // V8 functions; the registered callback now flips it to "function" when the // handle wraps a v8::Function. if crate::value::js_handle_is_function(value) { - get_cached(&TYPEOF_FUNCTION, "function") + get_cached(TYPEOF_FUNCTION, "function") } else { - get_cached(&TYPEOF_OBJECT, "object") + get_cached(TYPEOF_OBJECT, "object") } } else if jsval.is_pointer() { // Object/array/closure/symbol pointer - check via the side-table first. @@ -713,43 +706,43 @@ pub extern "C" fn js_value_typeof(value: f64) -> *mut StringHeader { // (possibly nested) [[ProxyTarget]] is callable. if crate::proxy::js_proxy_is_proxy(value) == 1 { return if crate::proxy::proxy_wraps_callable(value) { - get_cached(&TYPEOF_FUNCTION, "function") + get_cached(TYPEOF_FUNCTION, "function") } else { - get_cached(&TYPEOF_OBJECT, "object") + get_cached(TYPEOF_OBJECT, "object") }; } if crate::value::addr_class::is_above_handle_band(ptr as usize) { // Symbols: registered in SYMBOL_POINTERS (handles both gc_malloc'd // and Box-leaked symbols, which have no GcHeader). if crate::symbol::is_registered_symbol(ptr as usize) { - get_cached(&TYPEOF_SYMBOL, "symbol") + get_cached(TYPEOF_SYMBOL, "symbol") } else if crate::date::is_date_cell_addr(ptr as usize) { // Date is a NaN-boxed pointer to an 8-byte `DateCell` (#2089). // `typeof aDate === "object"`. Check this BEFORE reading the // `type_tag` at offset 12 below — the cell is only 8 bytes, so // that read would fall off the end of the allocation. - get_cached(&TYPEOF_OBJECT, "object") + get_cached(TYPEOF_OBJECT, "object") } else { // ClosureHeader has type_tag at offset 12 (after func_ptr:8 + capture_count:4) let type_tag = unsafe { *(ptr.add(crate::closure::CLOSURE_TYPE_TAG_OFFSET) as *const u32) }; if type_tag == crate::closure::CLOSURE_MAGIC { - get_cached(&TYPEOF_FUNCTION, "function") + get_cached(TYPEOF_FUNCTION, "function") } else if crate::object::is_class_object_ptr(ptr) { // #1789: a class-expression VALUE is a heap object stamped // with OBJECT_TYPE_CLASS — `typeof aClassObject === // "function"` (classes are callable in JS), matching the // INT32 ClassRef case below. - get_cached(&TYPEOF_FUNCTION, "function") + get_cached(TYPEOF_FUNCTION, "function") } else { - get_cached(&TYPEOF_OBJECT, "object") + get_cached(TYPEOF_OBJECT, "object") } } } else { - get_cached(&TYPEOF_OBJECT, "object") + get_cached(TYPEOF_OBJECT, "object") } } else if jsval.is_bigint() { - get_cached(&TYPEOF_BIGINT, "bigint") + get_cached(TYPEOF_BIGINT, "bigint") } else if jsval.is_int32() { // Refs #618 / #420 followup: class refs share INT32_TAG storage // shape (codegen emits `INT32_TAG | class_id` as the value form @@ -759,9 +752,9 @@ pub extern "C" fn js_value_typeof(value: f64) -> *mut StringHeader { let raw = jsval.bits() & 0xFFFF_FFFF; let class_id = raw as u32; if crate::object::is_class_id_registered(class_id) { - get_cached(&TYPEOF_FUNCTION, "function") + get_cached(TYPEOF_FUNCTION, "function") } else { - get_cached(&TYPEOF_NUMBER, "number") + get_cached(TYPEOF_NUMBER, "number") } } else { // Issue #654: typed-array pointers arrive as a raw `i64 → f64` @@ -776,7 +769,7 @@ pub extern "C" fn js_value_typeof(value: f64) -> *mut StringHeader { if top16 == 0 && bits >= 0x10000 { let addr = bits as usize; if crate::typedarray::lookup_typed_array_kind(addr).is_some() { - return get_cached(&TYPEOF_OBJECT, "object"); + return get_cached(TYPEOF_OBJECT, "object"); } } // Date is now a NaN-boxed `DateCell` pointer (#2089), handled in the @@ -792,12 +785,12 @@ pub extern "C" fn js_value_typeof(value: f64) -> *mut StringHeader { if value.is_finite() && value > 0.0 && value.fract() == 0.0 { if let Some(probe) = crate::object::stream_handle_kind_probe() { if unsafe { probe(value as usize) } != 0 { - return get_cached(&TYPEOF_OBJECT, "object"); + return get_cached(TYPEOF_OBJECT, "object"); } } } // Regular f64 number - get_cached(&TYPEOF_NUMBER, "number") + get_cached(TYPEOF_NUMBER, "number") } } From 370fdfd9a1ca429c807058cdb78b0c93e10c4951 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 17:14:20 +0200 Subject: [PATCH 18/23] perf(gc): validated-parent write barrier entry for codegen's generation-tested slot stores emit_write_barrier_slot_generation_tested dereferences the parent header (GC_FLAG_TENURED) before deciding to call the barrier, then handed the runtime a value that js_write_barrier_slot re-validated: a tag dispatch, an alignment/floor test and a page-generation classification that write_barrier_decoded_parent repeated one call later. The new js_write_barrier_slot_validated_parent takes the raw handle the gate proved and goes straight to the decoded barrier; every parent meeting the contract decides identically (an unregistered gc_malloc parent is refused by the inline-slot rule instead of by decode). On a 5k-entity ECS frame the bucket pushes classified each old parent twice per command. Claude-Session: https://claude.ai/code/session_01FUvFrRNZyc5qknBiJbYbby --- .../src/expr/barrier_stem_census_tests.rs | 2 +- .../src/expr/computed_store_rooting_tests.rs | 2 +- .../perry-codegen/src/expr/write_barrier.rs | 22 ++++++++- .../src/expr/write_pic_barrier_tests.rs | 2 +- crates/perry-codegen/src/gc_call_effects.rs | 1 + .../lower_call/ctor_prologue_store_tests.rs | 2 +- crates/perry-codegen/src/root_reload.rs | 1 + .../perry-codegen/src/runtime_decls/arrays.rs | 5 +++ crates/perry-runtime/src/gc/barrier/mod.rs | 33 ++++++++++++++ .../src/gc/tests/barrier_decoded_parent.rs | 45 +++++++++++++++++++ 10 files changed, 109 insertions(+), 6 deletions(-) diff --git a/crates/perry-codegen/src/expr/barrier_stem_census_tests.rs b/crates/perry-codegen/src/expr/barrier_stem_census_tests.rs index 1eabf77b20..80b6200dc5 100644 --- a/crates/perry-codegen/src/expr/barrier_stem_census_tests.rs +++ b/crates/perry-codegen/src/expr/barrier_stem_census_tests.rs @@ -94,7 +94,7 @@ pub(super) const VERIFIED_BARRIER_STEMS: &[(&str, StemKind)] = &[ ("put.pic", StemKind::PointerTestedStore), ]; -const BARRIER_CALL: &str = "call void @js_write_barrier_slot("; +const BARRIER_CALL: &str = "call void @js_write_barrier_slot"; const INCREMENTAL_GLOBAL: &str = "@PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT"; // --------------------------------------------------------------------------- diff --git a/crates/perry-codegen/src/expr/computed_store_rooting_tests.rs b/crates/perry-codegen/src/expr/computed_store_rooting_tests.rs index 264817023a..3e0519128a 100644 --- a/crates/perry-codegen/src/expr/computed_store_rooting_tests.rs +++ b/crates/perry-codegen/src/expr/computed_store_rooting_tests.rs @@ -978,7 +978,7 @@ fn growing_array_store_uses_the_reallocated_head_for_its_barrier() { .join("\n"); let barrier = realloc_body .lines() - .find(|line| line.contains("@js_write_barrier_slot(")) + .find(|line| line.contains("@js_write_barrier_slot")) .unwrap_or_else(|| panic!("realloc path lost its write barrier:\n{realloc_body}")); assert!( barrier.contains(&format!("i64 {new_head}")), diff --git a/crates/perry-codegen/src/expr/write_barrier.rs b/crates/perry-codegen/src/expr/write_barrier.rs index b84c06b582..23797d579a 100644 --- a/crates/perry-codegen/src/expr/write_barrier.rs +++ b/crates/perry-codegen/src/expr/write_barrier.rs @@ -221,7 +221,16 @@ pub(crate) fn emit_write_barrier_slot_generation_tested( ctx.current_block = barrier_idx; { let blk = ctx.block(); - emit_write_barrier_slot_on_block(blk, parent_bits, slot_addr, child_bits); + // The gate above just dereferenced `parent_handle`'s header, so the + // runtime need not re-validate or re-classify it: the validated-parent + // entry takes the raw handle and goes straight to the decoded barrier. + // `parent_bits` is deliberately unused on this arm — it is the same + // object, boxed or raw, and the raw handle is what the gate proved. + let _ = parent_bits; + blk.call_void( + "js_write_barrier_slot_validated_parent", + &[(I64, parent_handle), (I64, slot_addr), (I64, child_bits)], + ); blk.br(&done_label); } ctx.current_block = done_idx; @@ -305,7 +314,16 @@ pub(crate) fn emit_write_barrier_slot_value_and_generation_tested( ctx.current_block = barrier_idx; { let blk = ctx.block(); - emit_write_barrier_slot_on_block(blk, parent_bits, slot_addr, child_bits); + // The gate above just dereferenced `parent_handle`'s header, so the + // runtime need not re-validate or re-classify it: the validated-parent + // entry takes the raw handle and goes straight to the decoded barrier. + // `parent_bits` is deliberately unused on this arm — it is the same + // object, boxed or raw, and the raw handle is what the gate proved. + let _ = parent_bits; + blk.call_void( + "js_write_barrier_slot_validated_parent", + &[(I64, parent_handle), (I64, slot_addr), (I64, child_bits)], + ); blk.br(&done_label); } ctx.current_block = done_idx; diff --git a/crates/perry-codegen/src/expr/write_pic_barrier_tests.rs b/crates/perry-codegen/src/expr/write_pic_barrier_tests.rs index ece425f97d..8a3b0932f8 100644 --- a/crates/perry-codegen/src/expr/write_pic_barrier_tests.rs +++ b/crates/perry-codegen/src/expr/write_pic_barrier_tests.rs @@ -62,7 +62,7 @@ const ADDREF: &str = "call void @js_string_addref_if_heap_string("; /// The trailing `(` is what separates this from `..._aware(`. const NOTE: &str = "call void @js_gc_note_slot_layout("; const NOTE_AWARE: &str = "call void @js_gc_note_slot_layout_aware("; -const BARRIER_CALL: &str = "call void @js_write_barrier_slot("; +const BARRIER_CALL: &str = "call void @js_write_barrier_slot"; const OBJECT: u32 = 1; const VALUE: u32 = 2; diff --git a/crates/perry-codegen/src/gc_call_effects.rs b/crates/perry-codegen/src/gc_call_effects.rs index ed0c3b99e9..7df27f9a10 100644 --- a/crates/perry-codegen/src/gc_call_effects.rs +++ b/crates/perry-codegen/src/gc_call_effects.rs @@ -81,6 +81,7 @@ pub(crate) fn classify_direct_callee(name: &str) -> GcCallEffect { // `gc/barrier.rs`: remembered-set / incremental-marking maintenance. | "js_write_barrier" | "js_write_barrier_slot" + | "js_write_barrier_slot_validated_parent" | "js_write_barrier_root_heap_word" | "js_write_barrier_root_nanbox" // `gc/roots.rs`: registers one module-level global as a root. Audited diff --git a/crates/perry-codegen/src/lower_call/ctor_prologue_store_tests.rs b/crates/perry-codegen/src/lower_call/ctor_prologue_store_tests.rs index 8dcda31e96..bab1fb013a 100644 --- a/crates/perry-codegen/src/lower_call/ctor_prologue_store_tests.rs +++ b/crates/perry-codegen/src/lower_call/ctor_prologue_store_tests.rs @@ -133,7 +133,7 @@ fn an_outlined_pointer_bearing_shape_keeps_its_write_barrier() { ); assert!( ir.contains("ctor_prologue.barrier.maybe") - && ir.contains("call void @js_write_barrier_slot("), + && ir.contains("call void @js_write_barrier_slot"), "the direct pointer store must reach the ordinary write-barrier call:\n{ir}" ); } diff --git a/crates/perry-codegen/src/root_reload.rs b/crates/perry-codegen/src/root_reload.rs index ef8d8965ff..ad322ff5c2 100644 --- a/crates/perry-codegen/src/root_reload.rs +++ b/crates/perry-codegen/src/root_reload.rs @@ -201,6 +201,7 @@ const NON_COLLECTING: &[&str] = &[ "js_write_barrier", "js_write_barrier_root_nanbox", "js_write_barrier_slot", + "js_write_barrier_slot_validated_parent", "js_gc_register_global_root", // pure value predicates / bit twiddling. // diff --git a/crates/perry-codegen/src/runtime_decls/arrays.rs b/crates/perry-codegen/src/runtime_decls/arrays.rs index 6d55c3d482..fd3faf455e 100644 --- a/crates/perry-codegen/src/runtime_decls/arrays.rs +++ b/crates/perry-codegen/src/runtime_decls/arrays.rs @@ -171,6 +171,11 @@ pub fn declare_phase_b_arrays(module: &mut LlModule) { // js_gc_init_typed_shape_layout(obj: u64, slot_count: u32, raw_f64_mask_words: *const u64, raw_f64_mask_word_count: u32, pointer_mask_words: *const u64, pointer_mask_word_count: u32) module.declare_function("js_write_barrier", VOID, &[I64, I64]); module.declare_function("js_write_barrier_slot", VOID, &[I64, I64, I64]); + module.declare_function( + "js_write_barrier_slot_validated_parent", + VOID, + &[I64, I64, I64], + ); module.declare_function("js_write_barrier_root_nanbox", VOID, &[I64]); module.declare_function("js_write_barrier_root_heap_word", VOID, &[I64]); module.declare_function("js_gc_note_slot_layout", VOID, &[I64, I32, I64]); diff --git a/crates/perry-runtime/src/gc/barrier/mod.rs b/crates/perry-runtime/src/gc/barrier/mod.rs index f867b217a4..6418b2c7c3 100644 --- a/crates/perry-runtime/src/gc/barrier/mod.rs +++ b/crates/perry-runtime/src/gc/barrier/mod.rs @@ -1142,6 +1142,39 @@ pub extern "C" fn js_write_barrier_slot(parent: u64, slot_addr: u64, child: u64) write_barrier_slot_inner(parent, slot_addr as usize, child, false); } +/// [`js_write_barrier_slot`] for a parent the caller has ALREADY validated: +/// `parent_user` is the raw (untagged) user pointer of a live, non-forwarded +/// GC object whose header the emitted code dereferenced a few instructions +/// earlier (`emit_parent_may_need_remembering_check` reads `gc_flags`). +/// +/// Skips `decode_heap_addr(parent)` — a tag dispatch, an alignment/floor +/// test and a page-generation classification that `write_barrier_decoded_parent` +/// repeats one call later — and nothing else. For every parent that meets the +/// contract the two entries decide identically: a classified arena parent +/// reaches the same `barrier_parent_needs_remembering`, and an unregistered +/// (`gc_malloc`) parent, which `decode_heap_addr` would have turned into a +/// skip, is refused there instead because an inline slot never qualifies as +/// external. On a 5k-entity ECS frame the buckets' `push` stores were +/// classifying each old parent twice per command. +#[no_mangle] +pub extern "C" fn js_write_barrier_slot_validated_parent( + parent_user: u64, + slot_addr: u64, + child: u64, +) { + let Some(child_addr) = barrier_child_prologue(child) else { + return; + }; + if !barrier_remembering_active() { + return; + } + if parent_user == 0 { + bump_write_barrier_trace_counter(BarrierTraceCounter::NonPointerParentSkips); + return; + } + write_barrier_decoded_parent(parent_user as usize, slot_addr as usize, child_addr, false); +} + pub(super) fn write_barrier_slot_inner( parent: u64, slot_addr: usize, diff --git a/crates/perry-runtime/src/gc/tests/barrier_decoded_parent.rs b/crates/perry-runtime/src/gc/tests/barrier_decoded_parent.rs index d67635fed4..573521a039 100644 --- a/crates/perry-runtime/src/gc/tests/barrier_decoded_parent.rs +++ b/crates/perry-runtime/src/gc/tests/barrier_decoded_parent.rs @@ -64,6 +64,51 @@ fn runtime_write_barrier_slot_remembers_old_to_young_edge() { reset_remembered_set(); } +/// The validated-parent entry codegen takes behind its `GC_FLAG_TENURED` gate +/// must remember exactly what the tag-dispatching entry remembers. +#[test] +fn validated_parent_entry_matches_js_write_barrier_slot() { + let _guard = GcTestIsolationGuard::new(); + reset_remembered_set(); + + let young = crate::arena::arena_alloc_gc(40, 8, GC_TYPE_OBJECT) as usize; + let (old_obj, fields) = unsafe { alloc_old_test_object(1) }; + let child_bits = ptr_bits(young); + unsafe { + *fields = child_bits; + } + let dirty_page = crate::arena::generation_page_for_addr(fields as usize); + assert!(!old_page_dirty_for(dirty_page)); + + crate::gc::barrier::js_write_barrier_slot_validated_parent( + old_obj as u64, + fields as u64, + child_bits, + ); + + assert_eq!( + remembered_dirty_page_count(), + 1, + "old→young store through the validated-parent entry must dirty the slot page" + ); + assert!(old_page_dirty_for(dirty_page)); + + // A young parent is not remembered by either entry. + reset_remembered_set(); + let young_parent = crate::arena::arena_alloc_gc(40, 8, GC_TYPE_OBJECT) as usize; + crate::gc::barrier::js_write_barrier_slot_validated_parent( + young_parent as u64, + (young_parent + 8) as u64, + child_bits, + ); + assert_eq!( + remembered_dirty_page_count(), + 0, + "a young parent is fully traced by every minor and needs no record" + ); + reset_remembered_set(); +} + #[test] fn runtime_write_barrier_slot_matches_nanboxed_entry_point() { let _guard = GcTestIsolationGuard::new(); From 08572a559b05831ca4ab3916a287ddd2483260a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 17:14:20 +0200 Subject: [PATCH 19/23] perf(codegen): read plain closure captures with an inline load instead of js_closure_get_capture_bits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Inside a closure body the helper's null and bounds tests cannot fail — the closure pointer is the rooted %this_closure and the capture index is the layout this compiler assigned — so the read is the one load the call did, at closure + header + 8*index (the same address codegen/closure.rs walks at entry for boxed captures). A benchmark loop closure reading its captured world/entities paid a call per read (3% self). Claude-Session: https://claude.ai/code/session_01FUvFrRNZyc5qknBiJbYbby --- .../perry-codegen/src/expr/literals_vars.rs | 36 +++++++++++++------ 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/crates/perry-codegen/src/expr/literals_vars.rs b/crates/perry-codegen/src/expr/literals_vars.rs index 966c51cbe8..a67d13f71b 100644 --- a/crates/perry-codegen/src/expr/literals_vars.rs +++ b/crates/perry-codegen/src/expr/literals_vars.rs @@ -67,6 +67,29 @@ fn load_trusted_box_capture_bits(ctx: &mut FnCtx<'_>, capture: &TrustedBoxCaptur /// bypasses this rule. Limit the call to declared-string bindings: only those /// bindings can select in-place append, and erased annotations remain safe /// because the runtime helper checks the live tag. +/// Read capture slot `capture_idx` of the running closure as raw bits, inline. +/// +/// `js_closure_get_capture_bits` is a null test, a bounds test against the +/// header's capture count, and one load. Inside a closure body neither test +/// can fail: `closure_ptr` is the rooted `%this_closure` (re-read by +/// `current_closure_ptr_value`, so a relocation is already accounted for), and +/// `capture_idx` is the index this compiler assigned when it laid the closure +/// out — the same layout `codegen/closure.rs` walks at entry for boxed +/// captures. A benchmark loop closure reading its captured world/entities on +/// every iteration paid a call per read; this is the load the call did. +fn load_closure_capture_bits_inline( + ctx: &mut FnCtx<'_>, + closure_ptr: &str, + capture_idx: u32, +) -> String { + let offset = + crate::target_layout::closure_header_size_bytes(ctx.target_triple) + 8 * capture_idx as u64; + let blk = ctx.block(); + let slot_addr = blk.add(I64, closure_ptr, &offset.to_string()); + let slot_ptr = blk.inttoptr(I64, &slot_addr); + blk.load(I64, &slot_ptr) +} + fn demote_extracted_string_binding(ctx: &mut FnCtx<'_>, id: u32, value: &str) { let persistent_binding = ctx.closure_captures.contains_key(&id) || (ctx.boxed_vars.contains(&id) && !ctx.module_globals.contains_key(&id)) @@ -446,7 +469,6 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { } // Captured by closure (from outer scope): if let Some(&capture_idx) = ctx.closure_captures.get(id) { - let idx_str = capture_idx.to_string(); // If the captured id is a boxed var, the capture slot holds a // raw box pointer. Read the capture, extract the box pointer, // and deref via js_box_get_bits. @@ -464,23 +486,15 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { } else { "js_box_get_bits" }; + let box_ptr = load_closure_capture_bits_inline(ctx, &closure_ptr, capture_idx); let blk = ctx.block(); - let box_ptr = blk.call( - I64, - "js_closure_get_capture_bits", - &[(I64, &closure_ptr), (I32, &idx_str)], - ); let bits = blk.call(I64, getter, &[(I64, &box_ptr)]); let value = blk.bitcast_i64_to_double(&bits); demote_extracted_string_binding(ctx, *id, &value); return Ok(value); } let closure_ptr = super::current_closure_ptr_value(ctx, "captured local")?; - let bits = ctx.block().call( - I64, - "js_closure_get_capture_bits", - &[(I64, &closure_ptr), (I32, &idx_str)], - ); + let bits = load_closure_capture_bits_inline(ctx, &closure_ptr, capture_idx); let value = ctx.block().bitcast_i64_to_double(&bits); demote_extracted_string_binding(ctx, *id, &value); return Ok(value); From fbb6fe19da7985763c20e64270830419a68f91ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 17:26:52 +0200 Subject: [PATCH 20/23] perf(runtime): length = 0 keeps an all-pointer array all-pointer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An emptied array holds both vacuous layout claims; keep the one its history predicts. rebuild_array_layout reset every truncated-to-empty array to POINTER_FREE | RAW_F64, so a pool bucket that held pointers and is emptied for reuse paid a layout transition on the first push of every reuse — 5k js_gc_note_slot_layout calls per frame on the ECS command buffer, the one store its inline pointer-layout arm could not elide. An array that was SIDE_MASK | ALL_POINTERS now stays so (the state a declared [] literal starts in); a non-pointer store still demotes it through layout_note_slot exactly as for a declared literal. Everything else keeps the raw-f64 claim. Claude-Session: https://claude.ai/code/session_01FUvFrRNZyc5qknBiJbYbby --- .../src/array/header_gc_slots.rs | 25 +++++-- .../src/gc/tests/layout_trace/array_layout.rs | 65 +++++++++++++++++++ 2 files changed, 86 insertions(+), 4 deletions(-) diff --git a/crates/perry-runtime/src/array/header_gc_slots.rs b/crates/perry-runtime/src/array/header_gc_slots.rs index 77b88eedd4..82a58a2437 100644 --- a/crates/perry-runtime/src/array/header_gc_slots.rs +++ b/crates/perry-runtime/src/array/header_gc_slots.rs @@ -137,16 +137,33 @@ pub(crate) unsafe fn rebuild_array_layout(arr: *mut ArrayHeader) { crate::gc::layout_mark_unknown(arr as *mut u8); return; } + let was_all_pointer = super::header::array_object_flags_resolved(arr) + & (crate::gc::GC_LAYOUT_STATE_MASK | crate::gc::GC_LAYOUT_ALL_POINTERS) + == (crate::gc::GC_LAYOUT_SIDE_MASK | crate::gc::GC_LAYOUT_ALL_POINTERS); crate::gc::layout_rebuild_from_slots(arr as *mut u8, array_elements_ptr(arr), length); if length == 0 { // `layout_rebuild_from_slots` just left the head POINTER_FREE with its // per-object records dropped and the typed-intact bit cleared, which // is everything `refresh_array_numeric_layout` would redo for zero // slots via `rebuild_array_numeric_raw_f64` -> `layout_init_pointer_free` - // (a second header resolution, a second forget probe). All that is - // left of that path is the raw-f64 claim an empty array holds - // vacuously; there are no slots for the old-gen barrier replay either. - super::header::set_array_raw_f64_layout_flag(arr); + // (a second header resolution, a second forget probe). There are no + // slots for the old-gen barrier replay either. + // + // An empty array holds BOTH vacuous claims, so keep the one its + // history predicts. A pool bucket that held pointers and is emptied + // for reuse (`pooled.length = 0`) will take pointers again: leaving + // it `SIDE_MASK | ALL_POINTERS` — the state a declared `[]` literal + // starts in — lets every push take the inline pointer-layout arm, + // where resetting it to POINTER_FREE | RAW_F64 made the first push + // of every reuse pay a layout transition (5k per frame on the ECS + // command buffer). A non-pointer store into that state still demotes + // it to UNKNOWN through `layout_note_slot`, exactly as it does for a + // declared literal. Everything else keeps the raw-f64 claim. + if was_all_pointer { + crate::gc::layout_init_all_pointer_slots(arr as *mut u8); + } else { + super::header::set_array_raw_f64_layout_flag(arr); + } return; } super::header::refresh_array_numeric_layout_resolved(arr); diff --git a/crates/perry-runtime/src/gc/tests/layout_trace/array_layout.rs b/crates/perry-runtime/src/gc/tests/layout_trace/array_layout.rs index 0eccbeaa3c..996e72f564 100644 --- a/crates/perry-runtime/src/gc/tests/layout_trace/array_layout.rs +++ b/crates/perry-runtime/src/gc/tests/layout_trace/array_layout.rs @@ -270,6 +270,71 @@ fn test_array_mixed_bulk_producers_preserve_pointer_layout() { clear_mark_seeds(); } +/// `length = 0` keeps an all-pointer array all-pointer (the state a declared +/// `[]` literal starts in) so a reused pool bucket's first push needs no +/// layout transition, while a numeric array keeps its raw-f64 claim. +#[test] +fn test_truncate_to_zero_keeps_the_layout_the_history_predicts() { + clear_marks(); + clear_mark_seeds(); + + // Pointer bucket: three heap strings, then emptied. + let mut bucket = crate::array::js_array_alloc(4); + for name in [&b"a-child"[..], &b"b-child"[..], &b"c-child"[..]] { + let child = + crate::string::js_string_from_bytes(name.as_ptr(), name.len() as u32) as *mut u8; + let boxed = f64::from_bits(STRING_TAG | (child as u64 & POINTER_MASK)); + bucket = crate::array::js_array_push_f64(bucket, boxed); + } + let before = unsafe { crate::array::array_object_flags_resolved(bucket) }; + assert_eq!( + before & (crate::gc::GC_LAYOUT_STATE_MASK | crate::gc::GC_LAYOUT_ALL_POINTERS), + crate::gc::GC_LAYOUT_SIDE_MASK | crate::gc::GC_LAYOUT_ALL_POINTERS, + "premise: three pointer pushes leave the bucket SIDE_MASK | ALL_POINTERS" + ); + crate::array::js_array_set_length(bucket, 0.0); + let after = unsafe { crate::array::array_object_flags_resolved(bucket) }; + assert_eq!( + after & (crate::gc::GC_LAYOUT_STATE_MASK | crate::gc::GC_LAYOUT_ALL_POINTERS), + crate::gc::GC_LAYOUT_SIDE_MASK | crate::gc::GC_LAYOUT_ALL_POINTERS, + "an emptied all-pointer bucket must stay all-pointer for its next fill" + ); + assert_eq!( + after & crate::gc::GC_ARRAY_RAW_F64_LAYOUT, + 0, + "and must not claim a raw-f64 layout it will never use" + ); + // A non-pointer store into the emptied bucket still demotes the claim. + bucket = crate::array::js_array_push_f64(bucket, 7.0); + let demoted = unsafe { crate::array::array_object_flags_resolved(bucket) }; + assert_ne!( + demoted & (crate::gc::GC_LAYOUT_STATE_MASK | crate::gc::GC_LAYOUT_ALL_POINTERS), + crate::gc::GC_LAYOUT_SIDE_MASK | crate::gc::GC_LAYOUT_ALL_POINTERS, + "a number pushed into the kept all-pointer state must leave it" + ); + assert_eq!( + test_layout_pointer_slot_count(bucket as usize, 1), + None, + "the demotion is to the conservative tag scan (UNKNOWN), the same state a \ + declared literal takes on its first non-pointer store" + ); + + // Numeric array: the raw-f64 claim survives truncation as before. + let mut nums = crate::array::js_array_alloc(4); + nums = crate::array::js_array_push_f64(nums, 1.0); + nums = crate::array::js_array_push_f64(nums, 2.0); + crate::array::js_array_set_length(nums, 0.0); + let flags = unsafe { crate::array::array_object_flags_resolved(nums) }; + assert_eq!( + flags & crate::gc::GC_LAYOUT_STATE_MASK, + crate::gc::GC_LAYOUT_POINTER_FREE + ); + assert_ne!(flags & crate::gc::GC_ARRAY_RAW_F64_LAYOUT, 0); + + clear_marks(); + clear_mark_seeds(); +} + #[test] fn test_numeric_array_push_heap_value_transitions_and_traces() { clear_marks(); From cce40a37506fb508f9b47bed65a541be13fad1df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 17:44:33 +0200 Subject: [PATCH 21/23] chore: keep the follow-up under the lint gates - split indexing.rs (prototype-index latches, strict-store throwers, keys-array slot helpers -> array/indexing_support.rs), array/tests.rs (store-lane tests -> array/strict_store_tests.rs) and gc/barrier/mod.rs (slot-form barrier entry points -> gc/barrier_store.rs, where their runtime twin already lives) back under the 2000-line cap; pure moves plus visibility. - push_pop::resolved_plain_array_flags goes through array_gc_header instead of a hand-rolled address floor (addr-class ratchet). - cargo fmt over the files the branch touched. Claude-Session: https://claude.ai/code/session_01FUvFrRNZyc5qknBiJbYbby --- .../src/lower_call/typed_shape_init.rs | 3 +- crates/perry-runtime/src/array/fill_extend.rs | 2 +- crates/perry-runtime/src/array/header.rs | 7 - crates/perry-runtime/src/array/indexing.rs | 244 +---------------- .../src/array/indexing_support.rs | 248 ++++++++++++++++++ crates/perry-runtime/src/array/mod.rs | 21 +- crates/perry-runtime/src/array/push_pop.rs | 5 +- .../src/array/strict_store_tests.rs | 67 +++++ crates/perry-runtime/src/array/tests.rs | 63 ----- crates/perry-runtime/src/gc/barrier/mod.rs | 126 +-------- crates/perry-runtime/src/gc/barrier_store.rs | 131 ++++++++- crates/perry-runtime/src/gc/layout_tables.rs | 5 +- .../src/gc/tests/barrier_decoded_parent.rs | 4 +- 13 files changed, 471 insertions(+), 455 deletions(-) create mode 100644 crates/perry-runtime/src/array/indexing_support.rs create mode 100644 crates/perry-runtime/src/array/strict_store_tests.rs diff --git a/crates/perry-codegen/src/lower_call/typed_shape_init.rs b/crates/perry-codegen/src/lower_call/typed_shape_init.rs index 3cd4460eed..5aa9720dd0 100644 --- a/crates/perry-codegen/src/lower_call/typed_shape_init.rs +++ b/crates/perry-codegen/src/lower_call/typed_shape_init.rs @@ -213,8 +213,7 @@ fn emit_gated_forget_object_layout(ctx: &mut FnCtx<'_>, obj_handle: &str) { ctx.current_block = young_idx; { let blk = ctx.block(); - let young = - blk.load_atomic_monotonic(crate::types::I32, "@PERRY_YOUNG_LAYOUT_RECORDS", 4); + let young = blk.load_atomic_monotonic(crate::types::I32, "@PERRY_YOUNG_LAYOUT_RECORDS", 4); let any_young = blk.icmp_ne(crate::types::I32, &young, "0"); blk.cond_br(&any_young, &sketch_label, &done_label); } diff --git a/crates/perry-runtime/src/array/fill_extend.rs b/crates/perry-runtime/src/array/fill_extend.rs index e014ea18db..29853f7dab 100644 --- a/crates/perry-runtime/src/array/fill_extend.rs +++ b/crates/perry-runtime/src/array/fill_extend.rs @@ -8,7 +8,7 @@ //! loop-pattern lowering in `perry-codegen/src/stmt/loops.rs`. use super::header::{array_numeric_layout, NumericArrayLayout}; -use super::indexing::note_array_index_write; +use super::indexing_support::note_array_index_write; use super::*; use std::ptr; diff --git a/crates/perry-runtime/src/array/header.rs b/crates/perry-runtime/src/array/header.rs index eb8c796ff5..a9b7136fd2 100644 --- a/crates/perry-runtime/src/array/header.rs +++ b/crates/perry-runtime/src/array/header.rs @@ -493,13 +493,6 @@ pub(crate) unsafe fn array_named_property_get_by_name( }) } -/// Whether this Array owns any side-table properties. -/// -/// Numeric properties normally live in dense element storage, but a far -/// sparse index can enter this table and later fall below a grown capacity. -/// Bulk element operations use this predicate to decline a dense-only path -/// instead of leaving that second representation observable. -#[inline] /// Does this (already resolved) array head carry named properties in the /// side table? Answered by the monotone latch first: until some array has /// taken a named property, the table has always been empty. diff --git a/crates/perry-runtime/src/array/indexing.rs b/crates/perry-runtime/src/array/indexing.rs index b6c93c08a3..e75ae7c31e 100644 --- a/crates/perry-runtime/src/array/indexing.rs +++ b/crates/perry-runtime/src/array/indexing.rs @@ -1,7 +1,8 @@ //! Indexing — length / element get / element set / hybrid string-or-index dispatch. +use super::indexing_support::*; use super::*; use std::ptr; -use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; +use std::sync::atomic::Ordering; const MAX_DENSE_ARRAY_GROW_LENGTH: u32 = 1_000_000; @@ -15,143 +16,6 @@ const MAX_DENSE_ARRAY_GROW_LENGTH: u32 = 1_000_000; /// benchmark for 6 hours (Regression Check, v0.5.1129–v0.5.1150). const DENSE_ARRAY_GAP_LIMIT: u32 = 1024; -/// A strict-mode element write (`arr[i] = v`) to a **frozen** array's existing -/// index is `[[Set]]` on a non-writable data property with `Throw = true` -/// (ECMA-262 §10.4.2.4 → OrdinarySetWithOwnDescriptor step 2.b.i), so it must -/// throw a **TypeError** rather than silently no-op. Perry compiles everything -/// strict, so the codegen `arr[i] = v` fast paths — which call these -/// `js_array_set_f64*` helpers directly — carry the strict-`Set` contract. -/// Matches V8's message. (test262 built-ins/Array element-write-on-frozen.) -#[cold] -fn throw_frozen_array_index_write(index: u32) -> ! { - crate::collection_iter::throw_type_error(&format!( - "Cannot assign to read only property '{index}' of object '[object Array]'" - )); -} - -/// A strict-mode write that would *add* a new index to a non-extensible -/// (frozen / sealed / preventExtensions'd) array — `arr[i] = v` with -/// `i >= length` — is `CreateDataProperty` on a non-extensible object with -/// `Throw = true`, so it must throw a **TypeError**. Matches V8's message. -#[cold] -fn throw_array_not_extensible_add(index: u32) -> ! { - crate::collection_iter::throw_type_error(&format!( - "Cannot add property {index}, object is not extensible" - )); -} - -/// Sticky flag: someone installed an indexed property on `Array.prototype`. -/// An out-of-bounds element read on an ordinary array must fall through to -/// `Array.prototype[index]` (ECMA-262 OrdinaryGet -> prototype chain), but in -/// real code nobody adds numeric indices there, so the hot OOB path stays a -/// single relaxed atomic load until the (rare) write flips this. The address -/// it is compared against lives in [`super::prototype_addr`], which also owns -/// the GC hazard that address carries (#6981). -static ARRAY_PROTO_HAS_INDEX: AtomicBool = AtomicBool::new(false); - -/// Same idea for `Object.prototype`: a numeric index installed there -/// (`Object.prototype[2] = 2`, or a defineProperty accessor) shows through -/// array HOLES and OOB reads (chain: arr -> Array.prototype -> -/// Object.prototype; test262 concat/S15.4.4.4_A3_T3). Flipped by the object -/// index-write/defineProperty hooks; consulted by the typed-feedback guards -/// and the hole/OOB read fallbacks. -static OBJECT_PROTO_HAS_INDEX: AtomicBool = AtomicBool::new(false); - -/// Sticky summary of the process-wide conditions that invalidate codegen's -/// inline plain-array index guard. The generated guard loads this byte -/// directly; keeping the three rare prototype conditions behind one exported -/// byte avoids an out-of-line runtime call on every array read. -#[no_mangle] -pub static PERRY_ARRAY_INDEX_FAST_PATH_INVALIDATED: AtomicU8 = AtomicU8::new(0); - -#[inline] -pub(crate) fn invalidate_array_index_fast_path() { - PERRY_ARRAY_INDEX_FAST_PATH_INVALIDATED.store(1, Ordering::Relaxed); -} - -/// Test-only companion to -/// `prototype_chain::test_swap_array_static_proto_recorded`: swap the summary -/// byte generated code reads, returning the previous value. Only for a test -/// that knowingly set it and is putting the process back as it found it. -#[cfg(test)] -pub(crate) fn test_swap_array_index_fast_path_invalidated(value: u8) -> u8 { - PERRY_ARRAY_INDEX_FAST_PATH_INVALIDATED.swap(value, Ordering::Relaxed) -} - -/// Record (if `obj` is the canonical `Object.prototype`) that it now carries -/// an indexed property. Called from the object index-write / numeric -/// defineProperty paths; cheap (relaxed loads + compare). -#[inline] -pub(crate) fn note_object_prototype_index_write(obj: usize) { - if !OBJECT_PROTO_HAS_INDEX.load(Ordering::Relaxed) && obj != 0 && obj == object_prototype_addr() - { - OBJECT_PROTO_HAS_INDEX.store(true, Ordering::Relaxed); - invalidate_array_index_fast_path(); - } -} - -pub(crate) fn object_prototype_has_index_flag() -> bool { - OBJECT_PROTO_HAS_INDEX.load(Ordering::Relaxed) -} - -/// Sticky flag: user code replaced or deleted `Array.prototype[Symbol.iterator]`. -/// `js_get_iterator`'s array short-circuit assumes the builtin values iterator; -/// once this flips, GetIterator on an array must consult the (patched) method -/// per spec — or throw TypeError when it was deleted. Same single-relaxed-load -/// hot-path shape as `ARRAY_PROTO_HAS_INDEX` above. -static ARRAY_PROTO_ITERATOR_MODIFIED: AtomicBool = AtomicBool::new(false); - -/// The same fact as [`ARRAY_PROTO_ITERATOR_MODIFIED`], exported so GENERATED -/// code can read it (#7760 item 1). -/// -/// `for…of` over a statically-proven array desugars to an index loop -/// (`__i < __arr.length` / `__arr[__i]`) in HIR lowering, which never consults -/// the iteration protocol — so a patched `Array.prototype[Symbol.iterator]` was -/// ignored there even after the spread paths were fixed (#7542). The loop now -/// branches on this flag ONCE at entry, which is also what the spec wants: -/// `for…of` performs GetIterator exactly once, so a patch landing mid-loop must -/// not change the iterator already in hand. -/// -/// A separate `u8` global rather than exposing the `AtomicBool`: codegen emits -/// a plain volatile `i8` load, the same shape as -/// `PERRY_ARRAY_INDEX_FAST_PATH_INVALIDATED`, so the fast arm pays one load and -/// a predictable branch per LOOP — not per iteration — and the index loop -/// itself is emitted byte-identically to before. -#[no_mangle] -pub static PERRY_ARRAY_PROTO_ITERATOR_PATCHED: AtomicU8 = AtomicU8::new(0); - -/// Record (if `obj` is `Array.prototype` and `sym_key` is the well-known -/// `Symbol.iterator`) that the array iteration protocol has been tampered -/// with. Called from the symbol-property set/delete paths. -pub(crate) fn note_array_proto_iterator_write(obj: usize, sym_key: usize) { - if ARRAY_PROTO_ITERATOR_MODIFIED.load(Ordering::Relaxed) || obj == 0 || sym_key == 0 { - return; - } - if obj == array_prototype_addr() - && sym_key == crate::symbol::well_known_symbol("iterator") as usize - { - ARRAY_PROTO_ITERATOR_MODIFIED.store(true, Ordering::Relaxed); - // Publish to generated code. Release so a loop that observes the `1` - // also observes the prototype write that preceded it. - PERRY_ARRAY_PROTO_ITERATOR_PATCHED.store(1, Ordering::Release); - } -} - -pub(crate) fn array_proto_iterator_modified() -> bool { - ARRAY_PROTO_ITERATOR_MODIFIED.load(Ordering::Relaxed) -} - -/// Record (if `arr` is `Array.prototype`) that the prototype now carries an -/// indexed property, so subsequent out-of-bounds reads consult it. Called from -/// the array element-write paths; cheap (two relaxed atomic loads + compare). -#[inline] -pub(crate) fn note_array_index_write(arr: usize) { - if !ARRAY_PROTO_HAS_INDEX.load(Ordering::Relaxed) && arr != 0 && arr == array_prototype_addr() { - ARRAY_PROTO_HAS_INDEX.store(true, Ordering::Relaxed); - invalidate_array_index_fast_path(); - } -} - /// Out-of-bounds element read fallback: `Array.prototype[index]` when the /// prototype has indexed properties (see `ARRAY_PROTO_HAS_INDEX`). Returns the /// inherited value, or `undefined` if absent. Skipped entirely when the @@ -533,110 +397,6 @@ fn array_get_property_by_key(arr: *const ArrayHeader, key: *const crate::StringH f64::from_bits(value.bits()) } -#[no_mangle] -/// Reported length of an object's keys/property array, capped at its physical -/// capacity. -/// -/// Object property walks (the wide-key field-get index and `Object.assign`'s -/// source enumeration) size their work by the keys array's length. A dense -/// keys array's logical length can never exceed its capacity, so for a -/// well-formed array this is a no-op. But when a keys array is malformed and -/// `js_array_length` reports a bogus, oversized value (observed: a pointer- -/// sized length ~= the keys pointer's own low bits, far beyond the real key -/// count), an unclamped `for i in 0..len` / `HashMap::with_capacity(len)` turns -/// a single missing-property read or `Object.assign` into a multi-GB / minutes- -/// long spin. Capping to capacity bounds that work to physically-present slots. -/// -/// FOR DENSE KEYS/PROPERTY ARRAYS ONLY — general JS arrays may have -/// `length > capacity` (sparse), where this cap would be incorrect. -pub(crate) unsafe fn keys_array_len_capped_to_capacity(arr: *const ArrayHeader) -> usize { - // #7765: a well-formed dense keys array answers from its own two words. - // `js_array_length` re-derives the same number through a proxy probe, a - // second header read for its lazy/object arms, and a `clean_arr_ptr` - // forwarding walk — once per property read on the field-get funnel. - // `length <= capacity` is exactly the well-formed case; the sparse and - // corrupted shapes this cap exists for fall through unchanged. - if let Some(header) = crate::value::addr_class::try_read_gc_header(arr as usize) { - if header.obj_type == crate::gc::GC_TYPE_ARRAY - && header.gc_flags & crate::gc::GC_FLAG_FORWARDED == 0 - && (*arr).length <= (*arr).capacity - { - return (*arr).length as usize; - } - } - // A forwarding stub overwrites the old payload's `(length, capacity)` - // words with the target address. Resolve once, then read BOTH facts from - // the live header; mixing a resolved length with the stale from-space - // capacity can truncate an otherwise exact shape count. - let live = clean_arr_ptr(arr); - if live.is_null() { - return js_array_length(arr) as usize; - } - let raw = js_array_length(live) as usize; - raw.min((*live).capacity as usize) -} - -/// Read slot `index` of a dense internal keys/property array. -/// -/// The object field-get funnel has already proved `keys` is a live -/// `GC_TYPE_ARRAY` — it reads the `GcHeader` and returns `undefined` otherwise -/// — and has capped `index` below the array's own capacity (see -/// [`keys_array_len_capped_to_capacity`]). Those are precisely the two facts -/// [`js_array_get_f64`] re-establishes from scratch on every call: a -/// `clean_arr_ptr` forwarding walk, a lazy-header probe, the exotic-receiver -/// classifications and a descriptor-flag read — per key examined, per property -/// read. On `gc-handoff/apps/asyncpipe_big.ts` that one funnel was 78% of all -/// `js_array_get_f64` samples. -/// -/// Falls back to the general getter for anything it cannot serve on those -/// terms — a forwarded array (which `clean_arr_ptr` would relocate), one -/// carrying index descriptors, an out-of-range index, or a hole (which reads -/// through the prototype chain) — so no general semantics move. Keys arrays -/// are dense and descriptor-free, so the fallback is the cold arm. -#[inline] -pub(crate) unsafe fn keys_array_slot( - keys: *const ArrayHeader, - index: u32, -) -> crate::value::JSValue { - if let Some(header) = crate::value::addr_class::try_read_gc_header(keys as usize) { - if header.obj_type == crate::gc::GC_TYPE_ARRAY - && header.gc_flags & crate::gc::GC_FLAG_FORWARDED == 0 - && header._reserved & crate::gc::OBJ_FLAG_ARRAY_DESCRIPTORS == 0 - && index < (*keys).length - && index < (*keys).capacity - { - let elements = - (keys as *const u8).add(std::mem::size_of::()) as *const f64; - let raw = std::ptr::read(elements.add(index as usize)); - if raw.to_bits() != crate::value::TAG_HOLE { - return crate::value::JSValue::from_bits(raw.to_bits()); - } - } - } - #[cfg(test)] - KEYS_ARRAY_SLOT_FALLBACKS.with(|c| c.set(c.get().wrapping_add(1))); - crate::array::js_array_get(keys, index) -} - -#[cfg(test)] -thread_local! { -/// Times [`keys_array_slot`] could NOT serve a slot from the dense words and -/// had to delegate. Asserted in both directions by -/// `array::collection_tag_tests` — zero for the dense keys arrays the fast path -/// exists for, non-zero for every shape it must refuse — so a fast path that -/// silently stopped applying, or one that started swallowing a shape it should -/// have delegated, both go red. -/// -/// Per THREAD — `cargo test` runs every case on its own thread in one process, -/// so a process-global counter would be moved by whatever else is running. - static KEYS_ARRAY_SLOT_FALLBACKS: std::cell::Cell = const { std::cell::Cell::new(0) }; -} - -#[cfg(test)] -pub(crate) fn test_keys_array_slot_fallbacks() -> u64 { - KEYS_ARRAY_SLOT_FALLBACKS.with(|c| c.get()) -} - /// Auto-opt dead-strip anchor: codegen emits a bare `js_array_length` symbol in /// native-region wrappers (`__perry_wrap_*`) and elsewhere, so it must be a /// `#[no_mangle]` C export AND survive dead-stripping even when no Rust caller diff --git a/crates/perry-runtime/src/array/indexing_support.rs b/crates/perry-runtime/src/array/indexing_support.rs new file mode 100644 index 0000000000..df59ef5ac3 --- /dev/null +++ b/crates/perry-runtime/src/array/indexing_support.rs @@ -0,0 +1,248 @@ +//! Indexing support split out of `indexing.rs` to keep it under the repo's +//! 2000-line cap: the strict-store TypeError throwers, the prototype +//! indexed-property / iterator invalidation latches, and the dense keys-array +//! slot helpers. Pure move except for the `use` lines and `pub(super)` +//! visibility on items `indexing.rs` still calls. +use super::*; +use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; + +/// A strict-mode element write (`arr[i] = v`) to a **frozen** array's existing +/// index is `[[Set]]` on a non-writable data property with `Throw = true` +/// (ECMA-262 §10.4.2.4 → OrdinarySetWithOwnDescriptor step 2.b.i), so it must +/// throw a **TypeError** rather than silently no-op. Perry compiles everything +/// strict, so the codegen `arr[i] = v` fast paths — which call these +/// `js_array_set_f64*` helpers directly — carry the strict-`Set` contract. +/// Matches V8's message. (test262 built-ins/Array element-write-on-frozen.) +#[cold] +pub(super) fn throw_frozen_array_index_write(index: u32) -> ! { + crate::collection_iter::throw_type_error(&format!( + "Cannot assign to read only property '{index}' of object '[object Array]'" + )); +} + +/// A strict-mode write that would *add* a new index to a non-extensible +/// (frozen / sealed / preventExtensions'd) array — `arr[i] = v` with +/// `i >= length` — is `CreateDataProperty` on a non-extensible object with +/// `Throw = true`, so it must throw a **TypeError**. Matches V8's message. +#[cold] +pub(super) fn throw_array_not_extensible_add(index: u32) -> ! { + crate::collection_iter::throw_type_error(&format!( + "Cannot add property {index}, object is not extensible" + )); +} + +/// Sticky flag: someone installed an indexed property on `Array.prototype`. +/// An out-of-bounds element read on an ordinary array must fall through to +/// `Array.prototype[index]` (ECMA-262 OrdinaryGet -> prototype chain), but in +/// real code nobody adds numeric indices there, so the hot OOB path stays a +/// single relaxed atomic load until the (rare) write flips this. The address +/// it is compared against lives in [`super::prototype_addr`], which also owns +/// the GC hazard that address carries (#6981). +pub(super) static ARRAY_PROTO_HAS_INDEX: AtomicBool = AtomicBool::new(false); + +/// Same idea for `Object.prototype`: a numeric index installed there +/// (`Object.prototype[2] = 2`, or a defineProperty accessor) shows through +/// array HOLES and OOB reads (chain: arr -> Array.prototype -> +/// Object.prototype; test262 concat/S15.4.4.4_A3_T3). Flipped by the object +/// index-write/defineProperty hooks; consulted by the typed-feedback guards +/// and the hole/OOB read fallbacks. +pub(super) static OBJECT_PROTO_HAS_INDEX: AtomicBool = AtomicBool::new(false); + +/// Sticky summary of the process-wide conditions that invalidate codegen's +/// inline plain-array index guard. The generated guard loads this byte +/// directly; keeping the three rare prototype conditions behind one exported +/// byte avoids an out-of-line runtime call on every array read. +#[no_mangle] +pub static PERRY_ARRAY_INDEX_FAST_PATH_INVALIDATED: AtomicU8 = AtomicU8::new(0); + +#[inline] +pub(crate) fn invalidate_array_index_fast_path() { + PERRY_ARRAY_INDEX_FAST_PATH_INVALIDATED.store(1, Ordering::Relaxed); +} + +/// Test-only companion to +/// `prototype_chain::test_swap_array_static_proto_recorded`: swap the summary +/// byte generated code reads, returning the previous value. Only for a test +/// that knowingly set it and is putting the process back as it found it. +#[cfg(test)] +pub(crate) fn test_swap_array_index_fast_path_invalidated(value: u8) -> u8 { + PERRY_ARRAY_INDEX_FAST_PATH_INVALIDATED.swap(value, Ordering::Relaxed) +} + +/// Record (if `obj` is the canonical `Object.prototype`) that it now carries +/// an indexed property. Called from the object index-write / numeric +/// defineProperty paths; cheap (relaxed loads + compare). +#[inline] +pub(crate) fn note_object_prototype_index_write(obj: usize) { + if !OBJECT_PROTO_HAS_INDEX.load(Ordering::Relaxed) && obj != 0 && obj == object_prototype_addr() + { + OBJECT_PROTO_HAS_INDEX.store(true, Ordering::Relaxed); + invalidate_array_index_fast_path(); + } +} + +pub(crate) fn object_prototype_has_index_flag() -> bool { + OBJECT_PROTO_HAS_INDEX.load(Ordering::Relaxed) +} + +/// Sticky flag: user code replaced or deleted `Array.prototype[Symbol.iterator]`. +/// `js_get_iterator`'s array short-circuit assumes the builtin values iterator; +/// once this flips, GetIterator on an array must consult the (patched) method +/// per spec — or throw TypeError when it was deleted. Same single-relaxed-load +/// hot-path shape as `ARRAY_PROTO_HAS_INDEX` above. +pub(super) static ARRAY_PROTO_ITERATOR_MODIFIED: AtomicBool = AtomicBool::new(false); + +/// The same fact as [`ARRAY_PROTO_ITERATOR_MODIFIED`], exported so GENERATED +/// code can read it (#7760 item 1). +/// +/// `for…of` over a statically-proven array desugars to an index loop +/// (`__i < __arr.length` / `__arr[__i]`) in HIR lowering, which never consults +/// the iteration protocol — so a patched `Array.prototype[Symbol.iterator]` was +/// ignored there even after the spread paths were fixed (#7542). The loop now +/// branches on this flag ONCE at entry, which is also what the spec wants: +/// `for…of` performs GetIterator exactly once, so a patch landing mid-loop must +/// not change the iterator already in hand. +/// +/// A separate `u8` global rather than exposing the `AtomicBool`: codegen emits +/// a plain volatile `i8` load, the same shape as +/// `PERRY_ARRAY_INDEX_FAST_PATH_INVALIDATED`, so the fast arm pays one load and +/// a predictable branch per LOOP — not per iteration — and the index loop +/// itself is emitted byte-identically to before. +#[no_mangle] +pub static PERRY_ARRAY_PROTO_ITERATOR_PATCHED: AtomicU8 = AtomicU8::new(0); + +/// Record (if `obj` is `Array.prototype` and `sym_key` is the well-known +/// `Symbol.iterator`) that the array iteration protocol has been tampered +/// with. Called from the symbol-property set/delete paths. +pub(crate) fn note_array_proto_iterator_write(obj: usize, sym_key: usize) { + if ARRAY_PROTO_ITERATOR_MODIFIED.load(Ordering::Relaxed) || obj == 0 || sym_key == 0 { + return; + } + if obj == array_prototype_addr() + && sym_key == crate::symbol::well_known_symbol("iterator") as usize + { + ARRAY_PROTO_ITERATOR_MODIFIED.store(true, Ordering::Relaxed); + // Publish to generated code. Release so a loop that observes the `1` + // also observes the prototype write that preceded it. + PERRY_ARRAY_PROTO_ITERATOR_PATCHED.store(1, Ordering::Release); + } +} + +pub(crate) fn array_proto_iterator_modified() -> bool { + ARRAY_PROTO_ITERATOR_MODIFIED.load(Ordering::Relaxed) +} + +/// Record (if `arr` is `Array.prototype`) that the prototype now carries an +/// indexed property, so subsequent out-of-bounds reads consult it. Called from +/// the array element-write paths; cheap (two relaxed atomic loads + compare). +#[inline] +pub(crate) fn note_array_index_write(arr: usize) { + if !ARRAY_PROTO_HAS_INDEX.load(Ordering::Relaxed) && arr != 0 && arr == array_prototype_addr() { + ARRAY_PROTO_HAS_INDEX.store(true, Ordering::Relaxed); + invalidate_array_index_fast_path(); + } +} + +#[no_mangle] +/// Reported length of an object's keys/property array, capped at its physical +/// capacity. +/// +/// Object property walks (the wide-key field-get index and `Object.assign`'s +/// source enumeration) size their work by the keys array's length. A dense +/// keys array's logical length can never exceed its capacity, so for a +/// well-formed array this is a no-op. But when a keys array is malformed and +/// `js_array_length` reports a bogus, oversized value (observed: a pointer- +/// sized length ~= the keys pointer's own low bits, far beyond the real key +/// count), an unclamped `for i in 0..len` / `HashMap::with_capacity(len)` turns +/// a single missing-property read or `Object.assign` into a multi-GB / minutes- +/// long spin. Capping to capacity bounds that work to physically-present slots. +/// +/// FOR DENSE KEYS/PROPERTY ARRAYS ONLY — general JS arrays may have +/// `length > capacity` (sparse), where this cap would be incorrect. +pub(crate) unsafe fn keys_array_len_capped_to_capacity(arr: *const ArrayHeader) -> usize { + // #7765: a well-formed dense keys array answers from its own two words. + // `js_array_length` re-derives the same number through a proxy probe, a + // second header read for its lazy/object arms, and a `clean_arr_ptr` + // forwarding walk — once per property read on the field-get funnel. + // `length <= capacity` is exactly the well-formed case; the sparse and + // corrupted shapes this cap exists for fall through unchanged. + if let Some(header) = crate::value::addr_class::try_read_gc_header(arr as usize) { + if header.obj_type == crate::gc::GC_TYPE_ARRAY + && header.gc_flags & crate::gc::GC_FLAG_FORWARDED == 0 + && (*arr).length <= (*arr).capacity + { + return (*arr).length as usize; + } + } + // A forwarding stub overwrites the old payload's `(length, capacity)` + // words with the target address. Resolve once, then read BOTH facts from + // the live header; mixing a resolved length with the stale from-space + // capacity can truncate an otherwise exact shape count. + let live = clean_arr_ptr(arr); + if live.is_null() { + return js_array_length(arr) as usize; + } + let raw = js_array_length(live) as usize; + raw.min((*live).capacity as usize) +} + +/// Read slot `index` of a dense internal keys/property array. +/// +/// The object field-get funnel has already proved `keys` is a live +/// `GC_TYPE_ARRAY` — it reads the `GcHeader` and returns `undefined` otherwise +/// — and has capped `index` below the array's own capacity (see +/// [`keys_array_len_capped_to_capacity`]). Those are precisely the two facts +/// [`js_array_get_f64`] re-establishes from scratch on every call: a +/// `clean_arr_ptr` forwarding walk, a lazy-header probe, the exotic-receiver +/// classifications and a descriptor-flag read — per key examined, per property +/// read. On `gc-handoff/apps/asyncpipe_big.ts` that one funnel was 78% of all +/// `js_array_get_f64` samples. +/// +/// Falls back to the general getter for anything it cannot serve on those +/// terms — a forwarded array (which `clean_arr_ptr` would relocate), one +/// carrying index descriptors, an out-of-range index, or a hole (which reads +/// through the prototype chain) — so no general semantics move. Keys arrays +/// are dense and descriptor-free, so the fallback is the cold arm. +#[inline] +pub(crate) unsafe fn keys_array_slot( + keys: *const ArrayHeader, + index: u32, +) -> crate::value::JSValue { + if let Some(header) = crate::value::addr_class::try_read_gc_header(keys as usize) { + if header.obj_type == crate::gc::GC_TYPE_ARRAY + && header.gc_flags & crate::gc::GC_FLAG_FORWARDED == 0 + && header._reserved & crate::gc::OBJ_FLAG_ARRAY_DESCRIPTORS == 0 + && index < (*keys).length + && index < (*keys).capacity + { + let elements = + (keys as *const u8).add(std::mem::size_of::()) as *const f64; + let raw = std::ptr::read(elements.add(index as usize)); + if raw.to_bits() != crate::value::TAG_HOLE { + return crate::value::JSValue::from_bits(raw.to_bits()); + } + } + } + #[cfg(test)] + KEYS_ARRAY_SLOT_FALLBACKS.with(|c| c.set(c.get().wrapping_add(1))); + crate::array::js_array_get(keys, index) +} + +#[cfg(test)] +thread_local! { +/// Times [`keys_array_slot`] could NOT serve a slot from the dense words and +/// had to delegate. Asserted in both directions by +/// `array::collection_tag_tests` — zero for the dense keys arrays the fast path +/// exists for, non-zero for every shape it must refuse — so a fast path that +/// silently stopped applying, or one that started swallowing a shape it should +/// have delegated, both go red. +/// +/// Per THREAD — `cargo test` runs every case on its own thread in one process, +/// so a process-global counter would be moved by whatever else is running. + static KEYS_ARRAY_SLOT_FALLBACKS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn test_keys_array_slot_fallbacks() -> u64 { + KEYS_ARRAY_SLOT_FALLBACKS.with(|c| c.get()) +} diff --git a/crates/perry-runtime/src/array/mod.rs b/crates/perry-runtime/src/array/mod.rs index 58c1cf4b0c..7ba76812c7 100644 --- a/crates/perry-runtime/src/array/mod.rs +++ b/crates/perry-runtime/src/array/mod.rs @@ -13,6 +13,7 @@ mod header; mod header_gc_slots; mod immutable; mod indexing; +mod indexing_support; mod is_array; mod iter_methods; mod iter_object; @@ -36,6 +37,8 @@ mod forwarding_tests; #[cfg(test)] mod spread_dense_tests; #[cfg(test)] +mod strict_store_tests; +#[cfg(test)] mod subclass_tests; #[cfg(test)] mod tests; @@ -119,15 +122,9 @@ pub use self::immutable::{ js_array_to_sorted_default, js_array_to_sorted_with_comparator, js_array_to_spliced, js_array_with, js_arraylike_copy_within, }; -#[cfg(test)] -pub(crate) use self::indexing::test_keys_array_slot_fallbacks; pub(crate) use self::indexing::{ array_has_own_index, array_iteration_is_exotic, array_iteration_is_exotic_resolved, - array_proto_iterator_modified, array_prototype_has_index_flag, array_spec_get, - array_spec_has_index, array_spec_set, invalidate_array_index_fast_path, - keys_array_len_capped_to_capacity, keys_array_slot, note_array_proto_iterator_write, - note_object_prototype_index_write, object_prototype_has_index_flag, - PERRY_ARRAY_INDEX_FAST_PATH_INVALIDATED, + array_prototype_has_index_flag, array_spec_get, array_spec_has_index, array_spec_set, }; pub use self::indexing::{ js_array_get_element, js_array_get_element_f64, js_array_get_f64, js_array_get_f64_unchecked, @@ -136,6 +133,14 @@ pub use self::indexing::{ js_array_set_f64_extend, js_array_set_f64_extend_strict, js_array_set_f64_unchecked, js_array_set_index_or_string, js_array_set_index_or_string_strict, js_array_set_string_key, }; +#[cfg(test)] +pub(crate) use self::indexing_support::test_keys_array_slot_fallbacks; +pub(crate) use self::indexing_support::{ + array_proto_iterator_modified, invalidate_array_index_fast_path, + keys_array_len_capped_to_capacity, keys_array_slot, note_array_proto_iterator_write, + note_object_prototype_index_write, object_prototype_has_index_flag, + PERRY_ARRAY_INDEX_FAST_PATH_INVALIDATED, +}; pub use self::is_array::js_array_is_array; pub(crate) use self::iter_methods::throw_reduce_of_empty; pub use self::iter_methods::{ @@ -172,7 +177,7 @@ pub use self::subclass::{ array_subclass_dense_snapshot, array_subclass_has_iterator_override, is_array_subclass_instance, }; #[cfg(test)] -pub(crate) use indexing::test_swap_array_index_fast_path_invalidated; +pub(crate) use indexing_support::test_swap_array_index_fast_path_invalidated; // #7574 — array-like OBJECT receiver resolution for the raw `js_array_*` entry // points, plus the Array-exotic `length` maintenance the generic OBJECT index // store needs for a `class X extends Array` receiver. diff --git a/crates/perry-runtime/src/array/push_pop.rs b/crates/perry-runtime/src/array/push_pop.rs index 2ffb8cc9e5..dd3d3eda26 100644 --- a/crates/perry-runtime/src/array/push_pop.rs +++ b/crates/perry-runtime/src/array/push_pop.rs @@ -87,10 +87,7 @@ pub(crate) fn guard_writable_length(arr: *const ArrayHeader) { /// charge of those. #[inline] unsafe fn resolved_plain_array_flags(arr: *const ArrayHeader) -> Option { - if (arr as usize) < crate::gc::GC_HEADER_SIZE + 0x1000 { - return None; - } - let gc_header = (arr as *const u8).sub(crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader; + let gc_header = super::header::array_gc_header(arr)?; ((*gc_header).obj_type == crate::gc::GC_TYPE_ARRAY).then(|| (*gc_header)._reserved) } diff --git a/crates/perry-runtime/src/array/strict_store_tests.rs b/crates/perry-runtime/src/array/strict_store_tests.rs new file mode 100644 index 0000000000..abaabb9a35 --- /dev/null +++ b/crates/perry-runtime/src/array/strict_store_tests.rs @@ -0,0 +1,67 @@ +//! The strict element-store fast lane's unit tests, split out of `tests.rs` +//! to keep it under the repo's 2000-line cap. Pure move. + +use super::*; + +/// The strict element-store fast lane (`try_strict_dense_number_store`) must +/// store exactly what the general path stores, for both the NaN-boxed +/// receiver codegen passes and a raw head, and must decline every shape it +/// cannot prove: out-of-range indices, tagged or NaN values. +#[test] +fn strict_dense_number_store_fast_lane_matches_the_general_path() { + use super::indexing::test_strict_dense_number_store as lane; + unsafe { + let mut arr = js_array_alloc(4); + for i in 0..3 { + arr = js_array_push_f64(arr, i as f64); + } + let boxed = crate::value::js_nanbox_pointer(arr as i64).to_bits() as *mut ArrayHeader; + + assert!( + lane(boxed, 1, 41.5), + "boxed receiver, plain number, in range" + ); + assert_eq!(js_array_get_f64(arr, 1), 41.5); + assert!(lane(arr, 2, -7.0), "raw receiver"); + assert_eq!(js_array_get_f64(arr, 2), -7.0); + + // An INT32 box stores its canonical double on this raw-f64 layout. + let boxed_int = f64::from_bits(crate::value::INT32_TAG | 12); + assert!(lane(boxed, 1, boxed_int), "INT32 box is a number"); + assert_eq!(js_array_get_f64(arr, 1).to_bits(), 12.0f64.to_bits()); + assert!(!lane(arr, 3, 1.0), "index == length is an extension"); + assert!( + !lane(arr, 0, f64::from_bits(crate::value::TAG_UNDEFINED)), + "tagged value" + ); + assert!( + !lane(arr, 0, f64::NAN), + "NaN keeps canonicalization on the general path" + ); + assert!(!lane(std::ptr::null_mut(), 0, 1.0), "null receiver"); + assert!( + !lane( + f64::from_bits(crate::value::TAG_UNDEFINED).to_bits() as *mut ArrayHeader, + 0, + 1.0 + ), + "non-pointer receiver" + ); + assert_eq!( + js_array_get_f64(arr, 0), + 0.0, + "declined stores leave the slot alone" + ); + assert_eq!((*arr).length, 3); + + // The public strict entry answers the same shape through the lane and + // still returns the live head. + let out = js_array_set_f64_extend_strict(boxed, 0, 9.0); + assert_eq!(out, arr); + assert_eq!(js_array_get_f64(arr, 0), 9.0); + // …and extension still goes through the general path. + let out = js_array_set_f64_extend_strict(boxed, 3, 3.0); + assert_eq!((*out).length, 4); + assert_eq!(js_array_get_f64(out, 3), 3.0); + } +} diff --git a/crates/perry-runtime/src/array/tests.rs b/crates/perry-runtime/src/array/tests.rs index a3b9512cc3..e912b60872 100644 --- a/crates/perry-runtime/src/array/tests.rs +++ b/crates/perry-runtime/src/array/tests.rs @@ -1991,66 +1991,3 @@ fn array_prototype_method_discriminator_separates_foreign_builtins() { "never obtained a GC-quiet view of Array.prototype / String.prototype" ); } - -/// The strict element-store fast lane (`try_strict_dense_number_store`) must -/// store exactly what the general path stores, for both the NaN-boxed -/// receiver codegen passes and a raw head, and must decline every shape it -/// cannot prove: out-of-range indices, tagged or NaN values. -#[test] -fn strict_dense_number_store_fast_lane_matches_the_general_path() { - use super::indexing::test_strict_dense_number_store as lane; - unsafe { - let mut arr = js_array_alloc(4); - for i in 0..3 { - arr = js_array_push_f64(arr, i as f64); - } - let boxed = crate::value::js_nanbox_pointer(arr as i64).to_bits() as *mut ArrayHeader; - - assert!( - lane(boxed, 1, 41.5), - "boxed receiver, plain number, in range" - ); - assert_eq!(js_array_get_f64(arr, 1), 41.5); - assert!(lane(arr, 2, -7.0), "raw receiver"); - assert_eq!(js_array_get_f64(arr, 2), -7.0); - - // An INT32 box stores its canonical double on this raw-f64 layout. - let boxed_int = f64::from_bits(crate::value::INT32_TAG | 12); - assert!(lane(boxed, 1, boxed_int), "INT32 box is a number"); - assert_eq!(js_array_get_f64(arr, 1).to_bits(), 12.0f64.to_bits()); - assert!(!lane(arr, 3, 1.0), "index == length is an extension"); - assert!( - !lane(arr, 0, f64::from_bits(crate::value::TAG_UNDEFINED)), - "tagged value" - ); - assert!( - !lane(arr, 0, f64::NAN), - "NaN keeps canonicalization on the general path" - ); - assert!(!lane(std::ptr::null_mut(), 0, 1.0), "null receiver"); - assert!( - !lane( - f64::from_bits(crate::value::TAG_UNDEFINED).to_bits() as *mut ArrayHeader, - 0, - 1.0 - ), - "non-pointer receiver" - ); - assert_eq!( - js_array_get_f64(arr, 0), - 0.0, - "declined stores leave the slot alone" - ); - assert_eq!((*arr).length, 3); - - // The public strict entry answers the same shape through the lane and - // still returns the live head. - let out = js_array_set_f64_extend_strict(boxed, 0, 9.0); - assert_eq!(out, arr); - assert_eq!(js_array_get_f64(arr, 0), 9.0); - // …and extension still goes through the general path. - let out = js_array_set_f64_extend_strict(boxed, 3, 3.0); - assert_eq!((*out).length, 4); - assert_eq!(js_array_get_f64(out, 3), 3.0); - } -} diff --git a/crates/perry-runtime/src/gc/barrier/mod.rs b/crates/perry-runtime/src/gc/barrier/mod.rs index 6418b2c7c3..69d864bf8c 100644 --- a/crates/perry-runtime/src/gc/barrier/mod.rs +++ b/crates/perry-runtime/src/gc/barrier/mod.rs @@ -1122,130 +1122,6 @@ pub extern "C" fn js_write_barrier(parent: u64, child: u64) { js_write_barrier_slot(parent, 0, child); } -/// Gen-GC Phase C1: slot-aware write barrier. Called by -/// codegen-emitted store sites unless `PERRY_WRITE_BARRIERS=0`/ -/// `off`/`false` disabled barrier emission at compile time. -/// -/// Decode the parent + child as raw addresses. If parent's -/// GcHeader sits in the old-gen arena AND child's NaN-boxed -/// pointer (any of POINTER / STRING / BIGINT / SHORT_STRING) -/// resolves to a heap address inside the nursery, dirty the page -/// containing the written slot. A zero slot address falls back to -/// dirtying every occupied page in the parent object. -/// -/// Hot-path constraints: this fires on EVERY heap store in -/// compiled code by default. Must be cheap: -/// generation checks use arena page side metadata rather than -/// scanning every arena block. -#[no_mangle] -pub extern "C" fn js_write_barrier_slot(parent: u64, slot_addr: u64, child: u64) { - write_barrier_slot_inner(parent, slot_addr as usize, child, false); -} - -/// [`js_write_barrier_slot`] for a parent the caller has ALREADY validated: -/// `parent_user` is the raw (untagged) user pointer of a live, non-forwarded -/// GC object whose header the emitted code dereferenced a few instructions -/// earlier (`emit_parent_may_need_remembering_check` reads `gc_flags`). -/// -/// Skips `decode_heap_addr(parent)` — a tag dispatch, an alignment/floor -/// test and a page-generation classification that `write_barrier_decoded_parent` -/// repeats one call later — and nothing else. For every parent that meets the -/// contract the two entries decide identically: a classified arena parent -/// reaches the same `barrier_parent_needs_remembering`, and an unregistered -/// (`gc_malloc`) parent, which `decode_heap_addr` would have turned into a -/// skip, is refused there instead because an inline slot never qualifies as -/// external. On a 5k-entity ECS frame the buckets' `push` stores were -/// classifying each old parent twice per command. -#[no_mangle] -pub extern "C" fn js_write_barrier_slot_validated_parent( - parent_user: u64, - slot_addr: u64, - child: u64, -) { - let Some(child_addr) = barrier_child_prologue(child) else { - return; - }; - if !barrier_remembering_active() { - return; - } - if parent_user == 0 { - bump_write_barrier_trace_counter(BarrierTraceCounter::NonPointerParentSkips); - return; - } - write_barrier_decoded_parent(parent_user as usize, slot_addr as usize, child_addr, false); -} - -pub(super) fn write_barrier_slot_inner( - parent: u64, - slot_addr: usize, - child: u64, - external_slot: bool, -) { - // Decode child first: primitive stores are the overwhelmingly common - // case (every numeric array/field store) and need NEITHER the - // incremental-mark probe (nothing to mark) NOR the remembered set (no - // old→young edge) — so they must not pay the incremental barrier's - // unconditional thread-local access, which dominated tight numeric store - // loops (#6011: `ema[i] = ` spent more time in this preamble than - // in the store itself). - let Some(child_addr) = barrier_child_prologue(child) else { - return; - }; - if !barrier_remembering_active() { - return; - } - // Decode the parent — must be a NaN-boxed heap pointer. - let parent_addr = decode_heap_addr(parent); - if parent_addr == 0 { - bump_write_barrier_trace_counter(BarrierTraceCounter::NonPointerParentSkips); - return; - } - write_barrier_decoded_parent(parent_addr, slot_addr, child_addr, external_slot); -} - -/// The never-skippable half of the barrier: decode the stored child and shade -/// it for any in-progress incremental cycle. Returns the child's heap address, -/// or `None` when the store published no heap pointer at all (every numeric -/// array/field store — the #6011 fast path, which must stay the cheapest exit). -#[inline] -fn barrier_child_prologue(child: u64) -> Option { - let child_addr = decode_heap_addr(child); - bump_write_barrier_trace_counter(BarrierTraceCounter::Calls); - if child_addr == 0 { - bump_write_barrier_trace_counter(BarrierTraceCounter::NonPointerChildSkips); - return None; - } - incremental_mark_barrier_value(child); - Some(child_addr) -} - -/// #7187: should this barrier call do remembered-set work at all? -/// -/// Placed **after** [`barrier_child_prologue`] and **before** the parent -/// decode, in every entry point. Both halves of that placement are -/// load-bearing: -/// -/// * After the prologue, so the #6011 fast path (any number stored into any -/// slot — the overwhelmingly common store) pays literally nothing new, and -/// so SATB/insertion shading for an in-progress incremental cycle is never -/// skipped. An incremental cycle implies a collection has run implies -/// armed, so this could not bite today; writing the order down keeps a -/// later refactor from hoisting the check above the shading. -/// * Before the parent decode, so the unarmed window also skips -/// `decode_heap_addr`'s raw-pointer arm — itself a -/// `classify_heap_generation` on the bare-`u64` entry point. -/// -/// Cost once armed: one relaxed load of a `static` (`adrp`/`ldr`) plus a -/// perfectly-predicted, permanently-taken branch. -#[inline] -fn barrier_remembering_active() -> bool { - if barrier_remembering_armed() { - return true; - } - bump_write_barrier_trace_counter(BarrierTraceCounter::UnarmedSkips); - false -} - /// [`write_barrier_slot_inner`] for a caller that already holds the parent as /// a plain GC user pointer — see [`write_barrier_decoded_parent`] for why the /// `u64` round-trip is worth avoiding (#7187). @@ -2043,4 +1919,6 @@ pub(super) fn remembered_dirty_page_count() -> usize { // module purely for the 2000-line file-size gate; same module tree, same // visibility semantics (the statics they read are pub(super)/pub(crate)). mod maintenance; + +pub(super) use super::barrier_store::{barrier_child_prologue, barrier_remembering_active}; pub use maintenance::*; diff --git a/crates/perry-runtime/src/gc/barrier_store.rs b/crates/perry-runtime/src/gc/barrier_store.rs index c993495452..9ae974f446 100644 --- a/crates/perry-runtime/src/gc/barrier_store.rs +++ b/crates/perry-runtime/src/gc/barrier_store.rs @@ -5,9 +5,12 @@ //! `use` lines. use super::barrier::{ - incremental_mark_barrier_value, malloc_gc_parent_addr, write_barrier_slot_decoded, + bump_write_barrier_trace_counter, decode_heap_addr, incremental_mark_barrier_value, + malloc_gc_parent_addr, write_barrier_decoded_parent, write_barrier_slot_decoded, write_barriers_enabled, }; +use super::barrier_arming::barrier_remembering_armed; +use super::telemetry::BarrierTraceCounter; use super::*; /// Loop form of [`runtime_write_barrier_slot`] for one old-gen parent and a @@ -151,3 +154,129 @@ pub(crate) fn runtime_write_barrier_gc_slot(parent_addr: usize, slot_addr: usize ) && malloc_gc_parent_addr(parent_addr); write_barrier_slot_decoded(parent_addr, slot_addr, child_bits, parent_is_malloc_gc); } + +// --- slot-form barrier entry points (moved from `barrier/mod.rs`, #2000-line cap) --- + +/// Gen-GC Phase C1: slot-aware write barrier. Called by +/// codegen-emitted store sites unless `PERRY_WRITE_BARRIERS=0`/ +/// `off`/`false` disabled barrier emission at compile time. +/// +/// Decode the parent + child as raw addresses. If parent's +/// GcHeader sits in the old-gen arena AND child's NaN-boxed +/// pointer (any of POINTER / STRING / BIGINT / SHORT_STRING) +/// resolves to a heap address inside the nursery, dirty the page +/// containing the written slot. A zero slot address falls back to +/// dirtying every occupied page in the parent object. +/// +/// Hot-path constraints: this fires on EVERY heap store in +/// compiled code by default. Must be cheap: +/// generation checks use arena page side metadata rather than +/// scanning every arena block. +#[no_mangle] +pub extern "C" fn js_write_barrier_slot(parent: u64, slot_addr: u64, child: u64) { + write_barrier_slot_inner(parent, slot_addr as usize, child, false); +} + +/// [`js_write_barrier_slot`] for a parent the caller has ALREADY validated: +/// `parent_user` is the raw (untagged) user pointer of a live, non-forwarded +/// GC object whose header the emitted code dereferenced a few instructions +/// earlier (`emit_parent_may_need_remembering_check` reads `gc_flags`). +/// +/// Skips `decode_heap_addr(parent)` — a tag dispatch, an alignment/floor +/// test and a page-generation classification that `write_barrier_decoded_parent` +/// repeats one call later — and nothing else. For every parent that meets the +/// contract the two entries decide identically: a classified arena parent +/// reaches the same `barrier_parent_needs_remembering`, and an unregistered +/// (`gc_malloc`) parent, which `decode_heap_addr` would have turned into a +/// skip, is refused there instead because an inline slot never qualifies as +/// external. On a 5k-entity ECS frame the buckets' `push` stores were +/// classifying each old parent twice per command. +#[no_mangle] +pub extern "C" fn js_write_barrier_slot_validated_parent( + parent_user: u64, + slot_addr: u64, + child: u64, +) { + let Some(child_addr) = barrier_child_prologue(child) else { + return; + }; + if !barrier_remembering_active() { + return; + } + if parent_user == 0 { + bump_write_barrier_trace_counter(BarrierTraceCounter::NonPointerParentSkips); + return; + } + write_barrier_decoded_parent(parent_user as usize, slot_addr as usize, child_addr, false); +} + +pub(super) fn write_barrier_slot_inner( + parent: u64, + slot_addr: usize, + child: u64, + external_slot: bool, +) { + // Decode child first: primitive stores are the overwhelmingly common + // case (every numeric array/field store) and need NEITHER the + // incremental-mark probe (nothing to mark) NOR the remembered set (no + // old→young edge) — so they must not pay the incremental barrier's + // unconditional thread-local access, which dominated tight numeric store + // loops (#6011: `ema[i] = ` spent more time in this preamble than + // in the store itself). + let Some(child_addr) = barrier_child_prologue(child) else { + return; + }; + if !barrier_remembering_active() { + return; + } + // Decode the parent — must be a NaN-boxed heap pointer. + let parent_addr = decode_heap_addr(parent); + if parent_addr == 0 { + bump_write_barrier_trace_counter(BarrierTraceCounter::NonPointerParentSkips); + return; + } + write_barrier_decoded_parent(parent_addr, slot_addr, child_addr, external_slot); +} + +/// The never-skippable half of the barrier: decode the stored child and shade +/// it for any in-progress incremental cycle. Returns the child's heap address, +/// or `None` when the store published no heap pointer at all (every numeric +/// array/field store — the #6011 fast path, which must stay the cheapest exit). +#[inline] +pub(super) fn barrier_child_prologue(child: u64) -> Option { + let child_addr = decode_heap_addr(child); + bump_write_barrier_trace_counter(BarrierTraceCounter::Calls); + if child_addr == 0 { + bump_write_barrier_trace_counter(BarrierTraceCounter::NonPointerChildSkips); + return None; + } + incremental_mark_barrier_value(child); + Some(child_addr) +} + +/// #7187: should this barrier call do remembered-set work at all? +/// +/// Placed **after** [`barrier_child_prologue`] and **before** the parent +/// decode, in every entry point. Both halves of that placement are +/// load-bearing: +/// +/// * After the prologue, so the #6011 fast path (any number stored into any +/// slot — the overwhelmingly common store) pays literally nothing new, and +/// so SATB/insertion shading for an in-progress incremental cycle is never +/// skipped. An incremental cycle implies a collection has run implies +/// armed, so this could not bite today; writing the order down keeps a +/// later refactor from hoisting the check above the shading. +/// * Before the parent decode, so the unarmed window also skips +/// `decode_heap_addr`'s raw-pointer arm — itself a +/// `classify_heap_generation` on the bare-`u64` entry point. +/// +/// Cost once armed: one relaxed load of a `static` (`adrp`/`ldr`) plus a +/// perfectly-predicted, permanently-taken branch. +#[inline] +pub(super) fn barrier_remembering_active() -> bool { + if barrier_remembering_armed() { + return true; + } + bump_write_barrier_trace_counter(BarrierTraceCounter::UnarmedSkips); + false +} diff --git a/crates/perry-runtime/src/gc/layout_tables.rs b/crates/perry-runtime/src/gc/layout_tables.rs index 1c8e25e021..e5d71a1c03 100644 --- a/crates/perry-runtime/src/gc/layout_tables.rs +++ b/crates/perry-runtime/src/gc/layout_tables.rs @@ -730,7 +730,10 @@ pub(in crate::gc) fn typed_layouts_insert(user_ptr: usize, descriptor: TypedLayo pub(in crate::gc) fn slot_masks_insert(user_ptr: usize, mask: LayoutSlotMask) { mark_per_object_layouts_nonempty(); layout_addr_filter_add(user_ptr); - let fresh = hot_layout_slot_masks().borrow_mut().insert(user_ptr, mask).is_none(); + let fresh = hot_layout_slot_masks() + .borrow_mut() + .insert(user_ptr, mask) + .is_none(); if fresh { note_new_layout_record(user_ptr); } diff --git a/crates/perry-runtime/src/gc/tests/barrier_decoded_parent.rs b/crates/perry-runtime/src/gc/tests/barrier_decoded_parent.rs index 573521a039..ada657d3e6 100644 --- a/crates/perry-runtime/src/gc/tests/barrier_decoded_parent.rs +++ b/crates/perry-runtime/src/gc/tests/barrier_decoded_parent.rs @@ -80,7 +80,7 @@ fn validated_parent_entry_matches_js_write_barrier_slot() { let dirty_page = crate::arena::generation_page_for_addr(fields as usize); assert!(!old_page_dirty_for(dirty_page)); - crate::gc::barrier::js_write_barrier_slot_validated_parent( + crate::gc::barrier_store::js_write_barrier_slot_validated_parent( old_obj as u64, fields as u64, child_bits, @@ -96,7 +96,7 @@ fn validated_parent_entry_matches_js_write_barrier_slot() { // A young parent is not remembered by either entry. reset_remembered_set(); let young_parent = crate::arena::arena_alloc_gc(40, 8, GC_TYPE_OBJECT) as usize; - crate::gc::barrier::js_write_barrier_slot_validated_parent( + crate::gc::barrier_store::js_write_barrier_slot_validated_parent( young_parent as u64, (young_parent + 8) as u64, child_bits, From f835ef3f51a33009199f8b907df1f4a0fe269749 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 17:44:49 +0200 Subject: [PATCH 22/23] changelog: #8885 ECS command-path follow-up Claude-Session: https://claude.ai/code/session_01FUvFrRNZyc5qknBiJbYbby --- .../8885-ecs-followup-gc-bookkeeping.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 changelog.d/8885-ecs-followup-gc-bookkeeping.md diff --git a/changelog.d/8885-ecs-followup-gc-bookkeeping.md b/changelog.d/8885-ecs-followup-gc-bookkeeping.md new file mode 100644 index 0000000000..c11ccbb24f --- /dev/null +++ b/changelog.d/8885-ecs-followup-gc-bookkeeping.md @@ -0,0 +1,18 @@ +Removed most of the remaining GC-bookkeeping and dispatch overhead on the ECS +command path (follow-up to #8872). General compiler/runtime mechanisms: +branded `number & {…}` intersections and generic/imported type aliases now +lower to their primitive (they were `Any`, so every entity id was dynamic); +inline plain-double fast paths for dynamic compares and truthiness; +beta-reduction of called arrow-literal locals left behind by inlining; +`Map.get` heals stale array-growth forwarding stubs in place; a header-bit +fast lane in `clean_arr_ptr` and every strict array helper; a process-global +address sketch, an object/closure per-object-mask threshold of eight slots, a +death prune for the per-object layout tables and a live-young-record count +that together let the inline allocator skip the stale-layout probe; the +strict store lane, `pop` and `length =` resolve the header once; the +dirty-page and `typeof` caches move to hot TLS; the string-demote tag test +and plain closure-capture reads are inlined; a validated-parent write-barrier +entry; and `length = 0` keeps an all-pointer array all-pointer. On the +upstream `codehz/ecs` "5k entities: 3 commands each + sync" row the compiled +benchmark went from 7.30 ms/op to ~4.7 ms/op (−36%, paired runs on an idle +Mac mini; Node 26.5.1 is 1.76 ms/op). From f1177e38b4b9bef5d5241475f43a1fda804b500e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 27 Aug 2026 17:49:25 +0200 Subject: [PATCH 23/23] chore: drop the stale cold thread-local entries for arithmetic.rs and dirty_page_cache.rs Both moved their remaining thread_local! to perry_thread_local! in this branch, so the policy ratchet's allowlist recorded blocks that no longer exist ("a stale entry is one nobody has to justify"). Claude-Session: https://claude.ai/code/session_01FUvFrRNZyc5qknBiJbYbby --- scripts/thread_local_cold_allowlist.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/thread_local_cold_allowlist.json b/scripts/thread_local_cold_allowlist.json index 89f27d4433..9cc23ed1a8 100644 --- a/scripts/thread_local_cold_allowlist.json +++ b/scripts/thread_local_cold_allowlist.json @@ -1,13 +1,12 @@ { "_comment": "Files still declaring raw `thread_local!`. Every entry is a declaration that pays `_tlv_get_addr` on Darwin; the count is a ratchet, so adding one to an already-listed file fails too. New code should use `crate::perry_thread_local!` \u2014 see crates/perry-runtime/src/tls_hot.rs. Regenerate with scripts/check_thread_locals.py --update.", - "_hot_declarations": 261, + "_hot_declarations": 269, "files": { "crates/perry-runtime/src/agent.rs": 1, "crates/perry-runtime/src/arena/block.rs": 2, "crates/perry-runtime/src/arena/page_meta.rs": 2, "crates/perry-runtime/src/async_context.rs": 2, "crates/perry-runtime/src/async_hooks.rs": 3, - "crates/perry-runtime/src/builtins/arithmetic.rs": 1, "crates/perry-runtime/src/builtins/console.rs": 2, "crates/perry-runtime/src/builtins/formatting.rs": 5, "crates/perry-runtime/src/builtins/formatting/boxed_primitives.rs": 1, @@ -31,7 +30,6 @@ "crates/perry-runtime/src/gc/barrier/mod.rs": 2, "crates/perry-runtime/src/gc/barrier_arming.rs": 1, "crates/perry-runtime/src/gc/cycle.rs": 1, - "crates/perry-runtime/src/gc/dirty_page_cache.rs": 1, "crates/perry-runtime/src/gc/fromspace_scan.rs": 1, "crates/perry-runtime/src/gc/layout.rs": 1, "crates/perry-runtime/src/gc/layout_tables.rs": 1,