From 7e61cdf300ca5f35ff2115cf2be7e92fcff9178d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 29 Aug 2026 08:56:27 +0200 Subject: [PATCH 1/2] codegen: admit source-small guarded specializations to the pre-statepoint inliner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `GUARDED_SPECIALIZATION_PREINLINE_MAX_IR_BYTES` judges lowered IR rather than statement count, and its comment says why: one source statement can lower to a large property/index dispatch lattice, so statement count is a bad admission test. That is right, and it makes the 16 KiB ceiling reject the very methods worth flattening — the leaves that ARE one such statement. In wolf-ecs `SparseSet.add`, `ECS._hasComponent` and `ECS._archChange` are one statement each and lower to tens of KiB of guard lattice, so every call from `addComponent` stayed a native call boundary and nothing downstream could see through it. Statement count is a poor admission test but a sound BOUND: it limits how much source a caller can absorb. So the ceiling gains a second arm — a body within `inline_hot_small_size_cap()` statements is admitted up to 64 KiB — and the original 16 KiB arm is unchanged for everything else. This is deliberately the shape that cannot reach #8583's failure mode. The bundled IIFEs whose `rewrite-statepoints-for-gc` fan-out made `-Os` never finish are thousands of statements; they can never take the new arm however their IR measures. `PERRY_GUARDED_PREINLINE_MAX_IR_BYTES` overrides the raised ceiling for A/B without a rebuild. wolf-ecs, Mac mini, 11 alternating pairs against main `3291b6425d`: add_remove −1.18%/−1.23%, entity_cycle −1.60%/−1.60% (2 s / 50 ms windows; 11/11 except entity 50 ms at 10/11). A global 64 KiB bump measures the same, so the win comes entirely from the source-small leaves. Claude-Session: https://claude.ai/code/session_019WVcWKmYsUBnnFB7nBgbBJ --- .../8996-guarded-preinline-source-small.md | 3 ++ crates/perry-codegen/src/codegen/helpers.rs | 44 +++++++++++++++++++ crates/perry-codegen/src/codegen/method.rs | 3 +- .../src/codegen/method_trampolines.rs | 5 ++- 4 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 changelog.d/8996-guarded-preinline-source-small.md diff --git a/changelog.d/8996-guarded-preinline-source-small.md b/changelog.d/8996-guarded-preinline-source-small.md new file mode 100644 index 0000000000..a8f63d9fed --- /dev/null +++ b/changelog.d/8996-guarded-preinline-source-small.md @@ -0,0 +1,3 @@ +### Changed + +- A guarded method specialization that is small at the source level but lowers to a large dispatch lattice is now admitted to the pre-statepoint inliner (up to 64 KiB of IR instead of 16 KiB), so one-statement leaves such as a sparse-set `add` flatten into their callers instead of staying a native call boundary. diff --git a/crates/perry-codegen/src/codegen/helpers.rs b/crates/perry-codegen/src/codegen/helpers.rs index 26370c3425..660e2670ab 100644 --- a/crates/perry-codegen/src/codegen/helpers.rs +++ b/crates/perry-codegen/src/codegen/helpers.rs @@ -367,11 +367,55 @@ pub(super) fn apply_pshape_inline_policy( /// such as mutation-heavy ECS transitions by nearly an order of magnitude. pub(super) const GUARDED_SPECIALIZATION_PREINLINE_MAX_IR_BYTES: usize = 16 * 1024; +/// Raised ceiling for a body that is **small at the source level**. +/// +/// The constant above deliberately judges lowered IR rather than statement +/// count, because one source statement can lower to a large property/index +/// dispatch lattice. That is the right *admission* test and the wrong *bound*: +/// it also rejects the leaf methods that consist of a single such statement, +/// which are exactly the ones worth flattening into their callers. wolf-ecs is +/// the case in point — `SparseSet.add`, `ECS._hasComponent` and +/// `ECS._archChange` are one statement each and lower to tens of KiB of guard +/// lattice, so every call from `addComponent` stayed a native call boundary. +/// +/// Statement count is a sound bound on how much *source* a caller can absorb, +/// and it is what keeps this away from #8583's failure mode: the giant bundled +/// IIFEs whose `rewrite-statepoints-for-gc` fan-out made `-Os` never finish are +/// thousands of statements, so they can never reach this arm however their IR +/// measures. Overridable via `PERRY_GUARDED_PREINLINE_MAX_IR_BYTES` (the raised +/// ceiling) for A/B without a rebuild. +pub(super) fn guarded_specialization_source_small_max_ir_bytes() -> usize { + use std::sync::OnceLock; + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| { + std::env::var("PERRY_GUARDED_PREINLINE_MAX_IR_BYTES") + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(64 * 1024) + }) +} + +/// Statement ceiling for the raised budget. Shares +/// [`inline_hot_small_size_cap`]'s value: the same "this is a leaf, not a +/// subsystem" judgement, measured the same way. +#[inline] +pub(super) fn guarded_specialization_source_small(statements: usize) -> bool { + statements <= inline_hot_small_size_cap() +} + #[inline] pub(super) fn guarded_specialization_fits_preinline_budget(ir_bytes: usize) -> bool { ir_bytes <= GUARDED_SPECIALIZATION_PREINLINE_MAX_IR_BYTES } +/// [`guarded_specialization_fits_preinline_budget`] plus the source-small arm. +#[inline] +pub(super) fn guarded_specialization_admits_preinline(ir_bytes: usize, statements: usize) -> bool { + guarded_specialization_fits_preinline_budget(ir_bytes) + || (guarded_specialization_source_small(statements) + && ir_bytes <= guarded_specialization_source_small_max_ir_bytes()) +} + /// Maximum total (module-wide) direct call sites a function may have and still /// be hinted. This is the anti-bloat backstop: the raised `-inlinehint-threshold` /// lifts LLVM's ceiling for a hinted callee at *every* one of its call sites, so diff --git a/crates/perry-codegen/src/codegen/method.rs b/crates/perry-codegen/src/codegen/method.rs index a2c538d7af..e72d90c57c 100644 --- a/crates/perry-codegen/src/codegen/method.rs +++ b/crates/perry-codegen/src/codegen/method.rs @@ -1287,8 +1287,9 @@ pub(super) fn compile_method( let lowered = llmod .function_mut(lowered_function_index) .expect("just-lowered method function"); - if super::helpers::guarded_specialization_fits_preinline_budget( + if super::helpers::guarded_specialization_admits_preinline( lowered.estimated_ir_bytes(), + method.body.len(), ) { lowered.pre_statepoint_inline = true; } diff --git a/crates/perry-codegen/src/codegen/method_trampolines.rs b/crates/perry-codegen/src/codegen/method_trampolines.rs index 6af0003e3c..339d877d2c 100644 --- a/crates/perry-codegen/src/codegen/method_trampolines.rs +++ b/crates/perry-codegen/src/codegen/method_trampolines.rs @@ -225,9 +225,12 @@ pub(super) fn emit_guarded_nonnegative_index( // may flatten before statepoint rewriting. This makes tiny indexed leaves // disappear at their direct call sites while keeping large mutation bodies // behind one native call boundary. + let statements = method.body.len(); let preinline = llmod .function_estimated_ir_bytes(&clone_name) - .is_some_and(super::helpers::guarded_specialization_fits_preinline_budget); + .is_some_and(|ir_bytes| { + super::helpers::guarded_specialization_admits_preinline(ir_bytes, statements) + }); let target_triple = llmod.target_triple.clone(); let mut params: Vec<(LlvmType, String)> = Vec::with_capacity(method.params.len() + 1); params.push((DOUBLE, "%this_arg".to_string())); From 53f9ba822fc44101aabcf7e9e903288985cb297e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 29 Aug 2026 09:56:46 +0200 Subject: [PATCH 2/2] test(codegen): pin the source-small preinline admission, and fix the fragment number The new arm had no test. Added one over `guarded_specialization_admits_preinline` that pins both arms and, most importantly, the #8583 protection: a body past the statement cap is rejected at ANY IR size, which is the whole reason the statement count is a bound rather than an admission test. Written against the functions' own values, so a retuned default cannot make it vacuous. Sabotage-checked -- dropping the statement bound fails it. The changeset fragment was numbered 8996, which is an unrelated merged PR (`perf/elements-inline-push`). Renumbered to 9016; a wrong number is invisible until a release is cut and then misattributes the change (#8978). --- ...=> 9016-guarded-preinline-source-small.md} | 0 crates/perry-codegen/src/codegen/helpers.rs | 44 +++++++++++++++++++ 2 files changed, 44 insertions(+) rename changelog.d/{8996-guarded-preinline-source-small.md => 9016-guarded-preinline-source-small.md} (100%) diff --git a/changelog.d/8996-guarded-preinline-source-small.md b/changelog.d/9016-guarded-preinline-source-small.md similarity index 100% rename from changelog.d/8996-guarded-preinline-source-small.md rename to changelog.d/9016-guarded-preinline-source-small.md diff --git a/crates/perry-codegen/src/codegen/helpers.rs b/crates/perry-codegen/src/codegen/helpers.rs index 660e2670ab..c2fb659a7a 100644 --- a/crates/perry-codegen/src/codegen/helpers.rs +++ b/crates/perry-codegen/src/codegen/helpers.rs @@ -416,6 +416,50 @@ pub(super) fn guarded_specialization_admits_preinline(ir_bytes: usize, statement && ir_bytes <= guarded_specialization_source_small_max_ir_bytes()) } +#[cfg(test)] +mod guarded_preinline_admission_tests { + use super::*; + + /// Written against the functions' own values rather than literals, so a + /// retuned default cannot silently turn these into vacuous assertions. + #[test] + fn source_small_arm_admits_a_large_lattice_but_a_statement_bound_still_bounds_it() { + let raised = guarded_specialization_source_small_max_ir_bytes(); + let cap = inline_hot_small_size_cap(); + assert!( + raised > GUARDED_SPECIALIZATION_PREINLINE_MAX_IR_BYTES, + "the new arm only means something if its ceiling is higher than the original's", + ); + + // The case this change exists for: one-statement leaves whose guard + // lattice lowers well past the original 16 KiB ceiling. + let past_original = GUARDED_SPECIALIZATION_PREINLINE_MAX_IR_BYTES + 1; + assert!(!guarded_specialization_fits_preinline_budget(past_original)); + assert!(guarded_specialization_admits_preinline(past_original, 1)); + assert!(guarded_specialization_admits_preinline(raised, cap)); + + // #8583's protection, and the reason the statement count is a BOUND + // rather than an admission test: the giant bundled IIFEs are thousands + // of statements, so no IR size may let them through this arm. + assert!(!guarded_specialization_admits_preinline(raised, cap + 1)); + assert!(!guarded_specialization_admits_preinline( + past_original, + 5_000 + )); + + // The raised ceiling is still a ceiling. + assert!(!guarded_specialization_admits_preinline(raised + 1, 1)); + + // The original arm is unchanged: within 16 KiB, statement count is + // irrelevant, exactly as before this change. + assert!(guarded_specialization_admits_preinline( + GUARDED_SPECIALIZATION_PREINLINE_MAX_IR_BYTES, + 5_000, + )); + assert!(guarded_specialization_admits_preinline(0, usize::MAX)); + } +} + /// Maximum total (module-wide) direct call sites a function may have and still /// be hinted. This is the anti-bloat backstop: the raised `-inlinehint-threshold` /// lifts LLVM's ceiling for a hinted callee at *every* one of its call sites, so