From 6e73919c812f3baa62ddbbe5e1f4c2f1402db633 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 8 Aug 2026 04:55:58 +0200 Subject: [PATCH 1/5] perf(gc): born-tenured allocation for pretenure-accumulator push values (#7598) The dominant remaining cost on promote-heavy loops after #7594/#7596 is structural: every long-lived object is copied twice, Eden->survivor by the first copying minor and survivor->old by the next (measured on json_pipeline at 500k: 3.9 s of the 5.1 s build_out, 268 MB copied twice). Static pretenuring for the shape that causes it: an accumulator local admitted by the all-pointer terms (one binding, never rebound, only fresh-allocation pushes, no captures/boxes/globals) that is declared OUTSIDE every loop and pushed into only INSIDE one. Its cohort is live for the remainder of the loop by construction, so the pushed object is born in old-gen with GC_FLAG_TENURED via the existing born-tenured birth path (whose Old => TENURED obligation is contract-tested by every_old_gen_birth_path_sets_tenured since #7602). Two consumers of one mem::take'n flag (the #7590 take discipline: it reaches exactly the root allocation, nested literals read false): - lower_object_literal's shaped fast path (plain literals), and - lower_call/new.rs's outlined-call arm (the AnonShape form object literals actually reach codegen in) -- a pretenured site takes the outlined born-tenured call instead of the inline Eden bump; the ~140-cycle call is noise against the double copy it removes. Correctness is inherited, not asserted: constructor/field stores funnel through runtime_store_jsvalue_slot and the #7602-gated barrier, both of which read the LIVE parent header, so old->young field edges are remembered exactly as for a promoted object. The loop-position discriminator refuses the per-iteration accumulator (`for { const keep=[]; keep.push(..) }`) whose cohort dies every iteration -- pretenuring it would flood old-gen at allocation rate. A function-local depth-0 accumulator dropped at return IS still admitted; that adversarial case is measured in the PR alongside the win. --- crates/perry-codegen/src/codegen/closure.rs | 1 + crates/perry-codegen/src/codegen/function.rs | 1 + crates/perry-codegen/src/codegen/method.rs | 2 + .../src/collectors/all_pointer_arrays.rs | 237 ++++++++++++++++++ .../perry-codegen/src/collectors/hir_facts.rs | 18 ++ crates/perry-codegen/src/expr/array_push.rs | 20 ++ crates/perry-codegen/src/expr/mod.rs | 9 + .../perry-codegen/src/expr/object_literal.rs | 12 +- crates/perry-codegen/src/lower_call/new.rs | 23 +- .../src/runtime_decls/objects.rs | 6 + crates/perry-runtime/src/object/alloc.rs | 108 +++++++- 11 files changed, 427 insertions(+), 10 deletions(-) diff --git a/crates/perry-codegen/src/codegen/closure.rs b/crates/perry-codegen/src/codegen/closure.rs index a8c9d721eb..dbc5edd166 100644 --- a/crates/perry-codegen/src/codegen/closure.rs +++ b/crates/perry-codegen/src/codegen/closure.rs @@ -892,6 +892,7 @@ pub(super) fn compile_closure( current_block: 0, discard_expr_value: false, discard_this_expr: false, + pretenure_next_object_literal: false, func_names, strings, loop_targets: Vec::new(), diff --git a/crates/perry-codegen/src/codegen/function.rs b/crates/perry-codegen/src/codegen/function.rs index 946fb7286a..cc838adb2f 100644 --- a/crates/perry-codegen/src/codegen/function.rs +++ b/crates/perry-codegen/src/codegen/function.rs @@ -695,6 +695,7 @@ pub(super) fn compile_function( current_block: 0, discard_expr_value: false, discard_this_expr: false, + pretenure_next_object_literal: false, func_names, strings, loop_targets: Vec::new(), diff --git a/crates/perry-codegen/src/codegen/method.rs b/crates/perry-codegen/src/codegen/method.rs index e109ab52d0..340eaa468b 100644 --- a/crates/perry-codegen/src/codegen/method.rs +++ b/crates/perry-codegen/src/codegen/method.rs @@ -430,6 +430,7 @@ pub(super) fn compile_method( current_block: 0, discard_expr_value: false, discard_this_expr: false, + pretenure_next_object_literal: false, func_names, strings, loop_targets: Vec::new(), @@ -1487,6 +1488,7 @@ pub(super) fn compile_static_method( current_block: 0, discard_expr_value: false, discard_this_expr: false, + pretenure_next_object_literal: false, func_names, strings, loop_targets: Vec::new(), diff --git a/crates/perry-codegen/src/collectors/all_pointer_arrays.rs b/crates/perry-codegen/src/collectors/all_pointer_arrays.rs index 46ea09de50..d669190c07 100644 --- a/crates/perry-codegen/src/collectors/all_pointer_arrays.rs +++ b/crates/perry-codegen/src/collectors/all_pointer_arrays.rs @@ -194,6 +194,159 @@ pub(crate) fn collect_all_pointer_array_locals( /// distinct, so seeing it here is harmless — the kill walk above descends into /// closures too, and every use of such an id is inside the closure body it is /// scoped to. +/// #7598 — the subset of [`collect_all_pointer_array_locals`]-admitted locals +/// whose pushed objects should be born TENURED in old-gen. +/// +/// The all-pointer terms already prove the accumulator *shape* (one binding, +/// never rebound, every store a push of a fresh allocation, no captures / +/// boxes / globals). What they do not prove is *cohort lifetime*, and that is +/// the entire pretenuring bet: an object born old that dies young sits in +/// old-gen until a full reclaim. The discriminator is loop position: +/// +/// - the `let` must sit at **loop depth 0** — a per-iteration accumulator +/// (`for (…) { const keep = []; … keep.push(x) … }`) dies every iteration, +/// and pretenuring it floods old-gen with garbage at allocation rate; +/// - every push must sit at **depth ≥ 1** — the cohort accumulates across +/// iterations, so it is live for the remainder of the loop by construction. +/// +/// This is deliberately NOT a proof the array outlives the function; a +/// depth-0 accumulator that is dropped at function exit still pretenures, and +/// its cohort is then reclaimed by the proportional-band full cycles (#7596) +/// instead of dying free in the nursery. That trade is measured, not assumed — +/// see the adversarial arm in the PR. +pub(crate) fn collect_pretenure_accumulator_locals( + stmts: &[Stmt], + all_pointer_admitted: &HashSet, +) -> HashSet { + if all_pointer_admitted.is_empty() { + return HashSet::new(); + } + let mut decl_depth: HashMap = HashMap::new(); + let mut push_depths: HashMap> = HashMap::new(); + scan_depths(stmts, 0, &mut decl_depth, &mut push_depths); + all_pointer_admitted + .iter() + .copied() + .filter(|id| { + decl_depth.get(id) == Some(&0) + && push_depths + .get(id) + .is_some_and(|ds| !ds.is_empty() && ds.iter().all(|&d| d >= 1)) + }) + .collect() +} + +/// Depth-attributed scan: bindings and pushes recorded with their enclosing +/// real-loop count. Expressions directly attached to a statement (conditions, +/// initializers, the statement expression itself) are scanned deeply at that +/// statement's depth — a push nested inside a larger expression still counts, +/// at the depth of the statement carrying it. `for_each_expr` descends into +/// closure bodies too; a push on the same id from inside a closure records the +/// enclosing statement's depth, which is harmless — a captured id was already +/// refused by the all-pointer capture kill. +fn scan_depths( + stmts: &[Stmt], + depth: u32, + decl_depth: &mut HashMap, + push_depths: &mut HashMap>, +) { + let mut record = |expr: &Expr, at: u32, push_depths: &mut HashMap>| { + super::scalar_method_dispatch::for_each_expr(expr, &mut |e| { + if let Expr::ArrayPush { array_id, .. } = e { + push_depths.entry(*array_id).or_default().push(at); + } + }); + }; + for s in stmts { + match s { + Stmt::Let { id, init, .. } => { + if let Some(Expr::Array(_)) = init { + // First binding wins; a rebind was refused upstream. + decl_depth.entry(*id).or_insert(depth); + } + if let Some(init) = init { + record(init, depth, push_depths); + } + } + Stmt::Expr(expr) | Stmt::Throw(expr) | Stmt::Return(Some(expr)) => { + record(expr, depth, push_depths); + } + Stmt::If { + condition, + then_branch, + else_branch, + } => { + record(condition, depth, push_depths); + scan_depths(then_branch, depth, decl_depth, push_depths); + if let Some(eb) = else_branch { + scan_depths(eb, depth, decl_depth, push_depths); + } + } + Stmt::While { condition, body } | Stmt::DoWhile { condition, body } => { + record(condition, depth + 1, push_depths); + scan_depths(body, depth + 1, decl_depth, push_depths); + } + Stmt::For { + init, + condition, + update, + body, + } => { + if let Some(init) = init { + // The initializer runs once, outside the iteration. + scan_depths( + std::slice::from_ref(init.as_ref()), + depth, + decl_depth, + push_depths, + ); + } + if let Some(condition) = condition { + record(condition, depth + 1, push_depths); + } + if let Some(update) = update { + record(update, depth + 1, push_depths); + } + scan_depths(body, depth + 1, decl_depth, push_depths); + } + Stmt::Try { + body, + catch, + finally, + } => { + scan_depths(body, depth, decl_depth, push_depths); + if let Some(c) = catch { + scan_depths(&c.body, depth, decl_depth, push_depths); + } + if let Some(fin) = finally { + scan_depths(fin, depth, decl_depth, push_depths); + } + } + Stmt::Switch { + discriminant, + cases, + } => { + record(discriminant, depth, push_depths); + for c in cases { + if let Some(test) = &c.test { + record(test, depth, push_depths); + } + scan_depths(&c.body, depth, decl_depth, push_depths); + } + } + Stmt::Labeled { body, .. } => { + scan_depths( + std::slice::from_ref(body.as_ref()), + depth, + decl_depth, + push_depths, + ); + } + _ => {} + } + } +} + fn walk_stmts<'a>(stmts: &'a [Stmt], f: &mut impl FnMut(&'a Stmt)) { for s in stmts { f(s); @@ -445,4 +598,88 @@ mod tests { ]; assert!(!collect(&stmts).contains(&1)); } + + // ---- #7598 pretenure-accumulator loop-position terms ------------------- + + fn while_loop(body: Vec) -> Stmt { + Stmt::While { + condition: Expr::Bool(true), + body, + } + } + + fn collect_pretenure(stmts: &[Stmt]) -> HashSet { + let admitted = collect(stmts); + collect_pretenure_accumulator_locals(stmts, &admitted) + } + + /// The target shape: `const out = []` outside every loop, pushes inside. + #[test] + fn pretenure_admits_the_outer_accumulator_inner_push_shape() { + let stmts = vec![ + let_array(1, vec![]), + while_loop(vec![push(1, object_literal())]), + ]; + assert!(collect_pretenure(&stmts).contains(&1)); + } + + /// The per-iteration accumulator dies every iteration; pretenuring it + /// would flood old-gen with garbage at allocation rate. This is + /// push_bench's exact shape and MUST stay refused. + #[test] + fn pretenure_refuses_a_loop_local_accumulator() { + let stmts = vec![while_loop(vec![ + let_array(1, vec![]), + push(1, object_literal()), + ])]; + assert!(!collect_pretenure(&stmts).contains(&1)); + } + + /// A one-shot push outside any loop has no cohort to speak of. + #[test] + fn pretenure_refuses_pushes_outside_loops() { + let stmts = vec![let_array(1, vec![]), push(1, object_literal())]; + assert!(!collect_pretenure(&stmts).contains(&1)); + } + + /// One depth-0 push alongside loop pushes: refused — every push must be + /// inside a loop for the cohort claim to hold. + #[test] + fn pretenure_refuses_mixed_depth_pushes() { + let stmts = vec![ + let_array(1, vec![]), + push(1, object_literal()), + while_loop(vec![push(1, object_literal())]), + ]; + assert!(!collect_pretenure(&stmts).contains(&1)); + } + + /// A push nested inside a larger expression still counts, at the depth of + /// the statement carrying it — the depth scan is not statement-position + /// only. + #[test] + fn pretenure_sees_a_push_nested_in_an_expression() { + let stmts = vec![ + let_array(1, vec![]), + while_loop(vec![Stmt::Expr(Expr::BooleanCoerce(Box::new( + Expr::ArrayPush { + array_id: 1, + value: Box::new(object_literal()), + }, + )))]), + ]; + assert!(collect_pretenure(&stmts).contains(&1)); + } + + /// The all-pointer terms remain a prerequisite: a local they refuse + /// (rebind) is never pretenured, whatever its loop position. + #[test] + fn pretenure_requires_all_pointer_admission() { + let stmts = vec![ + let_array(1, vec![]), + Stmt::Expr(Expr::LocalSet(1, Box::new(Expr::Array(vec![])))), + while_loop(vec![push(1, object_literal())]), + ]; + assert!(!collect_pretenure(&stmts).contains(&1)); + } } diff --git a/crates/perry-codegen/src/collectors/hir_facts.rs b/crates/perry-codegen/src/collectors/hir_facts.rs index 16d51b46be..cff523f16a 100644 --- a/crates/perry-codegen/src/collectors/hir_facts.rs +++ b/crates/perry-codegen/src/collectors/hir_facts.rs @@ -97,6 +97,12 @@ pub(crate) struct ArrayFacts { /// why this fact governs *profitability* rather than the soundness of the /// elided per-store note (which the emitted header test owns). pub all_pointer_element_locals: HashSet, + /// #7598: the subset of `all_pointer_element_locals` whose pushed object + /// literals should be born TENURED in old-gen — an accumulator declared + /// outside every loop, filled only from inside one, so its cohort is live + /// for the remainder of the loop by construction. See + /// `collect_pretenure_accumulator_locals` for the loop-position terms. + pub pretenure_accumulator_locals: HashSet, } #[derive(Debug, Clone, Default)] @@ -261,6 +267,11 @@ impl TypeFacts { self.arrays.all_pointer_element_locals.contains(&local_id) } + /// #7598: object literals pushed into this local should be born tenured. + pub(crate) fn pretenure_accumulator(&self, local_id: u32) -> bool { + self.arrays.pretenure_accumulator_locals.contains(&local_id) + } + pub(crate) fn array_length_mutation_locals(&self) -> &HashSet { &self.effect.array_length_mutation_locals } @@ -486,6 +497,12 @@ pub(crate) fn collect_type_facts( boxed_vars, module_globals, ); + // #7598: the loop-position subset whose pushed literals are born tenured. + array_facts.pretenure_accumulator_locals = + super::all_pointer_arrays::collect_pretenure_accumulator_locals( + stmts, + &array_facts.all_pointer_element_locals, + ); let index_used_locals = super::index_uses::collect_index_used_locals(stmts); // Repsel Phase 1: under `PERRY_CANONICAL_I32_LOCALS` (default on), a // proven in-window const int-typed-array element load counts as a STRICT @@ -1406,6 +1423,7 @@ impl ArrayFactCollector { // Filled in by `collect_type_facts` — its own walk, with its // own admission terms, over the same statements. all_pointer_element_locals: HashSet::new(), + pretenure_accumulator_locals: HashSet::new(), }, EffectFacts { unknown_call_escape: self.unknown_call_escape, diff --git a/crates/perry-codegen/src/expr/array_push.rs b/crates/perry-codegen/src/expr/array_push.rs index be5eafe3c5..eb22d33b0b 100644 --- a/crates/perry-codegen/src/expr/array_push.rs +++ b/crates/perry-codegen/src/expr/array_push.rs @@ -126,8 +126,28 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr, value_discarded: bool) -> let value_is_numeric = is_numeric_expr(ctx, value); let require_numeric_layout = value_is_numeric && expr_has_numeric_pointer_free_array_layout(ctx, &array_expr); + // #7598: an accumulator declared outside every loop and filled + // inside one keeps its cohort live for the rest of the loop, so + // the pushed value is born tenured — skipping the + // Eden→survivor→old double copy the copying minor would otherwise + // pay for it. `Expr::New` is the shape object literals actually + // reach codegen in (the AnonShape transform synthesizes a class); + // `Expr::Object` covers literals the transform leaves alone. The + // flag is taken by the value's own allocation lowering + // (`lower_object_literal` / `lower_call/new.rs`), so it reaches + // exactly the root allocation and never a nested one. + if matches!(value.as_ref(), Expr::Object(_) | Expr::New { .. }) + && ctx.native_facts.pretenure_accumulator(*array_id) + { + ctx.pretenure_next_object_literal = true; + } let (v, v_bits) = lower_array_push_value(ctx, value, layout_note_needed, write_barrier_needed)?; + // Consumed by whichever allocation tier the value took. Cleared + // unconditionally: tiers that allocate elsewhere (dynamic-parent + // classes, method-closure literals) never read it, and it must + // not leak past this push (#7590's leak class). + ctx.pretenure_next_object_literal = false; let arr_box = lower_expr(ctx, &array_expr)?; // Repsel 4a.1 (#6904 recon): the guarded numeric push was an diff --git a/crates/perry-codegen/src/expr/mod.rs b/crates/perry-codegen/src/expr/mod.rs index 0511a5b11a..a2f18ffea6 100644 --- a/crates/perry-codegen/src/expr/mod.rs +++ b/crates/perry-codegen/src/expr/mod.rs @@ -250,6 +250,15 @@ pub(crate) struct FnCtx<'a> { /// reading the field, because they consult it *after* lowering their /// operands, by which point the field has been taken again. pub discard_this_expr: bool, + /// #7598: the next object literal lowered is a pretenure-accumulator push + /// value and should allocate born-tenured in old-gen. Set by + /// `array_push::lower` immediately before lowering the pushed value when + /// the target local is `native_facts.pretenure_accumulator`-admitted and + /// the value is a plain object literal; **taken** (`mem::take`) at the top + /// of `lower_object_literal`, so it reaches exactly the root literal — + /// nested literals in field initializers read `false` (the #7590 take + /// discipline). The push site asserts it was consumed after lowering. + pub pretenure_next_object_literal: bool, /// HIR FuncId → LLVM function name. Resolved at the top of /// `compile_module` so `FuncRef(id)` calls know what to emit. pub func_names: &'a std::collections::HashMap, diff --git a/crates/perry-codegen/src/expr/object_literal.rs b/crates/perry-codegen/src/expr/object_literal.rs index 43e61327d1..36a363b3e7 100644 --- a/crates/perry-codegen/src/expr/object_literal.rs +++ b/crates/perry-codegen/src/expr/object_literal.rs @@ -224,6 +224,11 @@ pub(crate) fn lower_object_literal( props: &[(String, Expr)], expected_ty: Option<&HirType>, ) -> Result { + // #7598: TAKEN here so it covers exactly this literal — field initializers + // containing nested literals lower below with the flag already cleared + // (the #7590 take discipline). Only the shaped fast path honors it; the + // by-name fallback (method closures) simply allocates young. + let pretenure = std::mem::take(&mut ctx.pretenure_next_object_literal); // #6951: the object handle is allocated BEFORE the property values are // lowered and lives in an SSA register across all of them. `{ a: s, b: f() }` // therefore had its half-built object swept by `f`'s collection, and the @@ -283,9 +288,14 @@ pub(crate) fn lower_object_literal( } let shape_id_str = shape_id.to_string(); + let alloc_fn = if pretenure { + "js_object_alloc_with_shape_pretenured" + } else { + "js_object_alloc_with_shape" + }; let obj_handle = ctx.block().call( I64, - "js_object_alloc_with_shape", + alloc_fn, &[ (I32, &shape_id_str), (I32, &n_str), diff --git a/crates/perry-codegen/src/lower_call/new.rs b/crates/perry-codegen/src/lower_call/new.rs index 021b0f43ee..706be020ad 100644 --- a/crates/perry-codegen/src/lower_call/new.rs +++ b/crates/perry-codegen/src/lower_call/new.rs @@ -625,7 +625,14 @@ fn lower_new_impl_inner( // NOTE the env test is `is_none()`: `PERRY_INLINE_NEW=""` *enables* // the inline path, because an empty string is `Some("")`. let force_inline_new = std::env::var_os("PERRY_INLINE_NEW").is_some(); - if !force_inline_new && !new_site_is_in_loop(ctx) { + // #7598: a pretenure-accumulator push value is born TENURED in + // old-gen via the outlined call, never the inline Eden bump — the + // ~140-cycle call it re-pays is noise against the Eden→survivor→old + // double copy it removes for a cohort that is live for the rest of + // the loop by construction. Taken here so it covers exactly this + // allocation; the constructor's own inner allocations read `false`. + let pretenure = std::mem::take(&mut ctx.pretenure_next_object_literal); + if pretenure || (!force_inline_new && !new_site_is_in_loop(ctx)) { let keys_slot = if let Some(s) = ctx.class_keys_slots.get(class_name).cloned() { s } else { @@ -635,14 +642,16 @@ fn lower_new_impl_inner( s }; let keys_ptr = ctx.block().load(I64, &keys_slot); - ctx.pending_declares.push(( - "js_object_alloc_class_inline_keys".to_string(), - I64, - vec![I32, I32, I32, I64], - )); + let alloc_fn = if pretenure { + "js_object_alloc_class_inline_keys_pretenured" + } else { + "js_object_alloc_class_inline_keys" + }; + ctx.pending_declares + .push((alloc_fn.to_string(), I64, vec![I32, I32, I32, I64])); ctx.block().call( I64, - "js_object_alloc_class_inline_keys", + alloc_fn, &[ (I32, &cid_str), (I32, &parent_cid_str), diff --git a/crates/perry-codegen/src/runtime_decls/objects.rs b/crates/perry-codegen/src/runtime_decls/objects.rs index d7cb0557af..27c2fcc6c5 100644 --- a/crates/perry-codegen/src/runtime_decls/objects.rs +++ b/crates/perry-codegen/src/runtime_decls/objects.rs @@ -72,6 +72,12 @@ pub fn declare_phase_b_objects(module: &mut LlModule) { // so subsequent field stores can use index-based set_field (skipping the // per-call linear key-search done by js_object_set_field_by_name). module.declare_function("js_object_alloc_with_shape", I64, &[I32, I32, PTR, I32]); + // #7598: born-tenured variant for pretenure-accumulator push literals. + module.declare_function( + "js_object_alloc_with_shape_pretenured", + I64, + &[I32, I32, PTR, I32], + ); // Index-based field setter (no key lookup). Hot-path target for object // literals with statically-known keys; the i-th field directly maps to // the i-th packed-keys entry above. diff --git a/crates/perry-runtime/src/object/alloc.rs b/crates/perry-runtime/src/object/alloc.rs index d7af244b9e..4ead8078a3 100644 --- a/crates/perry-runtime/src/object/alloc.rs +++ b/crates/perry-runtime/src/object/alloc.rs @@ -244,6 +244,50 @@ pub extern "C" fn js_object_alloc_class_inline_keys( parent_class_id: u32, field_count: u32, keys_array: *mut ArrayHeader, +) -> *mut ObjectHeader { + alloc_class_inline_keys_impl(class_id, parent_class_id, field_count, keys_array, false) +} + +/// #7598: `js_object_alloc_class_inline_keys` for an allocation site codegen +/// proved to feed a long-lived accumulator (`out = []` outside every loop, +/// filled by `out.push(...)` inside one — object literals arrive here as +/// synthesized AnonShape classes). The instance is born in old-gen with +/// `GC_FLAG_TENURED`, so the copying minor never pays the +/// Eden→survivor→old double copy its cohort was measured to cost (#7592: +/// 3.9 s of a 5.1 s phase). Constructor field stores need no special casing: +/// they funnel through `runtime_store_jsvalue_slot` / the #7602-gated +/// barrier, both of which read the LIVE parent header and remember old→young +/// edges exactly as for a promoted object. The `Old ⟹ TENURED` invariant +/// holds by construction: `arena_alloc_gc_old_born_tenured` sets the bit +/// itself and is pinned by `every_old_gen_birth_path_sets_tenured`. +#[no_mangle] +pub extern "C" fn js_object_alloc_class_inline_keys_pretenured( + class_id: u32, + parent_class_id: u32, + field_count: u32, + keys_array: *mut ArrayHeader, +) -> *mut ObjectHeader { + alloc_class_inline_keys_impl(class_id, parent_class_id, field_count, keys_array, true) +} + +/// Keepalive anchor — `js_object_alloc_class_inline_keys_pretenured` is a +/// generated-code-only callee, so the auto-optimize whole-program build would +/// otherwise dead-strip it (see the FFI-symbol-link-break class). +#[cfg(feature = "keepalive-anchors")] +#[used] +static KEEP_JS_OBJECT_ALLOC_CLASS_INLINE_KEYS_PRETENURED: extern "C" fn( + u32, + u32, + u32, + *mut ArrayHeader, +) -> *mut ObjectHeader = js_object_alloc_class_inline_keys_pretenured; + +fn alloc_class_inline_keys_impl( + class_id: u32, + parent_class_id: u32, + field_count: u32, + keys_array: *mut ArrayHeader, + pretenure: bool, ) -> *mut ObjectHeader { if parent_class_id != 0 { register_class(class_id, parent_class_id); @@ -263,7 +307,14 @@ pub extern "C" fn js_object_alloc_class_inline_keys( let fields_size = alloc_field_count * std::mem::size_of::(); let total_size = header_size + fields_size; - let ptr = arena_alloc_gc(total_size, 8, crate::gc::GC_TYPE_OBJECT) as *mut ObjectHeader; + // Both allocators take the PAYLOAD size and pad the GcHeader in + // themselves; the branches differ only in which generation a small object + // is born into. + let ptr = if pretenure { + crate::arena::arena_alloc_gc_old_born_tenured(total_size, 8, crate::gc::GC_TYPE_OBJECT) + } else { + arena_alloc_gc(total_size, 8, crate::gc::GC_TYPE_OBJECT) + } as *mut ObjectHeader; unsafe { (*ptr).object_type = crate::error::OBJECT_TYPE_REGULAR; @@ -607,13 +658,66 @@ pub extern "C" fn js_object_alloc_with_shape( field_count: u32, packed_keys: *const u8, packed_keys_len: u32, +) -> *mut ObjectHeader { + alloc_with_shape_impl(shape_id, field_count, packed_keys, packed_keys_len, false) +} + +/// #7598: `js_object_alloc_with_shape` for an allocation site codegen proved +/// to feed a long-lived accumulator (`out = []` outside every loop, filled by +/// `out.push({...})` inside one). The object is born in old-gen with +/// `GC_FLAG_TENURED`, so the copying minor never pays the Eden→survivor→old +/// double copy its cohort was measured to cost (#7592: 3.9 s of a 5.1 s +/// phase). Field stores need no special casing: they funnel through +/// `runtime_store_jsvalue_slot`, whose write barrier reads the LIVE parent +/// header — a born-tenured parent takes the old-parent path and remembers +/// old→young field edges exactly as a promoted object would. +/// +/// The `Old ⟹ TENURED` invariant #7602's gate rests on holds by construction +/// here: `arena_alloc_gc_old_born_tenured` sets the bit itself and is pinned +/// by `every_old_gen_birth_path_sets_tenured`. +#[no_mangle] +pub extern "C" fn js_object_alloc_with_shape_pretenured( + shape_id: u32, + field_count: u32, + packed_keys: *const u8, + packed_keys_len: u32, +) -> *mut ObjectHeader { + alloc_with_shape_impl(shape_id, field_count, packed_keys, packed_keys_len, true) +} + +/// Keepalive anchor — `js_object_alloc_with_shape_pretenured` is a +/// generated-code-only callee, so the auto-optimize whole-program build would +/// otherwise dead-strip it (see the FFI-symbol-link-break class). +#[cfg(feature = "keepalive-anchors")] +#[used] +static KEEP_JS_OBJECT_ALLOC_WITH_SHAPE_PRETENURED: extern "C" fn( + u32, + u32, + *const u8, + u32, +) -> *mut ObjectHeader = js_object_alloc_with_shape_pretenured; + +fn alloc_with_shape_impl( + shape_id: u32, + field_count: u32, + packed_keys: *const u8, + packed_keys_len: u32, + pretenure: bool, ) -> *mut ObjectHeader { let header_size = std::mem::size_of::(); // Allocate extra field slots for dynamic property growth (plain objects may get new fields) let alloc_field_count = std::cmp::max(field_count as usize, crate::object::INLINE_SLOT_FLOOR); let fields_size = alloc_field_count * 8; let total_size = header_size + fields_size; - let obj_ptr = arena_alloc_gc(total_size, 8, crate::gc::GC_TYPE_OBJECT) as *mut ObjectHeader; + // Both allocators take the PAYLOAD size and pad the GcHeader in themselves; + // `arena_alloc_gc`'s own large-object arm is exactly the born-tenured + // shape, so the two branches differ only in which generation a small + // object is born into. + let obj_ptr = if pretenure { + crate::arena::arena_alloc_gc_old_born_tenured(total_size, 8, crate::gc::GC_TYPE_OBJECT) + } else { + arena_alloc_gc(total_size, 8, crate::gc::GC_TYPE_OBJECT) + } as *mut ObjectHeader; unsafe { (*obj_ptr).object_type = crate::error::OBJECT_TYPE_REGULAR; From f0a2e760ac2bd10c84616357372f733f4667b225 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 8 Aug 2026 06:09:05 +0200 Subject: [PATCH 2/5] perf(gc): bump-and-defer old-gen births; restrict pretenure to run-once regions (#7598) Two fixes that turn the measured v1 loss (2.73s -> 4.19s at json 200k) into a win (-> 1.54s, RSS -112 MB, hash identical): 1. arena_alloc_gc_old_born_tenured_bump: no per-allocation old_free_take_exact probe, and page registration DEFERRED into a thread-local buffer flushed at old_pages_begin_gc_cycle (every cycle kind constructs through it) or at a 64k-entry cap. The per-object register_old_object_pages was the profile's top cost: two RefCell borrows, two Vec allocations, and a linear dedup scan of the page's object list -- quadratic as a page fills. Every reader of that index runs at GC time, so cycle-start visibility is sufficient. 2. Pretenure admission now requires the REGION to run exactly once (module main/init -- entry.rs's two fact graphs pass true, every function/method/closure region passes false). A function body's depth-0 accumulator is re-entered per call and its cohort dies at return: measured 6.6x slower with 4x RSS when pretenured. Only a run-once region makes "declared outside every loop" a cohort-lifetime proof. The parameter is explicit at every call site so a new region kind must choose. json_pipeline 500k: 8.4 -> 5.4 s, RSS -173 MB; adversarial and push_bench emit zero pretenured calls (IR-verified) and are unchanged. --- crates/perry-codegen/src/codegen/closure.rs | 3 +- crates/perry-codegen/src/codegen/entry.rs | 4 ++ crates/perry-codegen/src/codegen/function.rs | 1 + crates/perry-codegen/src/codegen/method.rs | 6 ++- .../src/collectors/all_pointer_arrays.rs | 2 +- .../perry-codegen/src/collectors/hir_facts.rs | 18 ++++++- crates/perry-runtime/src/arena/allocators.rs | 42 +++++++++++++++++ crates/perry-runtime/src/arena/mod.rs | 3 +- crates/perry-runtime/src/arena/page_meta.rs | 47 +++++++++++++++++++ crates/perry-runtime/src/object/alloc.rs | 4 +- 10 files changed, 121 insertions(+), 9 deletions(-) diff --git a/crates/perry-codegen/src/codegen/closure.rs b/crates/perry-codegen/src/codegen/closure.rs index dbc5edd166..af81dc65f7 100644 --- a/crates/perry-codegen/src/codegen/closure.rs +++ b/crates/perry-codegen/src/codegen/closure.rs @@ -802,7 +802,8 @@ pub(super) fn compile_closure( classes, &cross_module.compile_time_constants, &cross_module.module_dispatch, - ); + false, + ); // Representation-selection context gates (see codegen/function.rs). // Async-step closures (CPS-rewritten `async` closures — the rewrite clears diff --git a/crates/perry-codegen/src/codegen/entry.rs b/crates/perry-codegen/src/codegen/entry.rs index 54a1352105..8ac1ccd486 100644 --- a/crates/perry-codegen/src/codegen/entry.rs +++ b/crates/perry-codegen/src/codegen/entry.rs @@ -668,6 +668,7 @@ pub(super) fn compile_module_entry( classes, &cross_module.compile_time_constants, &cross_module.module_dispatch, + true, ); // #7109: the program-entry body participates in canonical (i32/u32/Str) // selection on exactly the per-value rules a function body uses. There @@ -708,6 +709,7 @@ pub(super) fn compile_module_entry( current_block: 0, discard_expr_value: false, discard_this_expr: false, + pretenure_next_object_literal: false, func_names, strings, loop_targets: Vec::new(), @@ -1337,6 +1339,7 @@ pub(super) fn compile_module_entry( classes, &cross_module.compile_time_constants, &cross_module.module_dispatch, + true, ); // #7109: the module-init body participates in canonical (i32/u32/Str) // selection on exactly the per-value rules a function body uses. There @@ -1375,6 +1378,7 @@ pub(super) fn compile_module_entry( current_block: 0, discard_expr_value: false, discard_this_expr: false, + pretenure_next_object_literal: false, func_names, strings, loop_targets: Vec::new(), diff --git a/crates/perry-codegen/src/codegen/function.rs b/crates/perry-codegen/src/codegen/function.rs index cc838adb2f..338fb01b0d 100644 --- a/crates/perry-codegen/src/codegen/function.rs +++ b/crates/perry-codegen/src/codegen/function.rs @@ -620,6 +620,7 @@ pub(super) fn compile_function( &cross_module.compile_time_constants, &cross_module.module_dispatch, &spec_ta_lens, + false, ); if let Some(plan) = spec_entry { diff --git a/crates/perry-codegen/src/codegen/method.rs b/crates/perry-codegen/src/codegen/method.rs index 340eaa468b..ec29bf86fb 100644 --- a/crates/perry-codegen/src/codegen/method.rs +++ b/crates/perry-codegen/src/codegen/method.rs @@ -388,7 +388,8 @@ pub(super) fn compile_method( classes, &cross_module.compile_time_constants, &cross_module.module_dispatch, - ); + false, + ); // Representation-selection context gates (see codegen/function.rs). let repsel_flags = crate::expr::RepselContextFlags::for_body( @@ -1449,7 +1450,8 @@ pub(super) fn compile_static_method( classes, &cross_module.compile_time_constants, &cross_module.module_dispatch, - ); + false, + ); // Representation-selection context gates (see codegen/function.rs). let repsel_flags = diff --git a/crates/perry-codegen/src/collectors/all_pointer_arrays.rs b/crates/perry-codegen/src/collectors/all_pointer_arrays.rs index d669190c07..17b1f96400 100644 --- a/crates/perry-codegen/src/collectors/all_pointer_arrays.rs +++ b/crates/perry-codegen/src/collectors/all_pointer_arrays.rs @@ -250,7 +250,7 @@ fn scan_depths( decl_depth: &mut HashMap, push_depths: &mut HashMap>, ) { - let mut record = |expr: &Expr, at: u32, push_depths: &mut HashMap>| { + let record = |expr: &Expr, at: u32, push_depths: &mut HashMap>| { super::scalar_method_dispatch::for_each_expr(expr, &mut |e| { if let Expr::ArrayPush { array_id, .. } = e { push_depths.entry(*array_id).or_default().push(at); diff --git a/crates/perry-codegen/src/collectors/hir_facts.rs b/crates/perry-codegen/src/collectors/hir_facts.rs index cff523f16a..e194e2a4e1 100644 --- a/crates/perry-codegen/src/collectors/hir_facts.rs +++ b/crates/perry-codegen/src/collectors/hir_facts.rs @@ -687,6 +687,7 @@ pub(crate) fn collect_native_region_fact_graph( classes: &HashMap, compile_time_constants: &HashMap, module_dispatch: &super::ModuleDispatchFacts, + region_runs_once: bool, ) -> NativeRegionFactGraph { collect_native_region_fact_graph_with_spec_lens( stmts, @@ -701,6 +702,7 @@ pub(crate) fn collect_native_region_fact_graph( compile_time_constants, module_dispatch, &HashMap::new(), + region_runs_once, ) } @@ -722,8 +724,9 @@ pub(crate) fn collect_native_region_fact_graph_with_spec_lens( compile_time_constants: &HashMap, module_dispatch: &super::ModuleDispatchFacts, spec_ta_lens: &HashMap, + region_runs_once: bool, ) -> NativeRegionFactGraph { - collect_type_facts( + let mut facts = collect_type_facts( stmts, params, flat_const_ids, @@ -736,7 +739,17 @@ pub(crate) fn collect_native_region_fact_graph_with_spec_lens( compile_time_constants, module_dispatch, spec_ta_lens, - ) + ); + // #7598: pretenure-accumulator admission additionally requires the REGION + // to run exactly once (module main/init). A function body's depth-0 + // accumulator is re-entered on every call and its cohort dies at return — + // measured 6.6x slower with 4x the RSS when pretenured (the adversarial + // arm in the PR). Only a run-once region makes "declared outside every + // loop" a cohort-lifetime proof. + if !region_runs_once { + facts.arrays.pretenure_accumulator_locals.clear(); + } + facts } // #854: thin wrapper over collect_type_facts, currently only exercised by this @@ -2114,6 +2127,7 @@ mod tests { &HashMap::new(), &HashMap::new(), &crate::collectors::ModuleDispatchFacts::default(), + true, ); assert!(graph.integer_locals().contains(&1)); diff --git a/crates/perry-runtime/src/arena/allocators.rs b/crates/perry-runtime/src/arena/allocators.rs index 1270929140..da357833c4 100644 --- a/crates/perry-runtime/src/arena/allocators.rs +++ b/crates/perry-runtime/src/arena/allocators.rs @@ -154,6 +154,48 @@ pub fn arena_alloc_gc_old(size: usize, align: usize, obj_type: u8) -> *mut u8 { unsafe { raw.add(GC_HEADER_SIZE) } } +/// #7598: the born-tenured allocator for the PRETENURE-ACCUMULATOR hot path — +/// a per-object rate the ordinary old-gen path was never built for. +/// Two deliberate departures from `arena_alloc_gc_old`, both measured to be +/// the difference between winning and losing the pretenure trade: +/// +/// - **No `old_free_take_exact` probe.** Hole reuse is an anti-growth +/// mechanism for occasional promotions; on a 100k-object burst the probe is +/// pure per-allocation overhead, and burst cohorts live and die together — +/// contiguous bump placement is what old-page defrag wants from them anyway. +/// - **Deferred page registration.** `register_old_object_pages` per object +/// pays two RefCell borrows, two Vec allocations, and a linear dedup scan of +/// the page's object list (quadratic as a page fills). Every reader of that +/// index runs at GC time, so the burst defers into +/// `defer_old_object_page_registration` and `old_pages_begin_gc_cycle` +/// flushes before any collector work reads it. +/// +/// Header init is identical to `arena_alloc_gc_old` + `GC_FLAG_TENURED` in +/// the same breath — the `Old ⟹ TENURED` contract (#7602) holds here exactly +/// as for the wrapper below. +pub(crate) fn arena_alloc_gc_old_born_tenured_bump( + size: usize, + align: usize, + obj_type: u8, +) -> *mut u8 { + use crate::gc::{GcHeader, GC_FLAG_ARENA, GC_FLAG_TENURED, GC_HEADER_SIZE}; + + let pad = align.max(8); + let total = (GC_HEADER_SIZE + size + pad - 1) & !(pad - 1); + let raw = arena_alloc_old(total, align); + unsafe { + let header = raw as *mut GcHeader; + (*header).obj_type = obj_type; + (*header).gc_flags = + GC_FLAG_ARENA | GC_FLAG_TENURED | crate::gc::gc_birth_extra_flags(); + crate::gc::gc_note_black_birth(header); + (*header)._reserved = 0; + (*header).size = total as u32; + } + super::page_meta::defer_old_object_page_registration(raw as usize, total); + unsafe { raw.add(GC_HEADER_SIZE) } +} + /// The old-gen + born-tenured shape `arena_alloc_gc` hands a LARGE object, for /// a caller that wants it on size-independent grounds. /// diff --git a/crates/perry-runtime/src/arena/mod.rs b/crates/perry-runtime/src/arena/mod.rs index 04027c5551..58ee661838 100644 --- a/crates/perry-runtime/src/arena/mod.rs +++ b/crates/perry-runtime/src/arena/mod.rs @@ -63,7 +63,8 @@ pub use allocators::{ arena_alloc_longlived, arena_alloc_old, js_arena_alloc, }; pub(crate) use allocators::{ - arena_alloc_gc_old_born_tenured, arena_alloc_gc_old_excluding_pages, arena_alloc_gc_survivor, + arena_alloc_gc_old_born_tenured, arena_alloc_gc_old_born_tenured_bump, + arena_alloc_gc_old_excluding_pages, arena_alloc_gc_survivor, }; // walk.rs diff --git a/crates/perry-runtime/src/arena/page_meta.rs b/crates/perry-runtime/src/arena/page_meta.rs index 0b4653b17e..f24f2d4f52 100644 --- a/crates/perry-runtime/src/arena/page_meta.rs +++ b/crates/perry-runtime/src/arena/page_meta.rs @@ -656,6 +656,50 @@ pub(crate) fn register_old_object_pages(header_addr: usize, total_size: usize) { update_old_page_meta_for_object(&added_pages, true); } +thread_local! { + /// #7598: page registrations deferred by the born-tenured BUMP allocator. + /// `register_old_object_pages` per allocation is the measured killer of + /// site-level pretenuring — per object it pays two `RefCell` borrows, two + /// `Vec` allocations, and a **linear `contains` scan of the page's object + /// list** (O(objects-per-page)² as a page fills; the dedup exists because + /// hole reuse can re-register an address that `unregister` never removed). + /// Every reader of the page-objects index runs at GC time (defrag page + /// selection, sweep accounting), so registration only has to be visible by + /// cycle start — `old_pages_begin_gc_cycle` flushes this buffer, and a + /// size cap bounds it between cycles. + static DEFERRED_OLD_PAGE_REGISTRATIONS: std::cell::RefCell> = + const { std::cell::RefCell::new(Vec::new()) }; +} + +/// Cap chosen so the buffer's worst-case footprint (16 B/entry × 64k = 1 MB) +/// stays a rounding error while flushes stay rare on allocation bursts. +const DEFERRED_OLD_PAGE_REGISTRATION_CAP: usize = 65_536; + +/// #7598: defer this object's page registration to the next flush. ONLY for +/// freshly bump-allocated born-tenured objects — the deferred entry relies on +/// the full `register_old_object_pages` (with its hole-reuse dedup) running at +/// flush time, it just runs it off the allocation hot path. +pub(crate) fn defer_old_object_page_registration(header_addr: usize, total_size: usize) { + let flush_now = DEFERRED_OLD_PAGE_REGISTRATIONS.with(|buf| { + let mut buf = buf.borrow_mut(); + buf.push((header_addr, total_size)); + buf.len() >= DEFERRED_OLD_PAGE_REGISTRATION_CAP + }); + if flush_now { + flush_deferred_old_page_registrations(); + } +} + +/// Drain the deferred buffer through the real registration path. Called from +/// `old_pages_begin_gc_cycle` (every collection begins with an accurate +/// index) and from the size-cap overflow in `defer_old_object_page_registration`. +pub(crate) fn flush_deferred_old_page_registrations() { + let pending = DEFERRED_OLD_PAGE_REGISTRATIONS.with(|buf| std::mem::take(&mut *buf.borrow_mut())); + for (header_addr, total_size) in pending { + register_old_object_pages(header_addr, total_size); + } +} + #[allow(dead_code)] pub(crate) fn unregister_old_object_pages(header_addr: usize, total_size: usize) { if header_addr == 0 || total_size == 0 { @@ -683,6 +727,9 @@ pub(crate) fn unregister_old_object_pages(header_addr: usize, total_size: usize) } pub(crate) fn old_pages_begin_gc_cycle() { + // #7598: born-tenured bump allocations defer their page registration; + // every collection must begin with an accurate page-objects index. + flush_deferred_old_page_registrations(); // #6181: the per-page `dirty_slots` reset used to iterate every old page // here (O(old pages) on every minor, growing with old-gen size). It is now // a single epoch bump — a page whose `dirty_slots_epoch` predates the new diff --git a/crates/perry-runtime/src/object/alloc.rs b/crates/perry-runtime/src/object/alloc.rs index 4ead8078a3..bb95015a7e 100644 --- a/crates/perry-runtime/src/object/alloc.rs +++ b/crates/perry-runtime/src/object/alloc.rs @@ -311,7 +311,7 @@ fn alloc_class_inline_keys_impl( // themselves; the branches differ only in which generation a small object // is born into. let ptr = if pretenure { - crate::arena::arena_alloc_gc_old_born_tenured(total_size, 8, crate::gc::GC_TYPE_OBJECT) + crate::arena::arena_alloc_gc_old_born_tenured_bump(total_size, 8, crate::gc::GC_TYPE_OBJECT) } else { arena_alloc_gc(total_size, 8, crate::gc::GC_TYPE_OBJECT) } as *mut ObjectHeader; @@ -714,7 +714,7 @@ fn alloc_with_shape_impl( // shape, so the two branches differ only in which generation a small // object is born into. let obj_ptr = if pretenure { - crate::arena::arena_alloc_gc_old_born_tenured(total_size, 8, crate::gc::GC_TYPE_OBJECT) + crate::arena::arena_alloc_gc_old_born_tenured_bump(total_size, 8, crate::gc::GC_TYPE_OBJECT) } else { arena_alloc_gc(total_size, 8, crate::gc::GC_TYPE_OBJECT) } as *mut ObjectHeader; From 8dff6499de3741162aaf1d913cd723539e163842 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 8 Aug 2026 06:34:36 +0200 Subject: [PATCH 3/5] style: cargo fmt + the fifth fact-graph test caller --- crates/perry-codegen/src/codegen/closure.rs | 4 ++-- crates/perry-codegen/src/codegen/method.rs | 8 ++++---- crates/perry-codegen/src/collectors/hir_facts.rs | 1 + crates/perry-runtime/src/arena/allocators.rs | 3 +-- crates/perry-runtime/src/arena/page_meta.rs | 3 ++- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/crates/perry-codegen/src/codegen/closure.rs b/crates/perry-codegen/src/codegen/closure.rs index af81dc65f7..cf8c614c94 100644 --- a/crates/perry-codegen/src/codegen/closure.rs +++ b/crates/perry-codegen/src/codegen/closure.rs @@ -802,8 +802,8 @@ pub(super) fn compile_closure( classes, &cross_module.compile_time_constants, &cross_module.module_dispatch, - false, - ); + false, + ); // Representation-selection context gates (see codegen/function.rs). // Async-step closures (CPS-rewritten `async` closures — the rewrite clears diff --git a/crates/perry-codegen/src/codegen/method.rs b/crates/perry-codegen/src/codegen/method.rs index ec29bf86fb..288e544389 100644 --- a/crates/perry-codegen/src/codegen/method.rs +++ b/crates/perry-codegen/src/codegen/method.rs @@ -388,8 +388,8 @@ pub(super) fn compile_method( classes, &cross_module.compile_time_constants, &cross_module.module_dispatch, - false, - ); + false, + ); // Representation-selection context gates (see codegen/function.rs). let repsel_flags = crate::expr::RepselContextFlags::for_body( @@ -1450,8 +1450,8 @@ pub(super) fn compile_static_method( classes, &cross_module.compile_time_constants, &cross_module.module_dispatch, - false, - ); + false, + ); // Representation-selection context gates (see codegen/function.rs). let repsel_flags = diff --git a/crates/perry-codegen/src/collectors/hir_facts.rs b/crates/perry-codegen/src/collectors/hir_facts.rs index e194e2a4e1..be2d212097 100644 --- a/crates/perry-codegen/src/collectors/hir_facts.rs +++ b/crates/perry-codegen/src/collectors/hir_facts.rs @@ -2036,6 +2036,7 @@ mod tests { &HashMap::new(), &constants, &crate::collectors::ModuleDispatchFacts::default(), + true, ); assert!(graph.known_noalias_buffer_locals().contains(&1)); diff --git a/crates/perry-runtime/src/arena/allocators.rs b/crates/perry-runtime/src/arena/allocators.rs index da357833c4..fec15392e7 100644 --- a/crates/perry-runtime/src/arena/allocators.rs +++ b/crates/perry-runtime/src/arena/allocators.rs @@ -186,8 +186,7 @@ pub(crate) fn arena_alloc_gc_old_born_tenured_bump( unsafe { let header = raw as *mut GcHeader; (*header).obj_type = obj_type; - (*header).gc_flags = - GC_FLAG_ARENA | GC_FLAG_TENURED | crate::gc::gc_birth_extra_flags(); + (*header).gc_flags = GC_FLAG_ARENA | GC_FLAG_TENURED | crate::gc::gc_birth_extra_flags(); crate::gc::gc_note_black_birth(header); (*header)._reserved = 0; (*header).size = total as u32; diff --git a/crates/perry-runtime/src/arena/page_meta.rs b/crates/perry-runtime/src/arena/page_meta.rs index f24f2d4f52..6aa3ac9f9a 100644 --- a/crates/perry-runtime/src/arena/page_meta.rs +++ b/crates/perry-runtime/src/arena/page_meta.rs @@ -694,7 +694,8 @@ pub(crate) fn defer_old_object_page_registration(header_addr: usize, total_size: /// `old_pages_begin_gc_cycle` (every collection begins with an accurate /// index) and from the size-cap overflow in `defer_old_object_page_registration`. pub(crate) fn flush_deferred_old_page_registrations() { - let pending = DEFERRED_OLD_PAGE_REGISTRATIONS.with(|buf| std::mem::take(&mut *buf.borrow_mut())); + let pending = + DEFERRED_OLD_PAGE_REGISTRATIONS.with(|buf| std::mem::take(&mut *buf.borrow_mut())); for (header_addr, total_size) in pending { register_old_object_pages(header_addr, total_size); } From 6d456814c12f5082cb3fd12f68a0be05f5ebd5a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 8 Aug 2026 06:40:58 +0200 Subject: [PATCH 4/5] docs(changelog): add fragment for #7623 --- ...-static-pretenure-run-once-accumulators.md | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 changelog.d/7623-static-pretenure-run-once-accumulators.md diff --git a/changelog.d/7623-static-pretenure-run-once-accumulators.md b/changelog.d/7623-static-pretenure-run-once-accumulators.md new file mode 100644 index 0000000000..4cead46553 --- /dev/null +++ b/changelog.d/7623-static-pretenure-run-once-accumulators.md @@ -0,0 +1,24 @@ +**perf(gc): static pretenuring for run-once accumulator loops (#7598)** + +The dominant cost on promote-heavy loops after #7594/#7596 was structural: +every long-lived object was copied twice, Eden→survivor by the first copying +minor and survivor→old by the next. Objects pushed into an accumulator that +codegen can prove long-lived — the all-pointer admission terms, plus the +`let` at loop depth 0, every push at depth ≥ 1, and the region running +exactly once (module main/init; `region_runs_once` is an explicit parameter +at every fact-graph call site) — are now born in old-gen with +`GC_FLAG_TENURED`, via a `mem::take`n per-site flag consumed by the root +allocation only. + +Two allocator fixes make it a win instead of the measured loss: the new +`arena_alloc_gc_old_born_tenured_bump` defers `register_old_object_pages` +(per-object it paid a linear dedup scan of the page's object list — quadratic +as a page fills) into a buffer flushed at `old_pages_begin_gc_cycle`, and +skips the hole-reuse probe; and function/method/closure regions refuse +admission outright, since a per-call accumulator's cohort dies at return +(measured 6.6× slower when pretenured). + +json_pipeline: 200k 2.40 → 1.54 s (peak RSS 543 → 431 MB), 500k ~8 → 5.4 s +(RSS −173 MB), output hash identical; bytes copied by minors 108 → 0.0 MB. +Mechanism is IR-verified: exactly one pretenured call site in the workload, +zero in the numeric-push and function-region benches. From eaa7057b75ac7cbe34b0ca635a281a42df7fccf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 8 Aug 2026 08:35:12 +0200 Subject: [PATCH 5/5] refactor(gc): strip #7623 to admission infrastructure per audit The two PR audits established that the pretenure mechanism was correct but the target was wrong: json_pipeline's minor-moved cohort (~113 MB) is the runtime-allocated parse tree, not codegen-visible literals (~12 MB total, ~1 MB live at minor time), and the measured 108 MB -> 0 was a confound -- the base arm predated #7613's promote-on-first-copy seed, which on current main fires in both arms. Removed: the born-tenured allocator entry points and their keepalive anchors (an unused #[no_mangle] + #[used] pair is unused configuration per the kill-policy, and un-strippable bytes per the hello-size anchor class), the codegen consumers, and the deferred-page-registration fix (extracted separately on perf/old-page-registration-deferral, crediting this PR's finding). Kept: collect_pretenure_accumulator_locals with its refusal tests, and the explicit region_runs_once parameter on both fact-graph builders (module main/init true, function/method/closure false) with a graph-level test pinning both polarities -- the admission half a future dynamic-feedback pretenurer needs. --- ...-static-pretenure-run-once-accumulators.md | 40 +++---- crates/perry-codegen/src/codegen/closure.rs | 1 - crates/perry-codegen/src/codegen/entry.rs | 2 - crates/perry-codegen/src/codegen/function.rs | 1 - crates/perry-codegen/src/codegen/method.rs | 2 - .../perry-codegen/src/collectors/hir_facts.rs | 44 +++++++ crates/perry-codegen/src/expr/array_push.rs | 20 ---- crates/perry-codegen/src/expr/mod.rs | 9 -- .../perry-codegen/src/expr/object_literal.rs | 12 +- crates/perry-codegen/src/lower_call/new.rs | 23 ++-- .../src/runtime_decls/objects.rs | 6 - crates/perry-runtime/src/arena/allocators.rs | 41 ------- crates/perry-runtime/src/arena/mod.rs | 3 +- crates/perry-runtime/src/arena/page_meta.rs | 48 -------- crates/perry-runtime/src/object/alloc.rs | 108 +----------------- 15 files changed, 73 insertions(+), 287 deletions(-) diff --git a/changelog.d/7623-static-pretenure-run-once-accumulators.md b/changelog.d/7623-static-pretenure-run-once-accumulators.md index 4cead46553..59d99789b5 100644 --- a/changelog.d/7623-static-pretenure-run-once-accumulators.md +++ b/changelog.d/7623-static-pretenure-run-once-accumulators.md @@ -1,24 +1,20 @@ -**perf(gc): static pretenuring for run-once accumulator loops (#7598)** +**gc: pretenure-accumulator admission infrastructure (#7598, scope-reduced per audit)** -The dominant cost on promote-heavy loops after #7594/#7596 was structural: -every long-lived object was copied twice, Eden→survivor by the first copying -minor and survivor→old by the next. Objects pushed into an accumulator that -codegen can prove long-lived — the all-pointer admission terms, plus the -`let` at loop depth 0, every push at depth ≥ 1, and the region running -exactly once (module main/init; `region_runs_once` is an explicit parameter -at every fact-graph call site) — are now born in old-gen with -`GC_FLAG_TENURED`, via a `mem::take`n per-site flag consumed by the root -allocation only. +Adds the static admission machinery for a future pretenurer, with no +allocator or codegen consumers: `collect_pretenure_accumulator_locals` +(accumulator `let` at loop depth 0, every push at depth ≥ 1, layered on the +all-pointer terms, refusal tests for the per-iteration and mixed-depth +shapes) and an explicit `region_runs_once` parameter on both fact-graph +builders — module main/init pass true, every function/method/closure region +false, with a graph-level test pinning both polarities. Only a run-once +region makes "declared outside every loop" a cohort-lifetime claim; a +function region's accumulator is re-entered per call (measured 6.6× slower +when pretenured). -Two allocator fixes make it a win instead of the measured loss: the new -`arena_alloc_gc_old_born_tenured_bump` defers `register_old_object_pages` -(per-object it paid a linear dedup scan of the page's object list — quadratic -as a page fills) into a buffer flushed at `old_pages_begin_gc_cycle`, and -skips the hole-reuse probe; and function/method/closure regions refuse -admission outright, since a per-call accumulator's cohort dies at return -(measured 6.6× slower when pretenured). - -json_pipeline: 200k 2.40 → 1.54 s (peak RSS 543 → 431 MB), 500k ~8 → 5.4 s -(RSS −173 MB), output hash identical; bytes copied by minors 108 → 0.0 MB. -Mechanism is IR-verified: exactly one pretenured call site in the workload, -zero in the numeric-push and function-region benches. +The originally proposed born-tenured allocation was removed after audit: +json_pipeline's minor-moved cohort is the runtime-allocated parse tree +(~113 MB), not codegen-visible literals (~1 MB live at minor time), and the +PR's measured win was a confound between arms that differed in whether +#7613's promote-on-first-copy seed fired. The deferred-page-registration +finding is extracted separately. Next routes for #7598: dynamic feedback or +allocation-context pretenure inside the JSON materialiser. diff --git a/crates/perry-codegen/src/codegen/closure.rs b/crates/perry-codegen/src/codegen/closure.rs index cf8c614c94..eb3e36901e 100644 --- a/crates/perry-codegen/src/codegen/closure.rs +++ b/crates/perry-codegen/src/codegen/closure.rs @@ -893,7 +893,6 @@ pub(super) fn compile_closure( current_block: 0, discard_expr_value: false, discard_this_expr: false, - pretenure_next_object_literal: false, func_names, strings, loop_targets: Vec::new(), diff --git a/crates/perry-codegen/src/codegen/entry.rs b/crates/perry-codegen/src/codegen/entry.rs index 8ac1ccd486..631a7235a2 100644 --- a/crates/perry-codegen/src/codegen/entry.rs +++ b/crates/perry-codegen/src/codegen/entry.rs @@ -709,7 +709,6 @@ pub(super) fn compile_module_entry( current_block: 0, discard_expr_value: false, discard_this_expr: false, - pretenure_next_object_literal: false, func_names, strings, loop_targets: Vec::new(), @@ -1378,7 +1377,6 @@ pub(super) fn compile_module_entry( current_block: 0, discard_expr_value: false, discard_this_expr: false, - pretenure_next_object_literal: false, func_names, strings, loop_targets: Vec::new(), diff --git a/crates/perry-codegen/src/codegen/function.rs b/crates/perry-codegen/src/codegen/function.rs index 338fb01b0d..5b43995b99 100644 --- a/crates/perry-codegen/src/codegen/function.rs +++ b/crates/perry-codegen/src/codegen/function.rs @@ -696,7 +696,6 @@ pub(super) fn compile_function( current_block: 0, discard_expr_value: false, discard_this_expr: false, - pretenure_next_object_literal: false, func_names, strings, loop_targets: Vec::new(), diff --git a/crates/perry-codegen/src/codegen/method.rs b/crates/perry-codegen/src/codegen/method.rs index 288e544389..4ba7d549a3 100644 --- a/crates/perry-codegen/src/codegen/method.rs +++ b/crates/perry-codegen/src/codegen/method.rs @@ -431,7 +431,6 @@ pub(super) fn compile_method( current_block: 0, discard_expr_value: false, discard_this_expr: false, - pretenure_next_object_literal: false, func_names, strings, loop_targets: Vec::new(), @@ -1490,7 +1489,6 @@ pub(super) fn compile_static_method( current_block: 0, discard_expr_value: false, discard_this_expr: false, - pretenure_next_object_literal: false, func_names, strings, loop_targets: Vec::new(), diff --git a/crates/perry-codegen/src/collectors/hir_facts.rs b/crates/perry-codegen/src/collectors/hir_facts.rs index be2d212097..da64c82340 100644 --- a/crates/perry-codegen/src/collectors/hir_facts.rs +++ b/crates/perry-codegen/src/collectors/hir_facts.rs @@ -1837,6 +1837,50 @@ mod tests { } } + /// #7598: the run-once region gate is what makes `pretenure_accumulator` + /// a cohort-lifetime fact rather than a shape fact — a function region's + /// accumulator is re-entered per call and its cohort dies at return + /// (measured 6.6x slower when pretenured). The admission machinery is + /// retained for a future dynamic-feedback pretenurer (see #7623's audit: + /// json_pipeline's moved cohort is runtime-allocated, so codegen-visible + /// literals were the wrong target); this test keeps the graph-level fact + /// live and pins the gate's direction at both polarities. + #[test] + fn pretenure_accumulator_fact_requires_a_run_once_region() { + let accumulator = Stmt::Let { + id: 1, + name: "out".into(), + ty: Type::Any, + mutable: false, + init: Some(Expr::Array(vec![])), + }; + let push_loop = Stmt::While { + condition: Expr::Bool(true), + body: vec![Stmt::Expr(Expr::ArrayPush { + array_id: 1, + value: Box::new(Expr::Object(vec![("v".to_string(), Expr::Integer(1))])), + })], + }; + let build = |region_runs_once: bool| { + collect_native_region_fact_graph( + &[accumulator.clone(), push_loop.clone()], + &[], + &HashSet::new(), + &HashSet::new(), + &HashSet::new(), + &HashSet::new(), + &HashMap::new(), + &HashMap::new(), + &HashMap::new(), + &HashMap::new(), + &crate::collectors::ModuleDispatchFacts::default(), + region_runs_once, + ) + }; + assert!(build(true).pretenure_accumulator(1)); + assert!(!build(false).pretenure_accumulator(1)); + } + fn const_number_let(id: u32, init: Expr) -> Stmt { Stmt::Let { id, diff --git a/crates/perry-codegen/src/expr/array_push.rs b/crates/perry-codegen/src/expr/array_push.rs index eb22d33b0b..be5eafe3c5 100644 --- a/crates/perry-codegen/src/expr/array_push.rs +++ b/crates/perry-codegen/src/expr/array_push.rs @@ -126,28 +126,8 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr, value_discarded: bool) -> let value_is_numeric = is_numeric_expr(ctx, value); let require_numeric_layout = value_is_numeric && expr_has_numeric_pointer_free_array_layout(ctx, &array_expr); - // #7598: an accumulator declared outside every loop and filled - // inside one keeps its cohort live for the rest of the loop, so - // the pushed value is born tenured — skipping the - // Eden→survivor→old double copy the copying minor would otherwise - // pay for it. `Expr::New` is the shape object literals actually - // reach codegen in (the AnonShape transform synthesizes a class); - // `Expr::Object` covers literals the transform leaves alone. The - // flag is taken by the value's own allocation lowering - // (`lower_object_literal` / `lower_call/new.rs`), so it reaches - // exactly the root allocation and never a nested one. - if matches!(value.as_ref(), Expr::Object(_) | Expr::New { .. }) - && ctx.native_facts.pretenure_accumulator(*array_id) - { - ctx.pretenure_next_object_literal = true; - } let (v, v_bits) = lower_array_push_value(ctx, value, layout_note_needed, write_barrier_needed)?; - // Consumed by whichever allocation tier the value took. Cleared - // unconditionally: tiers that allocate elsewhere (dynamic-parent - // classes, method-closure literals) never read it, and it must - // not leak past this push (#7590's leak class). - ctx.pretenure_next_object_literal = false; let arr_box = lower_expr(ctx, &array_expr)?; // Repsel 4a.1 (#6904 recon): the guarded numeric push was an diff --git a/crates/perry-codegen/src/expr/mod.rs b/crates/perry-codegen/src/expr/mod.rs index a2f18ffea6..0511a5b11a 100644 --- a/crates/perry-codegen/src/expr/mod.rs +++ b/crates/perry-codegen/src/expr/mod.rs @@ -250,15 +250,6 @@ pub(crate) struct FnCtx<'a> { /// reading the field, because they consult it *after* lowering their /// operands, by which point the field has been taken again. pub discard_this_expr: bool, - /// #7598: the next object literal lowered is a pretenure-accumulator push - /// value and should allocate born-tenured in old-gen. Set by - /// `array_push::lower` immediately before lowering the pushed value when - /// the target local is `native_facts.pretenure_accumulator`-admitted and - /// the value is a plain object literal; **taken** (`mem::take`) at the top - /// of `lower_object_literal`, so it reaches exactly the root literal — - /// nested literals in field initializers read `false` (the #7590 take - /// discipline). The push site asserts it was consumed after lowering. - pub pretenure_next_object_literal: bool, /// HIR FuncId → LLVM function name. Resolved at the top of /// `compile_module` so `FuncRef(id)` calls know what to emit. pub func_names: &'a std::collections::HashMap, diff --git a/crates/perry-codegen/src/expr/object_literal.rs b/crates/perry-codegen/src/expr/object_literal.rs index 36a363b3e7..43e61327d1 100644 --- a/crates/perry-codegen/src/expr/object_literal.rs +++ b/crates/perry-codegen/src/expr/object_literal.rs @@ -224,11 +224,6 @@ pub(crate) fn lower_object_literal( props: &[(String, Expr)], expected_ty: Option<&HirType>, ) -> Result { - // #7598: TAKEN here so it covers exactly this literal — field initializers - // containing nested literals lower below with the flag already cleared - // (the #7590 take discipline). Only the shaped fast path honors it; the - // by-name fallback (method closures) simply allocates young. - let pretenure = std::mem::take(&mut ctx.pretenure_next_object_literal); // #6951: the object handle is allocated BEFORE the property values are // lowered and lives in an SSA register across all of them. `{ a: s, b: f() }` // therefore had its half-built object swept by `f`'s collection, and the @@ -288,14 +283,9 @@ pub(crate) fn lower_object_literal( } let shape_id_str = shape_id.to_string(); - let alloc_fn = if pretenure { - "js_object_alloc_with_shape_pretenured" - } else { - "js_object_alloc_with_shape" - }; let obj_handle = ctx.block().call( I64, - alloc_fn, + "js_object_alloc_with_shape", &[ (I32, &shape_id_str), (I32, &n_str), diff --git a/crates/perry-codegen/src/lower_call/new.rs b/crates/perry-codegen/src/lower_call/new.rs index 706be020ad..021b0f43ee 100644 --- a/crates/perry-codegen/src/lower_call/new.rs +++ b/crates/perry-codegen/src/lower_call/new.rs @@ -625,14 +625,7 @@ fn lower_new_impl_inner( // NOTE the env test is `is_none()`: `PERRY_INLINE_NEW=""` *enables* // the inline path, because an empty string is `Some("")`. let force_inline_new = std::env::var_os("PERRY_INLINE_NEW").is_some(); - // #7598: a pretenure-accumulator push value is born TENURED in - // old-gen via the outlined call, never the inline Eden bump — the - // ~140-cycle call it re-pays is noise against the Eden→survivor→old - // double copy it removes for a cohort that is live for the rest of - // the loop by construction. Taken here so it covers exactly this - // allocation; the constructor's own inner allocations read `false`. - let pretenure = std::mem::take(&mut ctx.pretenure_next_object_literal); - if pretenure || (!force_inline_new && !new_site_is_in_loop(ctx)) { + if !force_inline_new && !new_site_is_in_loop(ctx) { let keys_slot = if let Some(s) = ctx.class_keys_slots.get(class_name).cloned() { s } else { @@ -642,16 +635,14 @@ fn lower_new_impl_inner( s }; let keys_ptr = ctx.block().load(I64, &keys_slot); - let alloc_fn = if pretenure { - "js_object_alloc_class_inline_keys_pretenured" - } else { - "js_object_alloc_class_inline_keys" - }; - ctx.pending_declares - .push((alloc_fn.to_string(), I64, vec![I32, I32, I32, I64])); + ctx.pending_declares.push(( + "js_object_alloc_class_inline_keys".to_string(), + I64, + vec![I32, I32, I32, I64], + )); ctx.block().call( I64, - alloc_fn, + "js_object_alloc_class_inline_keys", &[ (I32, &cid_str), (I32, &parent_cid_str), diff --git a/crates/perry-codegen/src/runtime_decls/objects.rs b/crates/perry-codegen/src/runtime_decls/objects.rs index 27c2fcc6c5..d7cb0557af 100644 --- a/crates/perry-codegen/src/runtime_decls/objects.rs +++ b/crates/perry-codegen/src/runtime_decls/objects.rs @@ -72,12 +72,6 @@ pub fn declare_phase_b_objects(module: &mut LlModule) { // so subsequent field stores can use index-based set_field (skipping the // per-call linear key-search done by js_object_set_field_by_name). module.declare_function("js_object_alloc_with_shape", I64, &[I32, I32, PTR, I32]); - // #7598: born-tenured variant for pretenure-accumulator push literals. - module.declare_function( - "js_object_alloc_with_shape_pretenured", - I64, - &[I32, I32, PTR, I32], - ); // Index-based field setter (no key lookup). Hot-path target for object // literals with statically-known keys; the i-th field directly maps to // the i-th packed-keys entry above. diff --git a/crates/perry-runtime/src/arena/allocators.rs b/crates/perry-runtime/src/arena/allocators.rs index fec15392e7..1270929140 100644 --- a/crates/perry-runtime/src/arena/allocators.rs +++ b/crates/perry-runtime/src/arena/allocators.rs @@ -154,47 +154,6 @@ pub fn arena_alloc_gc_old(size: usize, align: usize, obj_type: u8) -> *mut u8 { unsafe { raw.add(GC_HEADER_SIZE) } } -/// #7598: the born-tenured allocator for the PRETENURE-ACCUMULATOR hot path — -/// a per-object rate the ordinary old-gen path was never built for. -/// Two deliberate departures from `arena_alloc_gc_old`, both measured to be -/// the difference between winning and losing the pretenure trade: -/// -/// - **No `old_free_take_exact` probe.** Hole reuse is an anti-growth -/// mechanism for occasional promotions; on a 100k-object burst the probe is -/// pure per-allocation overhead, and burst cohorts live and die together — -/// contiguous bump placement is what old-page defrag wants from them anyway. -/// - **Deferred page registration.** `register_old_object_pages` per object -/// pays two RefCell borrows, two Vec allocations, and a linear dedup scan of -/// the page's object list (quadratic as a page fills). Every reader of that -/// index runs at GC time, so the burst defers into -/// `defer_old_object_page_registration` and `old_pages_begin_gc_cycle` -/// flushes before any collector work reads it. -/// -/// Header init is identical to `arena_alloc_gc_old` + `GC_FLAG_TENURED` in -/// the same breath — the `Old ⟹ TENURED` contract (#7602) holds here exactly -/// as for the wrapper below. -pub(crate) fn arena_alloc_gc_old_born_tenured_bump( - size: usize, - align: usize, - obj_type: u8, -) -> *mut u8 { - use crate::gc::{GcHeader, GC_FLAG_ARENA, GC_FLAG_TENURED, GC_HEADER_SIZE}; - - let pad = align.max(8); - let total = (GC_HEADER_SIZE + size + pad - 1) & !(pad - 1); - let raw = arena_alloc_old(total, align); - unsafe { - let header = raw as *mut GcHeader; - (*header).obj_type = obj_type; - (*header).gc_flags = GC_FLAG_ARENA | GC_FLAG_TENURED | crate::gc::gc_birth_extra_flags(); - crate::gc::gc_note_black_birth(header); - (*header)._reserved = 0; - (*header).size = total as u32; - } - super::page_meta::defer_old_object_page_registration(raw as usize, total); - unsafe { raw.add(GC_HEADER_SIZE) } -} - /// The old-gen + born-tenured shape `arena_alloc_gc` hands a LARGE object, for /// a caller that wants it on size-independent grounds. /// diff --git a/crates/perry-runtime/src/arena/mod.rs b/crates/perry-runtime/src/arena/mod.rs index 58ee661838..04027c5551 100644 --- a/crates/perry-runtime/src/arena/mod.rs +++ b/crates/perry-runtime/src/arena/mod.rs @@ -63,8 +63,7 @@ pub use allocators::{ arena_alloc_longlived, arena_alloc_old, js_arena_alloc, }; pub(crate) use allocators::{ - arena_alloc_gc_old_born_tenured, arena_alloc_gc_old_born_tenured_bump, - arena_alloc_gc_old_excluding_pages, arena_alloc_gc_survivor, + arena_alloc_gc_old_born_tenured, arena_alloc_gc_old_excluding_pages, arena_alloc_gc_survivor, }; // walk.rs diff --git a/crates/perry-runtime/src/arena/page_meta.rs b/crates/perry-runtime/src/arena/page_meta.rs index 6aa3ac9f9a..0b4653b17e 100644 --- a/crates/perry-runtime/src/arena/page_meta.rs +++ b/crates/perry-runtime/src/arena/page_meta.rs @@ -656,51 +656,6 @@ pub(crate) fn register_old_object_pages(header_addr: usize, total_size: usize) { update_old_page_meta_for_object(&added_pages, true); } -thread_local! { - /// #7598: page registrations deferred by the born-tenured BUMP allocator. - /// `register_old_object_pages` per allocation is the measured killer of - /// site-level pretenuring — per object it pays two `RefCell` borrows, two - /// `Vec` allocations, and a **linear `contains` scan of the page's object - /// list** (O(objects-per-page)² as a page fills; the dedup exists because - /// hole reuse can re-register an address that `unregister` never removed). - /// Every reader of the page-objects index runs at GC time (defrag page - /// selection, sweep accounting), so registration only has to be visible by - /// cycle start — `old_pages_begin_gc_cycle` flushes this buffer, and a - /// size cap bounds it between cycles. - static DEFERRED_OLD_PAGE_REGISTRATIONS: std::cell::RefCell> = - const { std::cell::RefCell::new(Vec::new()) }; -} - -/// Cap chosen so the buffer's worst-case footprint (16 B/entry × 64k = 1 MB) -/// stays a rounding error while flushes stay rare on allocation bursts. -const DEFERRED_OLD_PAGE_REGISTRATION_CAP: usize = 65_536; - -/// #7598: defer this object's page registration to the next flush. ONLY for -/// freshly bump-allocated born-tenured objects — the deferred entry relies on -/// the full `register_old_object_pages` (with its hole-reuse dedup) running at -/// flush time, it just runs it off the allocation hot path. -pub(crate) fn defer_old_object_page_registration(header_addr: usize, total_size: usize) { - let flush_now = DEFERRED_OLD_PAGE_REGISTRATIONS.with(|buf| { - let mut buf = buf.borrow_mut(); - buf.push((header_addr, total_size)); - buf.len() >= DEFERRED_OLD_PAGE_REGISTRATION_CAP - }); - if flush_now { - flush_deferred_old_page_registrations(); - } -} - -/// Drain the deferred buffer through the real registration path. Called from -/// `old_pages_begin_gc_cycle` (every collection begins with an accurate -/// index) and from the size-cap overflow in `defer_old_object_page_registration`. -pub(crate) fn flush_deferred_old_page_registrations() { - let pending = - DEFERRED_OLD_PAGE_REGISTRATIONS.with(|buf| std::mem::take(&mut *buf.borrow_mut())); - for (header_addr, total_size) in pending { - register_old_object_pages(header_addr, total_size); - } -} - #[allow(dead_code)] pub(crate) fn unregister_old_object_pages(header_addr: usize, total_size: usize) { if header_addr == 0 || total_size == 0 { @@ -728,9 +683,6 @@ pub(crate) fn unregister_old_object_pages(header_addr: usize, total_size: usize) } pub(crate) fn old_pages_begin_gc_cycle() { - // #7598: born-tenured bump allocations defer their page registration; - // every collection must begin with an accurate page-objects index. - flush_deferred_old_page_registrations(); // #6181: the per-page `dirty_slots` reset used to iterate every old page // here (O(old pages) on every minor, growing with old-gen size). It is now // a single epoch bump — a page whose `dirty_slots_epoch` predates the new diff --git a/crates/perry-runtime/src/object/alloc.rs b/crates/perry-runtime/src/object/alloc.rs index bb95015a7e..d7af244b9e 100644 --- a/crates/perry-runtime/src/object/alloc.rs +++ b/crates/perry-runtime/src/object/alloc.rs @@ -244,50 +244,6 @@ pub extern "C" fn js_object_alloc_class_inline_keys( parent_class_id: u32, field_count: u32, keys_array: *mut ArrayHeader, -) -> *mut ObjectHeader { - alloc_class_inline_keys_impl(class_id, parent_class_id, field_count, keys_array, false) -} - -/// #7598: `js_object_alloc_class_inline_keys` for an allocation site codegen -/// proved to feed a long-lived accumulator (`out = []` outside every loop, -/// filled by `out.push(...)` inside one — object literals arrive here as -/// synthesized AnonShape classes). The instance is born in old-gen with -/// `GC_FLAG_TENURED`, so the copying minor never pays the -/// Eden→survivor→old double copy its cohort was measured to cost (#7592: -/// 3.9 s of a 5.1 s phase). Constructor field stores need no special casing: -/// they funnel through `runtime_store_jsvalue_slot` / the #7602-gated -/// barrier, both of which read the LIVE parent header and remember old→young -/// edges exactly as for a promoted object. The `Old ⟹ TENURED` invariant -/// holds by construction: `arena_alloc_gc_old_born_tenured` sets the bit -/// itself and is pinned by `every_old_gen_birth_path_sets_tenured`. -#[no_mangle] -pub extern "C" fn js_object_alloc_class_inline_keys_pretenured( - class_id: u32, - parent_class_id: u32, - field_count: u32, - keys_array: *mut ArrayHeader, -) -> *mut ObjectHeader { - alloc_class_inline_keys_impl(class_id, parent_class_id, field_count, keys_array, true) -} - -/// Keepalive anchor — `js_object_alloc_class_inline_keys_pretenured` is a -/// generated-code-only callee, so the auto-optimize whole-program build would -/// otherwise dead-strip it (see the FFI-symbol-link-break class). -#[cfg(feature = "keepalive-anchors")] -#[used] -static KEEP_JS_OBJECT_ALLOC_CLASS_INLINE_KEYS_PRETENURED: extern "C" fn( - u32, - u32, - u32, - *mut ArrayHeader, -) -> *mut ObjectHeader = js_object_alloc_class_inline_keys_pretenured; - -fn alloc_class_inline_keys_impl( - class_id: u32, - parent_class_id: u32, - field_count: u32, - keys_array: *mut ArrayHeader, - pretenure: bool, ) -> *mut ObjectHeader { if parent_class_id != 0 { register_class(class_id, parent_class_id); @@ -307,14 +263,7 @@ fn alloc_class_inline_keys_impl( let fields_size = alloc_field_count * std::mem::size_of::(); let total_size = header_size + fields_size; - // Both allocators take the PAYLOAD size and pad the GcHeader in - // themselves; the branches differ only in which generation a small object - // is born into. - let ptr = if pretenure { - crate::arena::arena_alloc_gc_old_born_tenured_bump(total_size, 8, crate::gc::GC_TYPE_OBJECT) - } else { - arena_alloc_gc(total_size, 8, crate::gc::GC_TYPE_OBJECT) - } as *mut ObjectHeader; + let ptr = arena_alloc_gc(total_size, 8, crate::gc::GC_TYPE_OBJECT) as *mut ObjectHeader; unsafe { (*ptr).object_type = crate::error::OBJECT_TYPE_REGULAR; @@ -658,66 +607,13 @@ pub extern "C" fn js_object_alloc_with_shape( field_count: u32, packed_keys: *const u8, packed_keys_len: u32, -) -> *mut ObjectHeader { - alloc_with_shape_impl(shape_id, field_count, packed_keys, packed_keys_len, false) -} - -/// #7598: `js_object_alloc_with_shape` for an allocation site codegen proved -/// to feed a long-lived accumulator (`out = []` outside every loop, filled by -/// `out.push({...})` inside one). The object is born in old-gen with -/// `GC_FLAG_TENURED`, so the copying minor never pays the Eden→survivor→old -/// double copy its cohort was measured to cost (#7592: 3.9 s of a 5.1 s -/// phase). Field stores need no special casing: they funnel through -/// `runtime_store_jsvalue_slot`, whose write barrier reads the LIVE parent -/// header — a born-tenured parent takes the old-parent path and remembers -/// old→young field edges exactly as a promoted object would. -/// -/// The `Old ⟹ TENURED` invariant #7602's gate rests on holds by construction -/// here: `arena_alloc_gc_old_born_tenured` sets the bit itself and is pinned -/// by `every_old_gen_birth_path_sets_tenured`. -#[no_mangle] -pub extern "C" fn js_object_alloc_with_shape_pretenured( - shape_id: u32, - field_count: u32, - packed_keys: *const u8, - packed_keys_len: u32, -) -> *mut ObjectHeader { - alloc_with_shape_impl(shape_id, field_count, packed_keys, packed_keys_len, true) -} - -/// Keepalive anchor — `js_object_alloc_with_shape_pretenured` is a -/// generated-code-only callee, so the auto-optimize whole-program build would -/// otherwise dead-strip it (see the FFI-symbol-link-break class). -#[cfg(feature = "keepalive-anchors")] -#[used] -static KEEP_JS_OBJECT_ALLOC_WITH_SHAPE_PRETENURED: extern "C" fn( - u32, - u32, - *const u8, - u32, -) -> *mut ObjectHeader = js_object_alloc_with_shape_pretenured; - -fn alloc_with_shape_impl( - shape_id: u32, - field_count: u32, - packed_keys: *const u8, - packed_keys_len: u32, - pretenure: bool, ) -> *mut ObjectHeader { let header_size = std::mem::size_of::(); // Allocate extra field slots for dynamic property growth (plain objects may get new fields) let alloc_field_count = std::cmp::max(field_count as usize, crate::object::INLINE_SLOT_FLOOR); let fields_size = alloc_field_count * 8; let total_size = header_size + fields_size; - // Both allocators take the PAYLOAD size and pad the GcHeader in themselves; - // `arena_alloc_gc`'s own large-object arm is exactly the born-tenured - // shape, so the two branches differ only in which generation a small - // object is born into. - let obj_ptr = if pretenure { - crate::arena::arena_alloc_gc_old_born_tenured_bump(total_size, 8, crate::gc::GC_TYPE_OBJECT) - } else { - arena_alloc_gc(total_size, 8, crate::gc::GC_TYPE_OBJECT) - } as *mut ObjectHeader; + let obj_ptr = arena_alloc_gc(total_size, 8, crate::gc::GC_TYPE_OBJECT) as *mut ObjectHeader; unsafe { (*obj_ptr).object_type = crate::error::OBJECT_TYPE_REGULAR;