cranelift: per-table mutability tracking + call_indirect elisions on immutable funcref tables

crates/cranelift/src/func_environ.rs

@@ -1855,7 +1855,12 @@ impl FuncEnvironment<'_> {
             self.reference_type(table.ref_type.heap_type).0.bytes()
         };
 
-        let base_flags = if Some(table.limits.min) == table.limits.max {
+        // A table is fixed-size if min == max or if translation proved it
+        // is never mutated; either way the base address and element count
+        // are constant for the instance's lifetime.
+        let fixed_size =
+            !self.translation.tables_mutated[index] || Some(table.limits.min) == table.limits.max;
+        let base_flags = if fixed_size {
             func.dfg
                 .mem_flags
                 .insert(MemFlagsData::trusted().with_readonly().with_can_move())
@@ -1867,11 +1872,10 @@ impl FuncEnvironment<'_> {
             base: ptr,
             offset: Offset32::new(base_offset),
             global_type: pointer_type,
-            // A fixed-size table can't be resized so its base address won't change.
             flags: base_flags,
         });
 
-        let bound = if Some(table.limits.min) == table.limits.max {
+        let bound = if fixed_size {
             TableSize::Static {
                 bound: table.limits.min,
             }
@@ -2159,6 +2163,14 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
         callee: ir::Value,
         call_args: &[ir::Value],
     ) -> WasmResult<Option<CallRets>> {
+        // Fast path: if we can statically resolve this indirect call to a
+        // single defined function (immutable funcref table + constant
+        // callee index + matching signature), emit a direct call instead.
+        // See `try_static_resolve_indirect_call`.
+        if let Some(target) = self.try_static_resolve_indirect_call(table_index, ty_index, callee) {
+            return self.direct_call(target, sig_ref, call_args).map(Some);
+        }
+
         let (code_ptr, callee_vmctx) = match self.check_and_load_code_and_callee_vmctx(
             table_index,
             ty_index,
@@ -2173,6 +2185,198 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
             .map(Some)
     }
 
+    /// Try to statically resolve a `call_indirect` site to a single defined
+    /// function so the call can be lowered as a direct call.
+    ///
+    /// All four of these must hold for the resolution to succeed:
+    ///
+    /// 1. The target table must be provably immutable for the lifetime of
+    ///    any instance of this module: defined (not imported) and never the
+    ///    target of `table.set` / `table.fill` / `table.copy` (as the dst)
+    ///    / `table.grow` / `table.init`. This is the `tables_mutated` bit
+    ///    populated in `ModuleEnvironment::translate`.
+    ///
+    /// 2. The callee index value (the operand to `call_indirect`) must be a
+    ///    compile-time constant — i.e., the wasm did `i32.const N;
+    ///    call_indirect (table $t) (type $sig)`. This is what hand-lowered
+    ///    C++/Rust vtable calls and AOT-compiled JS-to-wasm dispatch tables
+    ///    look like in practice.
+    ///
+    /// 3. The slot at index `N` in the table must be precomputable from
+    ///    static `elem` segments: `module.table_initialization
+    ///    .initial_values[defined_index]` must be `TableInitialValue::Null
+    ///    { precomputed }` (i.e., not a fully-dynamic `Expr`-style init),
+    ///    and the index `N` must be in range and resolved to a concrete
+    ///    `FuncIndex` (not the reserved-value sentinel).
+    ///
+    /// 4. The function's signature in the module's interned type table
+    ///    must equal the `ty_index` declared by the `call_indirect` site.
+    ///    Otherwise the original semantics are "trap on signature
+    ///    mismatch", which we don't want to replace with a static direct
+    ///    call.
+    ///
+    /// Returns the resolved function on success, `None` otherwise (in
+    /// which case the caller falls back to a normal indirect call).
+    fn try_static_resolve_indirect_call(
+        &self,
+        table_index: TableIndex,
+        ty_index: TypeIndex,
+        callee: ir::Value,
+    ) -> Option<FuncIndex> {
+        let translation = self.env.translation;
+        let module = &translation.module;
+
+        // (1) Table must be provably immutable. Imported tables are
+        //     pre-marked as mutated in `ModuleEnvironment::translate`, so
+        //     this check also rules them out (along with the explicit
+        //     `defined_table_index` check below for clarity).
+        if translation.tables_mutated[table_index] {
+            return None;
+        }
+        let defined_table = module.defined_table_index(table_index)?;
+
+        // (2) Callee must be a constant `iconst`. Pattern adapted from
+        //     `bounds_checks::statically_known_in_bounds`.
+        let dfg = &self.builder.func.dfg;
+        let inst = dfg.value_def(callee).inst()?;
+        let imm = match dfg.insts[inst] {
+            ir::InstructionData::UnaryImm {
+                opcode: ir::Opcode::Iconst,
+                imm,
+            } => imm,
+            _ => return None,
+        };
+        let callee_ty = dfg.value_type(callee);
+        let callee_idx_u64 = imm
+            .zero_extend_from_width(callee_ty.bits())
+            .bits()
+            .cast_unsigned();
+
+        // (3) Slot must be precomputable from the static funcref image.
+        let precomputed = module.table_initialization.get(defined_table)?;
+        let slot = usize::try_from(callee_idx_u64).ok()?;
+        if slot >= precomputed.len() {
+            return None;
+        }
+        let target = precomputed[slot];
+        // `FuncIndex::reserved_value()` marks a null (uncovered) slot.
+        if target.is_reserved_value() {
+            return None;
+        }
+
+        // (4) Signature match. The site's declared `ty_index` and the
+        //     target function's declared signature must intern to the same
+        //     module type index.
+        let expected_ty = module.types[ty_index].unwrap_module_type_index();
+        let target_ty = module.functions[target]
+            .signature
+            .unwrap_module_type_index();
+        if expected_ty != target_ty {
+            return None;
+        }
+
+        Some(target)
+    }
+
+    /// Try to prove that the runtime signature check at a `call_indirect`
+    /// site through an untyped `funcref` table is redundant.
+    ///
+    /// True when:
+    ///
+    /// 1. The table is provably immutable (`tables_mutated[table_index] ==
+    ///    false`). Defined-not-imported is implied since imported tables
+    ///    are pre-marked as mutated.
+    ///
+    /// 2. The table is precomputable from static `elem` segments
+    ///    (`TableInitialValue::Null { precomputed }`).
+    ///
+    /// 3. Every non-null entry in `precomputed` has the same module-
+    ///    interned signature as the `ty_index` declared at the call site.
+    ///    Null slots are fine — they trap on the funcref-NULL load that
+    ///    happens after sig-check elision.
+    ///
+    /// When this returns true, the caller short-circuits to
+    /// `CheckIndirectCallTypeSignature::StaticMatch`, which removes the
+    /// sig load + compare from the hot path. Bounds-check on the table
+    /// index and the funcref-NULL check are still emitted by the
+    /// surrounding code, so the call still traps correctly on OOB or
+    /// null index — only the sig check is elided.
+    ///
+    /// This is the static analog of an inline-cache: instead of caching
+    /// the resolved target per call site, we observe at module-load that
+    /// the table contents make the sig check uninformative for the
+    /// lifetime of any instance.
+    /// True iff every slot in the precomputed `elem`-segment contents for
+    /// `table_index` is a concrete `FuncIndex` (no
+    /// `FuncIndex::reserved_value()` "no-entry" sentinel).
+    ///
+    /// Caller has already proven the table is immutable, so the contents
+    /// observed here are stable for the lifetime of any instance —
+    /// `false` here implies "no slot is ever null at runtime."
+    ///
+    /// When this is true, the runtime funcref-NULL check on the loaded
+    /// funcref pointer is provably redundant: any in-bounds index leads
+    /// to a non-null funcref. The bounds check still runs (so an
+    /// out-of-bounds index traps as before with `TRAP_TABLE_OUT_OF_BOUNDS`).
+    fn precomputed_table_has_no_null_slots(&self, table_index: TableIndex) -> bool {
+        let module = &self.env.translation.module;
+        let Some(defined_table) = module.defined_table_index(table_index) else {
+            return false;
+        };
+        let Some(precomputed) = module.table_initialization.get(defined_table) else {
+            return false;
+        };
+        if precomputed.is_empty() {
+            return false;
+        }
+        // Slots beyond `precomputed.len()` are null at runtime; coverage
+        // up to `limits.min` is required (caller proved immutable, so the
+        // table can't grow beyond min).
+        let table_min = module.tables[table_index].limits.min;
+        if (precomputed.len() as u64) < table_min {
+            return false;
+        }
+        precomputed.iter().all(|f| !f.is_reserved_value())
+    }
+
+    fn try_elide_sig_check_for_immutable_table(
+        &self,
+        table_index: TableIndex,
+        ty_index: TypeIndex,
+    ) -> bool {
+        let translation = self.env.translation;
+        let module = &translation.module;
+
+        if translation.tables_mutated[table_index] {
+            return false;
+        }
+        let defined_table = match module.defined_table_index(table_index) {
+            Some(d) => d,
+            None => return false,
+        };
+
+        let precomputed = match module.table_initialization.get(defined_table) {
+            Some(p) if !p.is_empty() => p,
+            _ => return false,
+        };
+
+        let expected_ty = module.types[ty_index].unwrap_module_type_index();
+        for &func_idx in precomputed.iter() {
+            // Null slots will trap on the funcref-NULL load anyway.
+            if func_idx.is_reserved_value() {
+                continue;
+            }
+            let actual_ty = module.functions[func_idx]
+                .signature
+                .unwrap_module_type_index();
+            if actual_ty != expected_ty {
+                return false;
+            }
+        }
+
+        true
+    }
+
     fn check_and_load_code_and_callee_vmctx(
         &mut self,
         table_index: TableIndex,
@@ -2230,6 +2434,34 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
         // table of typed functions and that type matches `ty_index`, then
         // there's no need to perform a typecheck.
         match table.ref_type.heap_type {
+            // Untyped `funcref` tables ordinarily need a runtime sig check.
+            // But if (a) the table is provably immutable (`tables_mutated`
+            // bit clear) and (b) every non-null entry in the precomputed
+            // static `elem` segments has the same `VMSharedTypeIndex` as
+            // the call site, then the runtime check is provably redundant
+            // and we can elide it the same way we do for typed-funcref
+            // tables.
+            //
+            // This is the AOT-IC-seeding analog: instead of caching the
+            // resolved target at the call site, we cache the *signature*
+            // at module-load time and skip the hot-path sig load+compare.
+            // Helps the megamorphic case (computed `call_indirect` index)
+            // that the static-monomorphization fast path above can't
+            // handle.
+            WasmHeapType::Func
+                if self.try_elide_sig_check_for_immutable_table(table_index, ty_index) =>
+            {
+                // If we additionally know every entry in the precomputed
+                // table is non-null, lower `may_be_null` to false so the
+                // downstream funcref-NULL check is also elided. This is
+                // only sound if the table can't be grown or have its
+                // entries cleared after init (i.e., immutable, which we
+                // already proved above).
+                let may_be_null = table.ref_type.nullable
+                    && !self.precomputed_table_has_no_null_slots(table_index);
+                return CheckIndirectCallTypeSignature::StaticMatch { may_be_null };
+            }
+
             // Functions do not have a statically known type in the table, a
             // typecheck is required. Fall through to below to perform the
             // actual typecheck.