diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 8bd81a6b46db..484c1060767a 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -1855,7 +1855,12 @@ impl FuncEnvironment<'_> { self.reference_type(table.ref_type.heap_type).0.bytes() }; - let base_flags = if Some(table.limits.min) == table.limits.max { + // A table is fixed-size if min == max or if translation proved it + // is never mutated; either way the base address and element count + // are constant for the instance's lifetime. + let fixed_size = + !self.translation.tables_mutated[index] || Some(table.limits.min) == table.limits.max; + let base_flags = if fixed_size { func.dfg .mem_flags .insert(MemFlagsData::trusted().with_readonly().with_can_move()) @@ -1867,11 +1872,10 @@ impl FuncEnvironment<'_> { base: ptr, offset: Offset32::new(base_offset), global_type: pointer_type, - // A fixed-size table can't be resized so its base address won't change. flags: base_flags, }); - let bound = if Some(table.limits.min) == table.limits.max { + let bound = if fixed_size { TableSize::Static { bound: table.limits.min, } @@ -2159,6 +2163,14 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> { callee: ir::Value, call_args: &[ir::Value], ) -> WasmResult> { + // Fast path: if we can statically resolve this indirect call to a + // single defined function (immutable funcref table + constant + // callee index + matching signature), emit a direct call instead. + // See `try_static_resolve_indirect_call`. + if let Some(target) = self.try_static_resolve_indirect_call(table_index, ty_index, callee) { + return self.direct_call(target, sig_ref, call_args).map(Some); + } + let (code_ptr, callee_vmctx) = match self.check_and_load_code_and_callee_vmctx( table_index, ty_index, @@ -2173,6 +2185,198 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> { .map(Some) } + /// Try to statically resolve a `call_indirect` site to a single defined + /// function so the call can be lowered as a direct call. + /// + /// All four of these must hold for the resolution to succeed: + /// + /// 1. The target table must be provably immutable for the lifetime of + /// any instance of this module: defined (not imported) and never the + /// target of `table.set` / `table.fill` / `table.copy` (as the dst) + /// / `table.grow` / `table.init`. This is the `tables_mutated` bit + /// populated in `ModuleEnvironment::translate`. + /// + /// 2. The callee index value (the operand to `call_indirect`) must be a + /// compile-time constant — i.e., the wasm did `i32.const N; + /// call_indirect (table $t) (type $sig)`. This is what hand-lowered + /// C++/Rust vtable calls and AOT-compiled JS-to-wasm dispatch tables + /// look like in practice. + /// + /// 3. The slot at index `N` in the table must be precomputable from + /// static `elem` segments: `module.table_initialization + /// .initial_values[defined_index]` must be `TableInitialValue::Null + /// { precomputed }` (i.e., not a fully-dynamic `Expr`-style init), + /// and the index `N` must be in range and resolved to a concrete + /// `FuncIndex` (not the reserved-value sentinel). + /// + /// 4. The function's signature in the module's interned type table + /// must equal the `ty_index` declared by the `call_indirect` site. + /// Otherwise the original semantics are "trap on signature + /// mismatch", which we don't want to replace with a static direct + /// call. + /// + /// Returns the resolved function on success, `None` otherwise (in + /// which case the caller falls back to a normal indirect call). + fn try_static_resolve_indirect_call( + &self, + table_index: TableIndex, + ty_index: TypeIndex, + callee: ir::Value, + ) -> Option { + let translation = self.env.translation; + let module = &translation.module; + + // (1) Table must be provably immutable. Imported tables are + // pre-marked as mutated in `ModuleEnvironment::translate`, so + // this check also rules them out (along with the explicit + // `defined_table_index` check below for clarity). + if translation.tables_mutated[table_index] { + return None; + } + let defined_table = module.defined_table_index(table_index)?; + + // (2) Callee must be a constant `iconst`. Pattern adapted from + // `bounds_checks::statically_known_in_bounds`. + let dfg = &self.builder.func.dfg; + let inst = dfg.value_def(callee).inst()?; + let imm = match dfg.insts[inst] { + ir::InstructionData::UnaryImm { + opcode: ir::Opcode::Iconst, + imm, + } => imm, + _ => return None, + }; + let callee_ty = dfg.value_type(callee); + let callee_idx_u64 = imm + .zero_extend_from_width(callee_ty.bits()) + .bits() + .cast_unsigned(); + + // (3) Slot must be precomputable from the static funcref image. + let precomputed = module.table_initialization.get(defined_table)?; + let slot = usize::try_from(callee_idx_u64).ok()?; + if slot >= precomputed.len() { + return None; + } + let target = precomputed[slot]; + // `FuncIndex::reserved_value()` marks a null (uncovered) slot. + if target.is_reserved_value() { + return None; + } + + // (4) Signature match. The site's declared `ty_index` and the + // target function's declared signature must intern to the same + // module type index. + let expected_ty = module.types[ty_index].unwrap_module_type_index(); + let target_ty = module.functions[target] + .signature + .unwrap_module_type_index(); + if expected_ty != target_ty { + return None; + } + + Some(target) + } + + /// Try to prove that the runtime signature check at a `call_indirect` + /// site through an untyped `funcref` table is redundant. + /// + /// True when: + /// + /// 1. The table is provably immutable (`tables_mutated[table_index] == + /// false`). Defined-not-imported is implied since imported tables + /// are pre-marked as mutated. + /// + /// 2. The table is precomputable from static `elem` segments + /// (`TableInitialValue::Null { precomputed }`). + /// + /// 3. Every non-null entry in `precomputed` has the same module- + /// interned signature as the `ty_index` declared at the call site. + /// Null slots are fine — they trap on the funcref-NULL load that + /// happens after sig-check elision. + /// + /// When this returns true, the caller short-circuits to + /// `CheckIndirectCallTypeSignature::StaticMatch`, which removes the + /// sig load + compare from the hot path. Bounds-check on the table + /// index and the funcref-NULL check are still emitted by the + /// surrounding code, so the call still traps correctly on OOB or + /// null index — only the sig check is elided. + /// + /// This is the static analog of an inline-cache: instead of caching + /// the resolved target per call site, we observe at module-load that + /// the table contents make the sig check uninformative for the + /// lifetime of any instance. + /// True iff every slot in the precomputed `elem`-segment contents for + /// `table_index` is a concrete `FuncIndex` (no + /// `FuncIndex::reserved_value()` "no-entry" sentinel). + /// + /// Caller has already proven the table is immutable, so the contents + /// observed here are stable for the lifetime of any instance — + /// `false` here implies "no slot is ever null at runtime." + /// + /// When this is true, the runtime funcref-NULL check on the loaded + /// funcref pointer is provably redundant: any in-bounds index leads + /// to a non-null funcref. The bounds check still runs (so an + /// out-of-bounds index traps as before with `TRAP_TABLE_OUT_OF_BOUNDS`). + fn precomputed_table_has_no_null_slots(&self, table_index: TableIndex) -> bool { + let module = &self.env.translation.module; + let Some(defined_table) = module.defined_table_index(table_index) else { + return false; + }; + let Some(precomputed) = module.table_initialization.get(defined_table) else { + return false; + }; + if precomputed.is_empty() { + return false; + } + // Slots beyond `precomputed.len()` are null at runtime; coverage + // up to `limits.min` is required (caller proved immutable, so the + // table can't grow beyond min). + let table_min = module.tables[table_index].limits.min; + if (precomputed.len() as u64) < table_min { + return false; + } + precomputed.iter().all(|f| !f.is_reserved_value()) + } + + fn try_elide_sig_check_for_immutable_table( + &self, + table_index: TableIndex, + ty_index: TypeIndex, + ) -> bool { + let translation = self.env.translation; + let module = &translation.module; + + if translation.tables_mutated[table_index] { + return false; + } + let defined_table = match module.defined_table_index(table_index) { + Some(d) => d, + None => return false, + }; + + let precomputed = match module.table_initialization.get(defined_table) { + Some(p) if !p.is_empty() => p, + _ => return false, + }; + + let expected_ty = module.types[ty_index].unwrap_module_type_index(); + for &func_idx in precomputed.iter() { + // Null slots will trap on the funcref-NULL load anyway. + if func_idx.is_reserved_value() { + continue; + } + let actual_ty = module.functions[func_idx] + .signature + .unwrap_module_type_index(); + if actual_ty != expected_ty { + return false; + } + } + + true + } + fn check_and_load_code_and_callee_vmctx( &mut self, table_index: TableIndex, @@ -2230,6 +2434,34 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> { // table of typed functions and that type matches `ty_index`, then // there's no need to perform a typecheck. match table.ref_type.heap_type { + // Untyped `funcref` tables ordinarily need a runtime sig check. + // But if (a) the table is provably immutable (`tables_mutated` + // bit clear) and (b) every non-null entry in the precomputed + // static `elem` segments has the same `VMSharedTypeIndex` as + // the call site, then the runtime check is provably redundant + // and we can elide it the same way we do for typed-funcref + // tables. + // + // This is the AOT-IC-seeding analog: instead of caching the + // resolved target at the call site, we cache the *signature* + // at module-load time and skip the hot-path sig load+compare. + // Helps the megamorphic case (computed `call_indirect` index) + // that the static-monomorphization fast path above can't + // handle. + WasmHeapType::Func + if self.try_elide_sig_check_for_immutable_table(table_index, ty_index) => + { + // If we additionally know every entry in the precomputed + // table is non-null, lower `may_be_null` to false so the + // downstream funcref-NULL check is also elided. This is + // only sound if the table can't be grown or have its + // entries cleared after init (i.e., immutable, which we + // already proved above). + let may_be_null = table.ref_type.nullable + && !self.precomputed_table_has_no_null_slots(table_index); + return CheckIndirectCallTypeSignature::StaticMatch { may_be_null }; + } + // Functions do not have a statically known type in the table, a // typecheck is required. Fall through to below to perform the // actual typecheck. diff --git a/crates/environ/src/compile/module_environ.rs b/crates/environ/src/compile/module_environ.rs index 542181e55fd6..192c090feda2 100644 --- a/crates/environ/src/compile/module_environ.rs +++ b/crates/environ/src/compile/module_environ.rs @@ -76,6 +76,26 @@ pub struct ModuleTranslation<'data> { /// trampolines for each of these signatures are required. pub exported_signatures: Vec, + /// Per-table flag indicating whether the table is ever mutated by any + /// function defined in this module via `table.set` / `table.fill` / + /// `table.copy` (as the destination) / `table.grow` / `table.init`. + /// + /// `false` (the default) means the table's contents are determined + /// entirely by its `elem` segments and any active initializer, and never + /// change at runtime — provably immutable for the lifetime of any + /// instance of this module. + /// + /// `true` means the contents can change at runtime (or the table is + /// imported, in which case we conservatively assume the importer + /// mutates it). + /// + /// This is groundwork for later passes that turn `call_indirect` + /// through provably-immutable function tables into direct calls when + /// the dispatched-to slot is statically known. Set during module + /// translation (see `analyze_table_mutability`); read by Cranelift + /// lowering and by Pulley AOT IC seeding. + pub tables_mutated: SecondaryMap, + /// DWARF debug information, if enabled, parsed from the module. pub debuginfo: DebugInfoData<'data>, @@ -193,6 +213,7 @@ impl<'data> ModuleTranslation<'data> { function_body_inputs: PrimaryMap::default(), known_imported_functions: SecondaryMap::default(), exported_signatures: Vec::default(), + tables_mutated: SecondaryMap::default(), debuginfo: DebugInfoData::default(), has_unparsed_debuginfo: false, data_align: None, @@ -315,6 +336,8 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> { self.translate_payload(payload?)?; } + analyze_table_mutability(&mut self.result)?; + Ok(self.result) } @@ -1548,3 +1571,85 @@ impl ModuleTranslation<'_> { self.module.startup = ModuleStartup::IfMemoriesNeedInit(ty); } } + +/// Walk every defined function body, recording in +/// `translation.tables_mutated` each table that is the destination of any +/// runtime mutation opcode (`table.set`, `table.fill`, `table.copy` as the +/// destination, `table.grow`, `table.init`). +/// +/// Imported tables are conservatively pre-marked as mutated since the +/// importer can mutate them in ways we can't see. Active `elem` segments +/// applied at instantiation time are NOT counted as mutations — they are +/// part of the table's *initial* state, not a runtime change. +/// +/// `elem.drop` drops a passive element segment but does not write to any +/// table directly, so it is intentionally not counted here. Conservatively, +/// any `table.init` from a passive segment marks the destination table as +/// mutated. +fn analyze_table_mutability<'data>( + translation: &mut ModuleTranslation<'data>, +) -> Result<()> { + // Resize the table-mutability map to cover every table in the module + // (imports + defined). `SecondaryMap` defaults to `false` for all + // unset entries, which is the correct "definitely-not-mutated" default + // for defined tables we haven't observed any mutations on yet. + let num_tables = translation.module.tables.len(); + if num_tables == 0 { + return Ok(()); + } + + // Mark all imported tables as mutated up front. The importer can + // mutate them in ways this module can't see, so the conservative + // assumption is that they are not stable across calls. + let num_imported = translation.module.num_imported_tables; + for i in 0..num_imported { + translation.tables_mutated[TableIndex::from_u32(i as u32)] = true; + } + + // Mark all *exported* tables as mutated as well. A host (or another + // instance importing the export) can call `Table::set` / + // `Table::grow` via the public wasmtime API on any exported table, + // and those mutations are not visible in this module's bytecode. + // The `call_indirect` optimizations that read this bit must + // therefore treat exported tables as conservatively non-stable. + for (_, entity_index) in &translation.module.exports { + if let EntityIndex::Table(table_index) = entity_index { + translation.tables_mutated[*table_index] = true; + } + } + + // Walk every defined function body and look for table-mutation opcodes. + // The cost is O(total opcodes), one extra pass on top of the validator; + // typical large modules (sqlite3 ~50K opcodes) take well under a + // millisecond. + for (_, body_data) in &translation.function_body_inputs { + let mut reader = body_data.body.get_operators_reader()?; + while !reader.eof() { + use wasmparser::Operator; + match reader.read()? { + Operator::TableSet { table } + | Operator::TableFill { table } + | Operator::TableGrow { table } => { + translation.tables_mutated[TableIndex::from_u32(table)] = true; + } + Operator::TableCopy { + dst_table, + src_table: _, + } => { + // `src_table` is read-only in `table.copy`; only the + // destination is mutated. + translation.tables_mutated[TableIndex::from_u32(dst_table)] = true; + } + Operator::TableInit { + table, + elem_index: _, + } => { + translation.tables_mutated[TableIndex::from_u32(table)] = true; + } + _ => {} + } + } + } + + Ok(()) +} diff --git a/crates/environ/tests/table_mutability.rs b/crates/environ/tests/table_mutability.rs new file mode 100644 index 000000000000..562966a708e4 --- /dev/null +++ b/crates/environ/tests/table_mutability.rs @@ -0,0 +1,307 @@ +//! Integration tests for `analyze_table_mutability` and the surrounding +//! precompute ordering invariants. +//! +//! The per-table mutability bit is the foundation of the `call_indirect` +//! optimizations in `crates/cranelift/src/func_environ.rs` +//! (constant-index direct call, sig-check elision, NULL elision, bound- +//! load elision). A false negative here — failing to mark a table as +//! mutated when it actually is — would silently turn correct calls into +//! incorrect direct calls or skip required runtime checks. A false +//! positive — marking an immutable table as mutated — is merely a missed +//! optimization. Pin the analysis behaviour with focused module-level +//! tests so any regression surfaces immediately, not after a downstream +//! optimization fires on a now-invalid premise. +//! +//! Test scenario inspiration drawn from comparable bugs in peer +//! interpreters that have shipped fixes for analogous IC-invalidation +//! mistakes: +//! +//! - **Luau** (`LOP_NAMECALL`): inline cache had to be invalidated on +//! `table.insert` / metatable change. Analogous wasm risk: `table.grow` +//! not invalidating an immutability proof, so see `table_grow_marks…`. +//! - **JavaScriptCore** (`ic_table`): inline-cache corruption from missed +//! shape transitions. Analogous risk: over-marking, e.g. `table.copy` +//! wrongly marking the SOURCE table as mutated would forbid downstream +//! optimizations on a perfectly read-only table. See +//! `table_copy_marks_destination_only_not_source`. +//! - **Hermes** (`HiddenClass` cache): property cache misses with +//! `Object.defineProperty`. Analogous risk: `table.init` (active- +//! segment init at runtime) being treated as a no-op rather than a +//! write. See `table_init_marks_destination`. +//! +//! Lives in `tests/` rather than as a `#[cfg(test)] mod` inside +//! `module_environ.rs` because the latter triggers a pre-existing +//! upstream compile failure in `key.rs` / `module_artifacts.rs` (their +//! `arbitrary::Arbitrary` derives are stale relative to the workspace's +//! pinned `arbitrary 1.4.2`). Integration tests build against the lib +//! as a normal dependency and so do not set `cfg(test)` on +//! `wasmtime-environ` itself. + +use wasmparser::{Parser, Validator, WasmFeatures}; +use wasmtime_environ::{ + ModuleEnvironment, ModuleTypesBuilder, StaticModuleIndex, TableIndex, Tunables, +}; + +/// Translate `wat` and return the resulting `tables_mutated` bits, in +/// table-index order. Helper to keep individual tests short. +fn translate_and_get_mutability(wat: &str) -> Vec { + let bytes = wat::parse_str(wat).expect("WAT parse failed"); + let tunables = Tunables::default_host(); + // WASM2 covers reference-types + bulk-memory, which is what every + // table-mutating opcode below needs (`table.set`, `table.fill`, + // `table.grow`, `table.copy`, `table.init`, `elem.drop`). + let features = WasmFeatures::WASM2; + let mut validator = Validator::new_with_features(features); + let mut types = ModuleTypesBuilder::new(&validator); + let env = ModuleEnvironment::new( + &tunables, + &mut validator, + &mut types, + StaticModuleIndex::from_u32(0), + ); + let parser = Parser::new(0); + let translation = env.translate(parser, &bytes).expect("translate failed"); + let n: u32 = translation.module.tables.len().try_into().unwrap(); + (0..n) + .map(|i| translation.tables_mutated[TableIndex::from_u32(i)]) + .collect() +} + +/// A table only used as the source of `call_indirect` and `table.get` is +/// provably immutable. (Both ops READ the table; neither writes it.) The +/// table is intentionally NOT exported — exported tables are +/// conservatively pre-marked as mutated (see +/// `exported_tables_are_pre_marked` for the export case) since the host +/// can mutate them via the public wasmtime API. +#[test] +fn read_only_table_is_immutable() { + let bits = translate_and_get_mutability( + r#" + (module + (table 4 funcref) + (func $f (result i32) i32.const 42) + (elem (i32.const 0) $f $f $f $f) + (func (export "call_zero") (result i32) + i32.const 0 + call_indirect (param) (result i32)) + (func (export "read_zero") (result funcref) + i32.const 0 + table.get 0)) + "#, + ); + assert_eq!(bits, vec![false], "no opcode mutated this table"); +} + +/// Exported tables are always pre-marked as mutated, regardless of +/// whether any opcode in this module touches them. The host can call +/// `Table::set` / `Table::grow` via the public wasmtime API on any +/// exported table, and another module that imports the export can also +/// mutate it. Without this rule, downstream optimizations would +/// happily elide null traps and sig checks on exported tables on the +/// (false) assumption that the table contents are stable. +#[test] +fn exported_tables_are_pre_marked() { + let bits = translate_and_get_mutability( + r#" + (module + (table (export "t") 4 funcref) + (func $f (result i32) i32.const 42) + (elem (i32.const 0) $f $f $f $f)) + "#, + ); + assert_eq!(bits, vec![true]); +} + +/// `table.set` marks its destination as mutated. +#[test] +fn table_set_marks_destination() { + let bits = translate_and_get_mutability( + r#" + (module + (table 4 funcref) + (func $f (result i32) i32.const 0) + (func (export "do_set") + i32.const 1 + ref.func $f + table.set 0)) + "#, + ); + assert_eq!(bits, vec![true]); +} + +/// `table.fill` marks its destination as mutated. +#[test] +fn table_fill_marks_destination() { + let bits = translate_and_get_mutability( + r#" + (module + (table 4 funcref) + (func $f (result i32) i32.const 0) + (func (export "do_fill") + i32.const 0 + ref.func $f + i32.const 4 + table.fill 0)) + "#, + ); + assert_eq!(bits, vec![true]); +} + +/// `table.grow` is treated as mutating — analogous to Luau's NAMECALL IC +/// needing to invalidate on table-shape change. +#[test] +fn table_grow_marks_destination() { + let bits = translate_and_get_mutability( + r#" + (module + (table 4 funcref) + (func (export "do_grow") (result i32) + ref.null func + i32.const 1 + table.grow 0)) + "#, + ); + assert_eq!(bits, vec![true]); +} + +/// `table.copy` marks the DESTINATION but explicitly NOT the source. The +/// source is read-only (its contents aren't changed by the op); marking +/// it as mutated would forbid downstream optimizations from treating it +/// as immutable, which would be incorrect over-conservatism — the JSC +/// `ic_table` analogue. +#[test] +fn table_copy_marks_destination_only_not_source() { + let bits = translate_and_get_mutability( + r#" + (module + (table $dst (export "dst") 4 funcref) + (table $src 4 funcref) + (func $f (result i32) i32.const 0) + (elem (table $src) (i32.const 0) func $f $f $f $f) + (func (export "do_copy") + i32.const 0 ;; dst offset + i32.const 0 ;; src offset + i32.const 4 ;; len + table.copy $dst $src)) + "#, + ); + assert_eq!( + bits, + vec![true, false], + "dst should be mutated, src should remain immutable", + ); +} + +/// `table.init` writes to the destination table from a passive elem +/// segment, so it is treated as mutation (the destination's contents +/// change at runtime). +#[test] +fn table_init_marks_destination() { + let bits = translate_and_get_mutability( + r#" + (module + (table 4 funcref) + (func $f (result i32) i32.const 0) + (elem $e funcref (ref.func $f) (ref.func $f)) + (func (export "do_init") + i32.const 0 ;; dst + i32.const 0 ;; src offset within elem + i32.const 2 ;; len + table.init 0 $e)) + "#, + ); + assert_eq!(bits, vec![true]); +} + +/// `elem.drop` drops a passive element segment but does NOT write to any +/// table — distinct from `table.init` which DOES write. A pessimistic +/// implementation that marked all tables as mutated on `elem.drop` would +/// hand out false positives and shut off optimizations on perfectly- +/// immutable tables. +#[test] +fn elem_drop_does_not_mark_tables() { + let bits = translate_and_get_mutability( + r#" + (module + (table 4 funcref) + (func $f (result i32) i32.const 0) + (elem $e funcref (ref.func $f)) + (func (export "do_drop") + elem.drop $e)) + "#, + ); + assert_eq!(bits, vec![false]); +} + +/// Imported tables are always pre-marked as mutated, regardless of +/// whether any opcode in this module touches them. The importer can +/// mutate the table in ways this module can't see. +#[test] +fn imported_tables_are_pre_marked() { + let bits = translate_and_get_mutability( + r#" + (module + (import "host" "t" (table 4 funcref))) + "#, + ); + assert_eq!(bits, vec![true]); +} + +/// A mutation in ONE function correctly marks the table — the analysis +/// has to walk every function body, not just the first. +#[test] +fn mutation_in_any_function_counts() { + let bits = translate_and_get_mutability( + r#" + (module + (table 4 funcref) + (func $f (result i32) i32.const 0) + (func (export "innocent") (result i32) + i32.const 0 + call_indirect (param) (result i32)) + (func (export "guilty") + i32.const 0 + ref.func $f + table.set 0)) + "#, + ); + assert_eq!(bits, vec![true]); +} + +/// Two tables, one mutated, one not. The analysis tracks per-table — a +/// mutation on one must not leak to the other. +#[test] +fn mutation_isolated_to_target_table() { + let bits = translate_and_get_mutability( + r#" + (module + (table $a 4 funcref) + (table $b 4 funcref) + (func $f (result i32) i32.const 0) + (func (export "mut_a") + i32.const 0 + ref.func $f + table.set $a)) + "#, + ); + assert_eq!( + bits, + vec![true, false], + "$a should be mutated, $b should remain immutable", + ); +} + +/// Translating without any tables at all must not panic. (Defensive: the +/// analysis indexes a `SecondaryMap` keyed by `TableIndex`, and we want +/// to confirm an empty module produces an empty result rather than e.g. +/// a default-allocated single entry.) +#[test] +fn module_with_no_tables_produces_empty_mutability_vec() { + let bits = translate_and_get_mutability( + r#" + (module + (func (export "noop"))) + "#, + ); + assert!(bits.is_empty(), "no tables ⇒ no mutability bits"); +} diff --git a/tests/disas/call-indirect-immutable-elide-null.wat b/tests/disas/call-indirect-immutable-elide-null.wat new file mode 100644 index 000000000000..35e2e0c7f0db --- /dev/null +++ b/tests/disas/call-indirect-immutable-elide-null.wat @@ -0,0 +1,116 @@ +;;! target = "x86_64" + +;; Immutable funcref table where every slot is filled by the elem +;; segment (no "no-entry" gaps). With both the sig check AND the +;; funcref-NULL check elided, the dispatch path is reduced to: +;; - bounds check (static) +;; - lazy-init brif + masking +;; - load code+vmctx +;; - call_indirect +;; +;; In particular the cold block that handles the runtime trap-on-null +;; path should not exist after the funcref load: the static-match path +;; with `may_be_null = false` skips both the sig check and any +;; downstream null-handling. + +(module + (table 3 3 funcref) + + (func $f1 (result i32) i32.const 1) + (func $f2 (result i32) i32.const 2) + (func $f3 (result i32) i32.const 3) + + (func (export "call_it") (param i32) (result i32) + local.get 0 + call_indirect (result i32)) + + ;; Fully cover the table — no null slot anywhere. + (elem (i32.const 0) func $f1 $f2 $f3)) +;; function u0:0(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly region0 gv0+8 +;; gv2 = load.i64 notrap aligned gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @003f v3 = iconst.i32 1 +;; @0041 jump block1 +;; +;; block1: +;; @0041 return v3 ; v3 = 1 +;; } +;; +;; function u0:1(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly region0 gv0+8 +;; gv2 = load.i64 notrap aligned gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @0044 v3 = iconst.i32 2 +;; @0046 jump block1 +;; +;; block1: +;; @0046 return v3 ; v3 = 2 +;; } +;; +;; function u0:2(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly region0 gv0+8 +;; gv2 = load.i64 notrap aligned gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @0049 v3 = iconst.i32 3 +;; @004b jump block1 +;; +;; block1: +;; @004b return v3 ; v3 = 3 +;; } +;; +;; function u0:3(i64 vmctx, i64, i32) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 1342177280 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly region0 gv0+8 +;; gv2 = load.i64 notrap aligned gv1+24 +;; gv3 = vmctx +;; gv4 = load.i64 notrap aligned readonly can_move gv3+48 +;; sig0 = (i64 vmctx, i64) -> i32 tail +;; sig1 = (i64 vmctx, i32, i64) -> i64 tail +;; fn0 = colocated u805306368:7 sig1 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i32): +;; @0050 v4 = iconst.i32 3 +;; @0050 v5 = icmp uge v2, v4 ; v4 = 3 +;; @0050 v6 = uextend.i64 v2 +;; @0050 v7 = load.i64 notrap aligned readonly can_move v0+48 +;; @0050 v8 = iconst.i64 3 +;; @0050 v9 = ishl v6, v8 ; v8 = 3 +;; @0050 v10 = iadd v7, v9 +;; @0050 v11 = iconst.i64 0 +;; @0050 v12 = select_spectre_guard v5, v11, v10 ; v11 = 0 +;; @0050 v13 = load.i64 user6 aligned region1 v12 +;; @0050 v14 = iconst.i64 -2 +;; @0050 v15 = band v13, v14 ; v14 = -2 +;; @0050 brif v13, block3(v15), block2 +;; +;; block2 cold: +;; @0050 v17 = iconst.i32 0 +;; @0050 v18 = uextend.i64 v2 +;; @0050 v19 = call fn0(v0, v17, v18) ; v17 = 0 +;; @0050 jump block3(v19) +;; +;; block3(v16: i64): +;; @0050 v20 = load.i64 notrap aligned readonly v16+8 +;; @0050 v21 = load.i64 notrap aligned readonly v16+24 +;; @0050 v22 = call_indirect sig0, v20(v21, v0) +;; @0053 jump block1 +;; +;; block1: +;; @0053 return v22 +;; } diff --git a/tests/disas/call-indirect-immutable-elide-sig.wat b/tests/disas/call-indirect-immutable-elide-sig.wat new file mode 100644 index 000000000000..d5d892f6d99a --- /dev/null +++ b/tests/disas/call-indirect-immutable-elide-sig.wat @@ -0,0 +1,115 @@ +;;! target = "x86_64" + +;; Immutable funcref table where every elem-segment entry has the same +;; declared type as the call site. This module's `tables_mutated` bit +;; for table 0 is clear (no opcode in any function writes to it), and +;; all three slots resolve to the same module type as the call site. +;; That triggers `try_elide_sig_check_for_immutable_table` → +;; `CheckIndirectCallTypeSignature::StaticMatch`, removing the runtime +;; signature load + compare from the dispatch hot path. +;; +;; Look for the absence of `load.i32 user6 aligned readonly v_+16` (the +;; sig-id load) and the matching `icmp eq / trapz user7` on the call +;; site. Compare with `indirect-call-no-caching.wat` for the +;; non-elided shape. + +(module + (table 10 10 funcref) + + (func $f1 (result i32) i32.const 1) + (func $f2 (result i32) i32.const 2) + (func $f3 (result i32) i32.const 3) + + (func (export "call_it") (param i32) (result i32) + local.get 0 + call_indirect (result i32)) + + (elem (i32.const 0) func $f1 $f2 $f3)) +;; function u0:0(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly region0 gv0+8 +;; gv2 = load.i64 notrap aligned gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @003f v3 = iconst.i32 1 +;; @0041 jump block1 +;; +;; block1: +;; @0041 return v3 ; v3 = 1 +;; } +;; +;; function u0:1(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly region0 gv0+8 +;; gv2 = load.i64 notrap aligned gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @0044 v3 = iconst.i32 2 +;; @0046 jump block1 +;; +;; block1: +;; @0046 return v3 ; v3 = 2 +;; } +;; +;; function u0:2(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly region0 gv0+8 +;; gv2 = load.i64 notrap aligned gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @0049 v3 = iconst.i32 3 +;; @004b jump block1 +;; +;; block1: +;; @004b return v3 ; v3 = 3 +;; } +;; +;; function u0:3(i64 vmctx, i64, i32) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 1342177280 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly region0 gv0+8 +;; gv2 = load.i64 notrap aligned gv1+24 +;; gv3 = vmctx +;; gv4 = load.i64 notrap aligned readonly can_move gv3+48 +;; sig0 = (i64 vmctx, i64) -> i32 tail +;; sig1 = (i64 vmctx, i32, i64) -> i64 tail +;; fn0 = colocated u805306368:7 sig1 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i32): +;; @0050 v4 = iconst.i32 10 +;; @0050 v5 = icmp uge v2, v4 ; v4 = 10 +;; @0050 v6 = uextend.i64 v2 +;; @0050 v7 = load.i64 notrap aligned readonly can_move v0+48 +;; @0050 v8 = iconst.i64 3 +;; @0050 v9 = ishl v6, v8 ; v8 = 3 +;; @0050 v10 = iadd v7, v9 +;; @0050 v11 = iconst.i64 0 +;; @0050 v12 = select_spectre_guard v5, v11, v10 ; v11 = 0 +;; @0050 v13 = load.i64 user6 aligned region1 v12 +;; @0050 v14 = iconst.i64 -2 +;; @0050 v15 = band v13, v14 ; v14 = -2 +;; @0050 brif v13, block3(v15), block2 +;; +;; block2 cold: +;; @0050 v17 = iconst.i32 0 +;; @0050 v18 = uextend.i64 v2 +;; @0050 v19 = call fn0(v0, v17, v18) ; v17 = 0 +;; @0050 jump block3(v19) +;; +;; block3(v16: i64): +;; @0050 v20 = load.i64 user7 aligned readonly v16+8 +;; @0050 v21 = load.i64 notrap aligned readonly v16+24 +;; @0050 v22 = call_indirect sig0, v20(v21, v0) +;; @0053 jump block1 +;; +;; block1: +;; @0053 return v22 +;; } diff --git a/tests/disas/call-indirect-immutable-static-bound.wat b/tests/disas/call-indirect-immutable-static-bound.wat new file mode 100644 index 000000000000..05c3ffd748ab --- /dev/null +++ b/tests/disas/call-indirect-immutable-static-bound.wat @@ -0,0 +1,115 @@ +;;! target = "x86_64" + +;; Table declared with min < max (a "dynamic-declared" table) that is +;; never written to in the module. Without the per-table mutability +;; bit, Cranelift would emit `load.i64 v0+56` per dispatch to fetch +;; the current bound. With it, `make_table` lowers to +;; `TableSize::Static` and the bound becomes an immediate. +;; +;; Look for: bounds-check `iconst.i32 16` (the declared min, used as +;; static bound) and NO `load.i64 ... v0+56` for the current_elements +;; field. (`+48` for the funcref base is still loaded — that's the +;; element-data pointer, separate from the bound.) + +(module + ;; min=16, max=64 — distinct, so without our optimization the + ;; bound would be loaded per dispatch from `current_elements`. + (table 16 64 funcref) + + (func $f1 (result i32) i32.const 1) + (func $f2 (result i32) i32.const 2) + (func $f3 (result i32) i32.const 3) + + (func (export "call_it") (param i32) (result i32) + local.get 0 + call_indirect (result i32)) + + (elem (i32.const 0) func $f1 $f2 $f3)) +;; function u0:0(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly region0 gv0+8 +;; gv2 = load.i64 notrap aligned gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @003f v3 = iconst.i32 1 +;; @0041 jump block1 +;; +;; block1: +;; @0041 return v3 ; v3 = 1 +;; } +;; +;; function u0:1(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly region0 gv0+8 +;; gv2 = load.i64 notrap aligned gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @0044 v3 = iconst.i32 2 +;; @0046 jump block1 +;; +;; block1: +;; @0046 return v3 ; v3 = 2 +;; } +;; +;; function u0:2(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly region0 gv0+8 +;; gv2 = load.i64 notrap aligned gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @0049 v3 = iconst.i32 3 +;; @004b jump block1 +;; +;; block1: +;; @004b return v3 ; v3 = 3 +;; } +;; +;; function u0:3(i64 vmctx, i64, i32) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 1342177280 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly region0 gv0+8 +;; gv2 = load.i64 notrap aligned gv1+24 +;; gv3 = vmctx +;; gv4 = load.i64 notrap aligned readonly can_move gv3+48 +;; sig0 = (i64 vmctx, i64) -> i32 tail +;; sig1 = (i64 vmctx, i32, i64) -> i64 tail +;; fn0 = colocated u805306368:7 sig1 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i32): +;; @0050 v4 = iconst.i32 16 +;; @0050 v5 = icmp uge v2, v4 ; v4 = 16 +;; @0050 v6 = uextend.i64 v2 +;; @0050 v7 = load.i64 notrap aligned readonly can_move v0+48 +;; @0050 v8 = iconst.i64 3 +;; @0050 v9 = ishl v6, v8 ; v8 = 3 +;; @0050 v10 = iadd v7, v9 +;; @0050 v11 = iconst.i64 0 +;; @0050 v12 = select_spectre_guard v5, v11, v10 ; v11 = 0 +;; @0050 v13 = load.i64 user6 aligned region1 v12 +;; @0050 v14 = iconst.i64 -2 +;; @0050 v15 = band v13, v14 ; v14 = -2 +;; @0050 brif v13, block3(v15), block2 +;; +;; block2 cold: +;; @0050 v17 = iconst.i32 0 +;; @0050 v18 = uextend.i64 v2 +;; @0050 v19 = call fn0(v0, v17, v18) ; v17 = 0 +;; @0050 jump block3(v19) +;; +;; block3(v16: i64): +;; @0050 v20 = load.i64 user7 aligned readonly v16+8 +;; @0050 v21 = load.i64 notrap aligned readonly v16+24 +;; @0050 v22 = call_indirect sig0, v20(v21, v0) +;; @0053 jump block1 +;; +;; block1: +;; @0053 return v22 +;; } diff --git a/tests/disas/call-indirect-mutable-keeps-sigcheck.wat b/tests/disas/call-indirect-mutable-keeps-sigcheck.wat new file mode 100644 index 000000000000..03318a349ef7 --- /dev/null +++ b/tests/disas/call-indirect-mutable-keeps-sigcheck.wat @@ -0,0 +1,159 @@ +;;! target = "x86_64" + +;; Counterpart to `call-indirect-immutable-elide-sig.wat`. Same module +;; shape — same elem segment, same uniform call-site type — but one +;; function writes to the table via `table.set`. That sets the +;; `tables_mutated` bit and disables sig-check elision. +;; +;; Look for the runtime sig load + compare on the call site: +;; load.i32 user6 aligned readonly v_+16 +;; icmp eq +;; trapz user7 +;; (versus the elided form in the immutable test). + +(module + (table 10 10 funcref) + + (func $f1 (result i32) i32.const 1) + (func $f2 (result i32) i32.const 2) + (func $f3 (result i32) i32.const 3) + + ;; Mutator: this clears the immutability proof for table 0. + (func (export "mutate") (param i32) + local.get 0 + ref.func $f1 + table.set 0) + + (func (export "call_it") (param i32) (result i32) + local.get 0 + call_indirect (result i32)) + + (elem (i32.const 0) func $f1 $f2 $f3)) +;; function u0:0(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly region0 gv0+8 +;; gv2 = load.i64 notrap aligned gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @004d v3 = iconst.i32 1 +;; @004f jump block1 +;; +;; block1: +;; @004f return v3 ; v3 = 1 +;; } +;; +;; function u0:1(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly region0 gv0+8 +;; gv2 = load.i64 notrap aligned gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @0052 v3 = iconst.i32 2 +;; @0054 jump block1 +;; +;; block1: +;; @0054 return v3 ; v3 = 2 +;; } +;; +;; function u0:2(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly region0 gv0+8 +;; gv2 = load.i64 notrap aligned gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @0057 v3 = iconst.i32 3 +;; @0059 jump block1 +;; +;; block1: +;; @0059 return v3 ; v3 = 3 +;; } +;; +;; function u0:3(i64 vmctx, i64, i32) tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 1342177280 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly region0 gv0+8 +;; gv2 = load.i64 notrap aligned gv1+24 +;; gv3 = vmctx +;; gv4 = load.i64 notrap aligned readonly can_move gv3+48 +;; sig0 = (i64 vmctx, i32) -> i64 tail +;; fn0 = colocated u805306368:6 sig0 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i32): +;; @005e v3 = iconst.i32 0 +;; @005e v4 = call fn0(v0, v3) ; v3 = 0 +;; @0060 v5 = iconst.i32 10 +;; @0060 v6 = icmp uge v2, v5 ; v5 = 10 +;; @0060 v7 = uextend.i64 v2 +;; @0060 v8 = load.i64 notrap aligned readonly can_move v0+48 +;; @0060 v9 = iconst.i64 3 +;; @0060 v10 = ishl v7, v9 ; v9 = 3 +;; @0060 v11 = iadd v8, v10 +;; @0060 v12 = iconst.i64 0 +;; @0060 v13 = select_spectre_guard v6, v12, v11 ; v12 = 0 +;; @0060 v14 = iconst.i64 1 +;; @0060 v15 = bor v4, v14 ; v14 = 1 +;; @0060 store user6 aligned region1 v15, v13 +;; @0062 jump block1 +;; +;; block1: +;; @0062 return +;; } +;; +;; function u0:4(i64 vmctx, i64, i32) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 1342177280 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" +;; region2 = 40 "VMContext+0x28" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly region0 gv0+8 +;; gv2 = load.i64 notrap aligned gv1+24 +;; gv3 = vmctx +;; gv4 = load.i64 notrap aligned readonly can_move gv3+48 +;; sig0 = (i64 vmctx, i64) -> i32 tail +;; sig1 = (i64 vmctx, i32, i64) -> i64 tail +;; fn0 = colocated u805306368:7 sig1 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i32): +;; @0067 v4 = iconst.i32 10 +;; @0067 v5 = icmp uge v2, v4 ; v4 = 10 +;; @0067 v6 = uextend.i64 v2 +;; @0067 v7 = load.i64 notrap aligned readonly can_move v0+48 +;; @0067 v8 = iconst.i64 3 +;; @0067 v9 = ishl v6, v8 ; v8 = 3 +;; @0067 v10 = iadd v7, v9 +;; @0067 v11 = iconst.i64 0 +;; @0067 v12 = select_spectre_guard v5, v11, v10 ; v11 = 0 +;; @0067 v13 = load.i64 user6 aligned region1 v12 +;; @0067 v14 = iconst.i64 -2 +;; @0067 v15 = band v13, v14 ; v14 = -2 +;; @0067 brif v13, block3(v15), block2 +;; +;; block2 cold: +;; @0067 v17 = iconst.i32 0 +;; @0067 v18 = uextend.i64 v2 +;; @0067 v19 = call fn0(v0, v17, v18) ; v17 = 0 +;; @0067 jump block3(v19) +;; +;; block3(v16: i64): +;; @0067 v20 = load.i64 notrap aligned readonly can_move region2 v0+40 +;; @0067 v21 = load.i32 notrap aligned readonly can_move v20 +;; @0067 v22 = load.i32 user7 aligned readonly v16+16 +;; @0067 v23 = icmp eq v22, v21 +;; @0067 v24 = uextend.i32 v23 +;; @0067 trapz v24, user8 +;; @0067 v25 = load.i64 notrap aligned readonly v16+8 +;; @0067 v26 = load.i64 notrap aligned readonly v16+24 +;; @0067 v27 = call_indirect sig0, v25(v26, v0) +;; @006a jump block1 +;; +;; block1: +;; @006a return v27 +;; } diff --git a/tests/disas/gc/call-indirect-final-type.wat b/tests/disas/gc/call-indirect-final-type.wat index 0406261611bf..13ffa96bec62 100644 --- a/tests/disas/gc/call-indirect-final-type.wat +++ b/tests/disas/gc/call-indirect-final-type.wat @@ -23,47 +23,38 @@ ;; gv1 = load.i64 notrap aligned readonly region0 gv0+8 ;; gv2 = load.i64 notrap aligned gv1+24 ;; gv3 = vmctx -;; gv4 = load.i64 notrap aligned gv3+48 -;; gv5 = load.i64 notrap aligned gv3+56 +;; gv4 = load.i64 notrap aligned readonly can_move gv3+48 ;; sig0 = (i64 vmctx, i64, i32) -> i32 tail ;; sig1 = (i64 vmctx, i32, i64) -> i64 tail ;; fn0 = colocated u805306368:7 sig1 ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: i32, v3: i32): -;; @002b v5 = load.i64 notrap aligned v0+56 -;; @002b v9 = load.i64 notrap aligned v0+48 -;; @002b v6 = ireduce.i32 v5 -;; @002b v7 = icmp uge v3, v6 -;; @002b v13 = iconst.i64 0 -;; @002b v8 = uextend.i64 v3 -;; @002b v10 = iconst.i64 3 -;; @002b v11 = ishl v8, v10 ; v10 = 3 -;; @002b v12 = iadd v9, v11 -;; @002b v14 = select_spectre_guard v7, v13, v12 ; v13 = 0 -;; @002b v15 = load.i64 user6 aligned region1 v14 -;; @002b v16 = iconst.i64 -2 -;; @002b v17 = band v15, v16 ; v16 = -2 -;; @002b brif v15, block3(v17), block2 +;; @002b v12 = iconst.i64 0 +;; @002b v14 = load.i64 user6 aligned region1 v12 ; v12 = 0 +;; @002b v15 = iconst.i64 -2 +;; @002b v16 = band v14, v15 ; v15 = -2 +;; @002b brif v14, block3(v16), block2 ;; ;; block2 cold: -;; @002b v19 = iconst.i32 0 -;; @002b v21 = call fn0(v0, v19, v8) ; v19 = 0 -;; @002b jump block3(v21) +;; @002b v5 = iconst.i32 0 +;; @002b v7 = uextend.i64 v3 +;; @002b v20 = call fn0(v0, v5, v7) ; v5 = 0 +;; @002b jump block3(v20) ;; -;; block3(v18: i64): -;; @002b v24 = load.i32 user7 aligned readonly v18+16 -;; @002b v22 = load.i64 notrap aligned readonly can_move region2 v0+40 -;; @002b v23 = load.i32 notrap aligned readonly can_move v22 -;; @002b v25 = icmp eq v24, v23 -;; @002b trapz v25, user8 -;; @002b v27 = load.i64 notrap aligned readonly v18+8 -;; @002b v28 = load.i64 notrap aligned readonly v18+24 -;; @002b v29 = call_indirect sig0, v27(v28, v0, v2) +;; block3(v17: i64): +;; @002b v23 = load.i32 user7 aligned readonly v17+16 +;; @002b v21 = load.i64 notrap aligned readonly can_move region2 v0+40 +;; @002b v22 = load.i32 notrap aligned readonly can_move v21 +;; @002b v24 = icmp eq v23, v22 +;; @002b trapz v24, user8 +;; @002b v26 = load.i64 notrap aligned readonly v17+8 +;; @002b v27 = load.i64 notrap aligned readonly v17+24 +;; @002b v28 = call_indirect sig0, v26(v27, v0, v2) ;; @002e jump block1 ;; ;; block1: -;; @002e return v29 +;; @002e return v28 ;; } ;; ;; function u0:1(i64 vmctx, i64, i32, i32) -> i32 tail { @@ -74,41 +65,32 @@ ;; gv1 = load.i64 notrap aligned readonly region0 gv0+8 ;; gv2 = load.i64 notrap aligned gv1+24 ;; gv3 = vmctx -;; gv4 = load.i64 notrap aligned gv3+48 -;; gv5 = load.i64 notrap aligned gv3+56 +;; gv4 = load.i64 notrap aligned readonly can_move gv3+48 ;; sig0 = (i64 vmctx, i64, i32) -> i32 tail ;; sig1 = (i64 vmctx, i32, i64) -> i64 tail ;; fn0 = colocated u805306368:7 sig1 ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: i32, v3: i32): -;; @0035 v5 = load.i64 notrap aligned v0+56 -;; @0035 v9 = load.i64 notrap aligned v0+48 -;; @0035 v6 = ireduce.i32 v5 -;; @0035 v7 = icmp uge v3, v6 -;; @0035 v13 = iconst.i64 0 -;; @0035 v8 = uextend.i64 v3 -;; @0035 v10 = iconst.i64 3 -;; @0035 v11 = ishl v8, v10 ; v10 = 3 -;; @0035 v12 = iadd v9, v11 -;; @0035 v14 = select_spectre_guard v7, v13, v12 ; v13 = 0 -;; @0035 v15 = load.i64 user6 aligned region1 v14 -;; @0035 v16 = iconst.i64 -2 -;; @0035 v17 = band v15, v16 ; v16 = -2 -;; @0035 brif v15, block3(v17), block2 +;; @0035 v12 = iconst.i64 0 +;; @0035 v14 = load.i64 user6 aligned region1 v12 ; v12 = 0 +;; @0035 v15 = iconst.i64 -2 +;; @0035 v16 = band v14, v15 ; v15 = -2 +;; @0035 brif v14, block3(v16), block2 ;; ;; block2 cold: -;; @0035 v19 = iconst.i32 0 -;; @0035 v21 = call fn0(v0, v19, v8) ; v19 = 0 -;; @0035 jump block3(v21) +;; @0035 v5 = iconst.i32 0 +;; @0035 v7 = uextend.i64 v3 +;; @0035 v20 = call fn0(v0, v5, v7) ; v5 = 0 +;; @0035 jump block3(v20) ;; -;; block3(v18: i64): -;; @0035 v24 = load.i32 user7 aligned readonly v18+16 -;; @0035 v22 = load.i64 notrap aligned readonly can_move region2 v0+40 -;; @0035 v23 = load.i32 notrap aligned readonly can_move v22 -;; @0035 v25 = icmp eq v24, v23 -;; @0035 trapz v25, user8 -;; @0035 v27 = load.i64 notrap aligned readonly v18+8 -;; @0035 v28 = load.i64 notrap aligned readonly v18+24 -;; @0035 return_call_indirect sig0, v27(v28, v0, v2) +;; block3(v17: i64): +;; @0035 v23 = load.i32 user7 aligned readonly v17+16 +;; @0035 v21 = load.i64 notrap aligned readonly can_move region2 v0+40 +;; @0035 v22 = load.i32 notrap aligned readonly can_move v21 +;; @0035 v24 = icmp eq v23, v22 +;; @0035 trapz v24, user8 +;; @0035 v26 = load.i64 notrap aligned readonly v17+8 +;; @0035 v27 = load.i64 notrap aligned readonly v17+24 +;; @0035 return_call_indirect sig0, v26(v27, v0, v2) ;; } diff --git a/tests/disas/indirect-call-no-caching.wat b/tests/disas/indirect-call-no-caching.wat index ae42c54f4c27..1a2e852558bb 100644 --- a/tests/disas/indirect-call-no-caching.wat +++ b/tests/disas/indirect-call-no-caching.wat @@ -68,7 +68,6 @@ ;; function u0:3(i64 vmctx, i64, i32) -> i32 tail { ;; region0 = 8 "VMContext+0x8" ;; region1 = 1342177280 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" -;; region2 = 40 "VMContext+0x28" ;; gv0 = vmctx ;; gv1 = load.i64 notrap aligned readonly region0 gv0+8 ;; gv2 = load.i64 notrap aligned gv1+24 @@ -101,17 +100,11 @@ ;; @0050 jump block3(v19) ;; ;; block3(v16: i64): -;; @0050 v20 = load.i64 notrap aligned readonly can_move region2 v0+40 -;; @0050 v21 = load.i32 notrap aligned readonly can_move v20 -;; @0050 v22 = load.i32 user7 aligned readonly v16+16 -;; @0050 v23 = icmp eq v22, v21 -;; @0050 v24 = uextend.i32 v23 -;; @0050 trapz v24, user8 -;; @0050 v25 = load.i64 notrap aligned readonly v16+8 -;; @0050 v26 = load.i64 notrap aligned readonly v16+24 -;; @0050 v27 = call_indirect sig0, v25(v26, v0) +;; @0050 v20 = load.i64 user7 aligned readonly v16+8 +;; @0050 v21 = load.i64 notrap aligned readonly v16+24 +;; @0050 v22 = call_indirect sig0, v20(v21, v0) ;; @0053 jump block1 ;; ;; block1: -;; @0053 return v27 +;; @0053 return v22 ;; } diff --git a/tests/disas/readonly-funcrefs.wat b/tests/disas/readonly-funcrefs.wat index 9febf947e3b1..e341fbcc4dba 100644 --- a/tests/disas/readonly-funcrefs.wat +++ b/tests/disas/readonly-funcrefs.wat @@ -35,7 +35,6 @@ ;; function u0:1(i64 vmctx, i64, i32) tail { ;; region0 = 8 "VMContext+0x8" ;; region1 = 1342177280 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" -;; region2 = 40 "VMContext+0x28" ;; gv0 = vmctx ;; gv1 = load.i64 notrap aligned readonly region0 gv0+8 ;; gv2 = load.i64 notrap aligned gv1+24 @@ -67,14 +66,9 @@ ;; @0031 jump block3(v18) ;; ;; block3(v15: i64): -;; @0031 v21 = load.i32 user7 aligned readonly v15+16 -;; @0031 v19 = load.i64 notrap aligned readonly can_move region2 v0+40 -;; @0031 v20 = load.i32 notrap aligned readonly can_move v19 -;; @0031 v22 = icmp eq v21, v20 -;; @0031 trapz v22, user8 -;; @0031 v24 = load.i64 notrap aligned readonly v15+8 -;; @0031 v25 = load.i64 notrap aligned readonly v15+24 -;; @0031 call_indirect sig0, v24(v25, v0) +;; @0031 v19 = load.i64 user7 aligned readonly v15+8 +;; @0031 v20 = load.i64 notrap aligned readonly v15+24 +;; @0031 call_indirect sig0, v19(v20, v0) ;; @0034 jump block1 ;; ;; block1: diff --git a/tests/disas/startup-elem-active.wat b/tests/disas/startup-elem-active.wat index 0c3158f8c2b1..40cdfd2d91f4 100644 --- a/tests/disas/startup-elem-active.wat +++ b/tests/disas/startup-elem-active.wat @@ -42,37 +42,21 @@ ;; function u2415919104:0(i64 vmctx, i64) tail { ;; region0 = 1342177280 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" ;; gv0 = vmctx -;; gv1 = load.i64 notrap aligned gv0+48 -;; gv2 = load.i64 notrap aligned gv0+56 +;; gv1 = load.i64 notrap aligned readonly can_move gv0+48 ;; ;; block0(v0: i64, v1: i64): -;; v4 = load.i64 notrap aligned v0+56 -;; v5 = ireduce.i32 v4 -;; v6 = uextend.i64 v5 -;; v86 = iconst.i64 4 -;; v92 = icmp ult v6, v86 ; v86 = 4 -;; trapnz v92, user6 -;; v13 = load.i64 notrap aligned v0+48 -;; v103 = iconst.i32 21 -;; v2 = iconst.i32 1 -;; v114 = icmp ule v5, v2 ; v2 = 1 -;; v79 = iconst.i64 0 -;; v17 = iadd v13, v86 ; v86 = 4 -;; v34 = select_spectre_guard v114, v79, v17 ; v79 = 0 -;; store user6 aligned region0 v103, v34 ; v103 = 21 +;; v100 = iconst.i32 21 +;; v12 = load.i64 notrap aligned readonly can_move v0+48 +;; v79 = iconst.i64 4 +;; v16 = iadd v12, v79 ; v79 = 4 +;; store user6 aligned region0 v100, v16 ; v100 = 21 ;; v117 = iconst.i32 23 -;; v123 = iconst.i32 2 -;; v129 = icmp ule v5, v123 ; v123 = 2 -;; v131 = iconst.i64 8 -;; v49 = iadd v13, v131 ; v131 = 8 -;; v51 = select_spectre_guard v129, v79, v49 ; v79 = 0 -;; store user6 aligned region0 v117, v51 ; v117 = 23 -;; v133 = iconst.i32 25 -;; v3 = iconst.i32 3 -;; v144 = icmp ule v5, v3 ; v3 = 3 -;; v146 = iconst.i64 12 -;; v66 = iadd v13, v146 ; v146 = 12 -;; v68 = select_spectre_guard v144, v79, v66 ; v79 = 0 -;; store user6 aligned region0 v133, v68 ; v133 = 25 +;; v134 = iconst.i64 8 +;; v46 = iadd v12, v134 ; v134 = 8 +;; store user6 aligned region0 v117, v46 ; v117 = 23 +;; v136 = iconst.i32 25 +;; v152 = iconst.i64 12 +;; v62 = iadd v12, v152 ; v152 = 12 +;; store user6 aligned region0 v136, v62 ; v136 = 25 ;; return ;; } diff --git a/tests/disas/startup-table-initial-value.wat b/tests/disas/startup-table-initial-value.wat index 7b39ecc93333..a2cb9a5f6da2 100644 --- a/tests/disas/startup-table-initial-value.wat +++ b/tests/disas/startup-table-initial-value.wat @@ -35,31 +35,24 @@ ;; ;; function u2415919104:0(i64 vmctx, i64) tail { ;; gv0 = vmctx -;; gv1 = load.i64 notrap aligned gv0+48 -;; gv2 = load.i64 notrap aligned gv0+56 +;; gv1 = load.i64 notrap aligned readonly can_move gv0+48 ;; ;; block0(v0: i64, v1: i64): -;; v9 = load.i64 notrap aligned v0+56 -;; v10 = ireduce.i32 v9 -;; v11 = uextend.i64 v10 -;; v41 = iconst.i64 10 -;; v53 = icmp ult v11, v41 ; v41 = 10 -;; trapnz v53, user6 -;; v18 = load.i64 notrap aligned v0+48 +;; v17 = load.i64 notrap aligned readonly can_move v0+48 ;; v3 = iconst.i32 1 -;; v83 = iconst.i64 36 -;; v85 = iadd v18, v83 ; v83 = 36 -;; v20 = iconst.i64 4 -;; jump block1(v18) -;; -;; block1(v29: i64): -;; v88 = iconst.i32 1 -;; store notrap aligned v88, v29 ; v88 = 1 -;; v89 = iadd.i64 v18, v83 ; v83 = 36 -;; v90 = icmp eq v29, v89 -;; v91 = iconst.i64 4 -;; v92 = iadd v29, v91 ; v91 = 4 -;; brif v90, block2, block1(v92) +;; v84 = iconst.i64 36 +;; v86 = iadd v17, v84 ; v84 = 36 +;; v19 = iconst.i64 4 +;; jump block1(v17) +;; +;; block1(v28: i64): +;; v89 = iconst.i32 1 +;; store notrap aligned v89, v28 ; v89 = 1 +;; v90 = iadd.i64 v17, v84 ; v84 = 36 +;; v91 = icmp eq v28, v90 +;; v92 = iconst.i64 4 +;; v93 = iadd v28, v92 ; v92 = 4 +;; brif v91, block2, block1(v93) ;; ;; block2: ;; return diff --git a/tests/misc_testsuite/immutable-table-call-indirect.wast b/tests/misc_testsuite/immutable-table-call-indirect.wast new file mode 100644 index 000000000000..3b40cb9ab534 --- /dev/null +++ b/tests/misc_testsuite/immutable-table-call-indirect.wast @@ -0,0 +1,71 @@ +;;! reference_types = true + +;; call_indirect through tables that are never grown, exported, or mutated. +;; Compilation may use a constant bound and elide null/signature checks on +;; these shapes; runtime behavior must be unchanged: in-bounds calls work, +;; and out-of-bounds, null-slot, and signature-mismatch accesses still trap. + +;; Mixed-signature immutable table with a null hole. +(module + (type $i2i (func (param i32) (result i32))) + (type $v2i (func (result i32))) + (table 5 funcref) + (elem (i32.const 0) $add1 $ten $add1) + + (func $add1 (type $i2i) (i32.add (local.get 0) (i32.const 1))) + (func $ten (type $v2i) (i32.const 10)) + + (func (export "call-i2i") (param i32 i32) (result i32) + (call_indirect (type $i2i) (local.get 1) (local.get 0))) + (func (export "call-v2i") (param i32) (result i32) + (call_indirect (type $v2i) (local.get 0)))) + +(assert_return (invoke "call-i2i" (i32.const 0) (i32.const 41)) (i32.const 42)) +(assert_return (invoke "call-i2i" (i32.const 2) (i32.const 7)) (i32.const 8)) +(assert_return (invoke "call-v2i" (i32.const 1)) (i32.const 10)) + +;; Signature mismatch still traps. +(assert_trap (invoke "call-i2i" (i32.const 1) (i32.const 0)) "indirect call type mismatch") +(assert_trap (invoke "call-v2i" (i32.const 0)) "indirect call type mismatch") + +;; Null slots still trap: slot 3 was never initialized. +(assert_trap (invoke "call-i2i" (i32.const 3) (i32.const 0)) "uninitialized element") +(assert_trap (invoke "call-v2i" (i32.const 4)) "uninitialized element") + +;; Out of bounds still traps against the constant bound. +(assert_trap (invoke "call-i2i" (i32.const 5) (i32.const 0)) "undefined element") +(assert_trap (invoke "call-i2i" (i32.const -1) (i32.const 0)) "undefined element") + +;; Uniform-signature immutable table, fully initialized. +(module + (type $v2i (func (result i32))) + (table 3 funcref) + (elem (i32.const 0) $a $b $c) + + (func $a (type $v2i) (i32.const 1)) + (func $b (type $v2i) (i32.const 2)) + (func $c (type $v2i) (i32.const 3)) + + (func (export "call") (param i32) (result i32) + (call_indirect (type $v2i) (local.get 0))) + (func (export "call-wrong-type") (param i32 i32) (result i32) + (call_indirect (param i32) (result i32) (local.get 1) (local.get 0)))) + +(assert_return (invoke "call" (i32.const 0)) (i32.const 1)) +(assert_return (invoke "call" (i32.const 1)) (i32.const 2)) +(assert_return (invoke "call" (i32.const 2)) (i32.const 3)) +(assert_trap (invoke "call" (i32.const 3)) "undefined element") + +;; A caller whose expected type differs from the table's uniform type must +;; still observe the mismatch. +(assert_trap (invoke "call-wrong-type" (i32.const 0) (i32.const 0)) "indirect call type mismatch") + +;; Same shapes through a declared-growable (no max) table never actually +;; grown: an empty never-grown table has no valid index. +(module + (table 0 100 funcref) + (func (export "call-empty") (param i32) + (call_indirect (local.get 0)))) + +(assert_trap (invoke "call-empty" (i32.const 0)) "undefined element") +(assert_trap (invoke "call-empty" (i32.const 99)) "undefined element")