diff --git a/cranelift/wasm/src/sections_translator.rs b/cranelift/wasm/src/sections_translator.rs index 967612178a3c..eb87b8e3d1ec 100644 --- a/cranelift/wasm/src/sections_translator.rs +++ b/cranelift/wasm/src/sections_translator.rs @@ -97,6 +97,7 @@ pub fn parse_import_section<'data>( ImportSectionEntryType::Global(ref ty) => { environ.declare_global_import( Global { + wasm_ty: ty.content_type, ty: type_to_type(ty.content_type, environ).unwrap(), mutability: ty.mutable, initializer: GlobalInit::Import, @@ -227,6 +228,7 @@ pub fn parse_global_section( } }; let global = Global { + wasm_ty: content_type, ty: type_to_type(content_type, environ).unwrap(), mutability: mutable, initializer, diff --git a/cranelift/wasm/src/translation_utils.rs b/cranelift/wasm/src/translation_utils.rs index 19b05f760c78..ce0be42b9b7d 100644 --- a/cranelift/wasm/src/translation_utils.rs +++ b/cranelift/wasm/src/translation_utils.rs @@ -67,10 +67,18 @@ entity_impl!(DataIndex); pub struct ElemIndex(u32); entity_impl!(ElemIndex); -/// WebAssembly global. +/// A WebAssembly global. +/// +/// Note that we record both the original Wasm type and the Cranelift IR type +/// used to represent it. This is because multiple different kinds of Wasm types +/// might be represented with the same Cranelift IR type. For example, both a +/// Wasm `i64` and a `funcref` might be represented with a Cranelift `i64` on +/// 64-bit architectures, and when GC is not required for func refs. #[derive(Debug, Clone, Copy, Hash, Eq, PartialEq)] pub struct Global { - /// The type of the value stored in the global. + /// The Wasm type of the value stored in the global. + pub wasm_ty: crate::WasmType, + /// The Cranelift IR type of the value stored in the global. pub ty: ir::Type, /// A flag indicating whether the value may change at runtime. pub mutability: bool, diff --git a/crates/c-api/src/extern.rs b/crates/c-api/src/extern.rs index 9f50f20f9019..d0a472ff6711 100644 --- a/crates/c-api/src/extern.rs +++ b/crates/c-api/src/extern.rs @@ -1,7 +1,7 @@ use crate::host_ref::HostRef; use crate::wasm_externkind_t; use crate::{wasm_externtype_t, wasm_func_t, wasm_global_t, wasm_memory_t, wasm_table_t}; -use wasmtime::{ExternType, Func, Global, Memory, Table}; +use wasmtime::{ExternType, Func, FuncType, Global, Memory, Table}; #[derive(Clone)] pub struct wasm_extern_t { @@ -12,7 +12,7 @@ wasmtime_c_api_macros::declare_ref!(wasm_extern_t); #[derive(Clone)] pub(crate) enum ExternHost { - Func(HostRef), + Func(HostRef>), Global(HostRef), Memory(HostRef), Table(HostRef), @@ -42,7 +42,14 @@ pub extern "C" fn wasm_extern_kind(e: &wasm_extern_t) -> wasm_externkind_t { #[no_mangle] pub extern "C" fn wasm_extern_type(e: &wasm_extern_t) -> Box { let ty = match &e.which { - ExternHost::Func(f) => ExternType::Func(f.borrow().ty()), + ExternHost::Func(f) => ExternType::Func({ + let f = f.borrow(); + if f.is_none() { + FuncType::new(Default::default(), Default::default()) + } else { + f.as_ref().unwrap().ty() + } + }), ExternHost::Global(f) => ExternType::Global(f.borrow().ty()), ExternHost::Table(f) => ExternType::Table(f.borrow().ty()), ExternHost::Memory(f) => ExternType::Memory(f.borrow().ty()), diff --git a/crates/c-api/src/func.rs b/crates/c-api/src/func.rs index 5f62ec30b270..e9d67a20566a 100644 --- a/crates/c-api/src/func.rs +++ b/crates/c-api/src/func.rs @@ -6,7 +6,7 @@ use std::ffi::c_void; use std::panic::{self, AssertUnwindSafe}; use std::ptr; use std::str; -use wasmtime::{Caller, Extern, Func, Trap}; +use wasmtime::{Caller, Extern, Func, FuncType, Trap}; #[derive(Clone)] #[repr(transparent)] @@ -64,7 +64,7 @@ impl wasm_func_t { } } - pub(crate) fn func(&self) -> &HostRef { + pub(crate) fn func(&self) -> &HostRef> { match &self.ext.which { ExternHost::Func(f) => f, _ => unsafe { std::hint::unreachable_unchecked() }, @@ -76,8 +76,8 @@ impl wasm_func_t { } } -impl From> for wasm_func_t { - fn from(func: HostRef) -> wasm_func_t { +impl From>> for wasm_func_t { + fn from(func: HostRef>) -> wasm_func_t { wasm_func_t { ext: wasm_extern_t { which: ExternHost::Func(func), @@ -108,7 +108,7 @@ fn create_function( } Ok(()) }); - Box::new(HostRef::new(store, func).into()) + Box::new(HostRef::new(store, Some(func)).into()) } #[no_mangle] @@ -172,19 +172,26 @@ pub unsafe extern "C" fn wasm_func_call( args: *const wasm_val_t, results: *mut wasm_val_t, ) -> *mut wasm_trap_t { - let func = wasm_func.func().borrow(); - let mut trap = ptr::null_mut(); - let error = wasmtime_func_call( - wasm_func, - args, - func.param_arity(), - results, - func.result_arity(), - &mut trap, - ); - match error { - Some(err) => Box::into_raw(err.to_trap(&wasm_func.ext.externref().store().unwrap())), - None => trap, + if let Some(func) = &*wasm_func.func().borrow() { + let mut trap = ptr::null_mut(); + let error = wasmtime_func_call( + wasm_func, + args, + func.param_arity(), + results, + func.result_arity(), + &mut trap, + ); + match error { + Some(err) => Box::into_raw(err.to_trap(&wasm_func.ext.externref().store().unwrap())), + None => trap, + } + } else { + let store = wasm_func.ext.externref().store().unwrap(); + Box::into_raw(Box::new(wasm_trap_t::new( + &store, + Trap::new("attempt to call null func ref"), + ))) } } @@ -213,6 +220,10 @@ fn _wasmtime_func_call( ) -> Option> { let store = &func.ext.externref().store().unwrap(); let func = func.func().borrow(); + let func = match func.as_ref() { + Some(f) => f, + None => return Some(Box::new(anyhow!("attempt to call null func ref").into())), + }; if results.len() != func.result_arity() { return Some(Box::new(anyhow!("wrong number of results provided").into())); } @@ -254,17 +265,32 @@ fn _wasmtime_func_call( #[no_mangle] pub extern "C" fn wasm_func_type(f: &wasm_func_t) -> Box { - Box::new(wasm_functype_t::new(f.func().borrow().ty())) + if let Some(f) = &*f.func().borrow() { + Box::new(wasm_functype_t::new(f.ty())) + } else { + Box::new(wasm_functype_t::new(FuncType::new( + Default::default(), + Default::default(), + ))) + } } #[no_mangle] pub extern "C" fn wasm_func_param_arity(f: &wasm_func_t) -> usize { - f.func().borrow().param_arity() + if let Some(f) = &*f.func().borrow() { + f.param_arity() + } else { + 0 + } } #[no_mangle] pub extern "C" fn wasm_func_result_arity(f: &wasm_func_t) -> usize { - f.func().borrow().result_arity() + if let Some(f) = &*f.func().borrow() { + f.result_arity() + } else { + 0 + } } #[no_mangle] @@ -281,7 +307,8 @@ pub unsafe extern "C" fn wasmtime_caller_export_get( let export = caller.caller.get_export(name)?; let store = caller.caller.store(); let which = match export { - Extern::Func(f) => ExternHost::Func(HostRef::new(&store, f)), + Extern::Func(None) => ExternHost::Func(HostRef::new(&store, None)), + Extern::Func(Some(f)) => ExternHost::Func(HostRef::new(&store, Some(f))), Extern::Global(g) => ExternHost::Global(HostRef::new(&store, g)), Extern::Memory(m) => ExternHost::Memory(HostRef::new(&store, m)), Extern::Table(t) => ExternHost::Table(HostRef::new(&store, t)), diff --git a/crates/c-api/src/linker.rs b/crates/c-api/src/linker.rs index 0152d2d49b21..ee360f3a6164 100644 --- a/crates/c-api/src/linker.rs +++ b/crates/c-api/src/linker.rs @@ -117,6 +117,6 @@ pub unsafe extern "C" fn wasmtime_linker_get_default( Err(_) => return bad_utf8(), }; handle_result(linker.get_default(name), |f| { - *func = Box::into_raw(Box::new(HostRef::new(linker.store(), f).into())) + *func = Box::into_raw(Box::new(HostRef::new(linker.store(), Some(f)).into())) }) } diff --git a/crates/c-api/src/wasi.rs b/crates/c-api/src/wasi.rs index 6daad6aaef1a..6d4ce6990ec5 100644 --- a/crates/c-api/src/wasi.rs +++ b/crates/c-api/src/wasi.rs @@ -359,7 +359,7 @@ pub extern "C" fn wasi_instance_bind_import<'a>( .entry(name.to_string()) .or_insert_with(|| { Box::new(wasm_extern_t { - which: ExternHost::Func(HostRef::new(store, export.clone())), + which: ExternHost::Func(HostRef::new(store, Some(export.clone()))), }) }); Some(entry) diff --git a/crates/environ/src/func_environ.rs b/crates/environ/src/func_environ.rs index ac01865771a9..e092d637f6b6 100644 --- a/crates/environ/src/func_environ.rs +++ b/crates/environ/src/func_environ.rs @@ -10,8 +10,8 @@ use cranelift_codegen::ir::{AbiParam, ArgumentPurpose, Function, InstBuilder, Si use cranelift_codegen::isa::{self, TargetFrontendConfig}; use cranelift_entity::EntityRef; use cranelift_wasm::{ - self, FuncIndex, GlobalIndex, GlobalVariable, MemoryIndex, SignatureIndex, TableIndex, - TargetEnvironment, WasmError, WasmResult, + self, FuncIndex, GlobalIndex, GlobalVariable, MemoryIndex, SignatureIndex, TableElementType, + TableIndex, TargetEnvironment, WasmError, WasmResult, }; #[cfg(feature = "lightbeam")] use cranelift_wasm::{DefinedFuncIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex}; @@ -68,6 +68,10 @@ macro_rules! declare_builtin_functions { AbiParam::new(self.reference_type) } + fn pointer(&self) -> AbiParam { + AbiParam::new(self.pointer_type) + } + fn i32(&self) -> AbiParam { AbiParam::new(I32) } @@ -161,10 +165,10 @@ declare_builtin_functions! { memory_init(vmctx, i32, i32, i32, i32, i32) -> (); /// Returns an index for wasm's `data.drop` instruction. data_drop(vmctx, i32) -> (); - /// Returns an index for Wasm's `table.grow` instruction. - table_grow(vmctx, i32, i32, reference) -> (i32); - /// Returns an index for Wasm's `ref.func` instruction. - ref_func(vmctx, i32) -> (reference); + /// Returns an index for Wasm's `table.grow` instruction for `funcref`s. + table_grow_funcref(vmctx, i32, i32, pointer) -> (i32); + /// Returns an index for Wasm's `table.grow` instruction for `externref`s. + table_grow_externref(vmctx, i32, i32, reference) -> (i32); } impl BuiltinFunctionIndex { @@ -586,9 +590,7 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m }); let element_size = match self.module.table_plans[index].style { - TableStyle::CallerChecksSignature => { - u64::from(self.offsets.size_of_vmcaller_checked_anyfunc()) - } + TableStyle::CallerChecksSignature => self.pointer_type().bytes() as _, }; Ok(func.create_table(ir::TableData { @@ -607,12 +609,25 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m delta: ir::Value, init_value: ir::Value, ) -> WasmResult { - let table_index_arg = pos.ins().iconst(I32, table_index.as_u32() as i64); + let (func_idx, func_sig) = match self.module.table_plans[table_index].table.ty { + TableElementType::Func => ( + BuiltinFunctionIndex::table_grow_funcref(), + self.builtin_function_signatures + .table_grow_funcref(&mut pos.func), + ), + TableElementType::Val(ty) => { + debug_assert_eq!(ty, self.reference_type()); + ( + BuiltinFunctionIndex::table_grow_externref(), + self.builtin_function_signatures + .table_grow_externref(&mut pos.func), + ) + } + }; - let func_idx = BuiltinFunctionIndex::table_grow(); let (vmctx, func_addr) = self.translate_load_builtin_function_address(&mut pos, func_idx); - let func_sig = self.builtin_function_signatures.table_grow(&mut pos.func); + let table_index_arg = pos.ins().iconst(I32, table_index.as_u32() as i64); let call_inst = pos.ins().call_indirect( func_sig, func_addr, @@ -663,17 +678,10 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m mut pos: cranelift_codegen::cursor::FuncCursor<'_>, func_index: FuncIndex, ) -> WasmResult { - let func_sig = self.builtin_function_signatures.ref_func(&mut pos.func); - let ref_func_index = BuiltinFunctionIndex::ref_func(); - let func_index_arg = pos.ins().iconst(I32, func_index.as_u32() as i64); - - let (vmctx, func_addr) = - self.translate_load_builtin_function_address(&mut pos, ref_func_index); - let call_inst = pos - .ins() - .call_indirect(func_sig, func_addr, &[vmctx, func_index_arg]); - - Ok(pos.func.dfg.first_result(call_inst)) + let vmctx = self.vmctx(&mut pos.func); + let vmctx = pos.ins().global_value(self.pointer_type(), vmctx); + let offset = self.offsets.vmctx_anyfunc(func_index); + Ok(pos.ins().iadd_imm(vmctx, i64::from(offset))) } fn translate_custom_global_get( @@ -843,12 +851,21 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m let table_entry_addr = pos.ins().table_addr(pointer_type, table, callee, 0); - // Dereference table_entry_addr to get the function address. + // Dereference the table entry to get the pointer to the + // `VMCallerCheckedAnyfunc`. + let anyfunc_ptr = + pos.ins() + .load(pointer_type, ir::MemFlags::trusted(), table_entry_addr, 0); + // Note: `anyfunc_ptr` is always non-null, even for null func refs! In + // that case, the pointed-to `VMCallerCheckedAnyfunc`'s `func_ptr` + // member is null, which is checked below. + + // Dereference anyfunc pointer to get the function address. let mem_flags = ir::MemFlags::trusted(); let func_addr = pos.ins().load( pointer_type, mem_flags, - table_entry_addr, + anyfunc_ptr, i32::from(self.offsets.vmcaller_checked_anyfunc_func_ptr()), ); @@ -875,7 +892,7 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m let callee_sig_id = pos.ins().load( sig_id_type, mem_flags, - table_entry_addr, + anyfunc_ptr, i32::from(self.offsets.vmcaller_checked_anyfunc_type_index()), ); @@ -892,7 +909,7 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m let vmctx = pos.ins().load( pointer_type, mem_flags, - table_entry_addr, + anyfunc_ptr, i32::from(self.offsets.vmcaller_checked_anyfunc_vmctx()), ); real_call_args.push(vmctx); diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index 0d7f08073c06..6efa5914577a 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -15,6 +15,7 @@ // tables: [VMTableDefinition; module.num_defined_tables], // memories: [VMMemoryDefinition; module.num_defined_memories], // globals: [VMGlobalDefinition; module.num_defined_globals], +// anyfuncs: [VMCallerCheckedAnyfunc; module.num_imported_functions + module.num_defined_functions], // builtins: VMBuiltinFunctionsArray, // } @@ -62,6 +63,8 @@ pub struct VMOffsets { pub num_imported_memories: u32, /// The number of imported globals in the module. pub num_imported_globals: u32, + /// The number of defined functions in the module. + pub num_defined_functions: u32, /// The number of defined tables in the module. pub num_defined_tables: u32, /// The number of defined memories in the module. @@ -80,6 +83,7 @@ impl VMOffsets { num_imported_tables: cast_to_u32(module.num_imported_tables), num_imported_memories: cast_to_u32(module.num_imported_memories), num_imported_globals: cast_to_u32(module.num_imported_globals), + num_defined_functions: cast_to_u32(module.functions.len()), num_defined_tables: cast_to_u32(module.table_plans.len()), num_defined_memories: cast_to_u32(module.memory_plans.len()), num_defined_globals: cast_to_u32(module.globals.len()), @@ -390,8 +394,8 @@ impl VMOffsets { align(offset, 16) } - /// The offset of the builtin functions array. - pub fn vmctx_builtin_functions_begin(&self) -> u32 { + /// The offset of the `anyfuncs` array. + pub fn vmctx_anyfuncs_begin(&self) -> u32 { self.vmctx_globals_begin() .checked_add( self.num_defined_globals @@ -401,6 +405,19 @@ impl VMOffsets { .unwrap() } + /// The offset of the builtin functions array. + pub fn vmctx_builtin_functions_begin(&self) -> u32 { + self.vmctx_anyfuncs_begin() + .checked_add( + self.num_imported_functions + .checked_add(self.num_defined_functions) + .unwrap() + .checked_mul(u32::from(self.size_of_vmcaller_checked_anyfunc())) + .unwrap(), + ) + .unwrap() + } + /// Return the size of the `VMContext` allocation. pub fn size_of_vmctx(&self) -> u32 { self.vmctx_builtin_functions_begin() @@ -516,6 +533,19 @@ impl VMOffsets { .unwrap() } + /// Return the offset to the `VMCallerCheckedAnyfunc` for the given function + /// index (either imported or defined). + pub fn vmctx_anyfunc(&self, index: FuncIndex) -> u32 { + self.vmctx_anyfuncs_begin() + .checked_add( + index + .as_u32() + .checked_mul(u32::from(self.size_of_vmcaller_checked_anyfunc())) + .unwrap(), + ) + .unwrap() + } + /// Return the offset to the `body` field in `*const VMFunctionBody` index `index`. pub fn vmctx_vmfunction_import_body(&self, index: FuncIndex) -> u32 { self.vmctx_vmfunction_import(index) diff --git a/crates/fuzzing/src/oracles.rs b/crates/fuzzing/src/oracles.rs index 19fcc6b1f242..60b9761641de 100644 --- a/crates/fuzzing/src/oracles.rs +++ b/crates/fuzzing/src/oracles.rs @@ -170,7 +170,7 @@ pub fn differential_execution( for (name, f) in instance.exports().filter_map(|e| { let name = e.name(); - e.into_func().map(|f| (name, f)) + e.into_func().and_then(|f| f).map(|f| (name, f)) }) { // Always call the hang limit initializer first, so that we don't // infinite loop when calling another export. @@ -193,7 +193,7 @@ pub fn differential_execution( fn init_hang_limit(instance: &Instance) { match instance.get_export("hangLimitInitializer") { None => return, - Some(Extern::Func(f)) => { + Some(Extern::Func(Some(f))) => { f.call(&[]) .expect("initializing the hang limit should not fail"); } @@ -365,7 +365,7 @@ pub fn make_api_calls(api: crate::generators::api::ApiCalls) { let funcs = instance .exports() .filter_map(|e| match e.into_extern() { - Extern::Func(f) => Some(f.clone()), + Extern::Func(Some(f)) => Some(f.clone()), _ => None, }) .collect::>(); diff --git a/crates/fuzzing/src/oracles/dummy.rs b/crates/fuzzing/src/oracles/dummy.rs index 1dc35ed3ad78..67867f045b36 100644 --- a/crates/fuzzing/src/oracles/dummy.rs +++ b/crates/fuzzing/src/oracles/dummy.rs @@ -13,7 +13,7 @@ pub fn dummy_imports<'module>( import_tys .map(|imp| { Ok(match imp.ty() { - ExternType::Func(func_ty) => Extern::Func(dummy_func(&store, func_ty)), + ExternType::Func(func_ty) => Extern::Func(Some(dummy_func(&store, func_ty))), ExternType::Global(global_ty) => Extern::Global(dummy_global(&store, global_ty)?), ExternType::Table(table_ty) => Extern::Table(dummy_table(&store, table_ty)?), ExternType::Memory(mem_ty) => Extern::Memory(dummy_memory(&store, mem_ty)), diff --git a/crates/jit/src/imports.rs b/crates/jit/src/imports.rs index cf0f497e1535..9bd4b8e4714c 100644 --- a/crates/jit/src/imports.rs +++ b/crates/jit/src/imports.rs @@ -32,7 +32,9 @@ pub fn resolve_imports( match (import, &export) { (EntityIndex::Function(func_index), Some(Export::Function(f))) => { let import_signature = module.local.native_func_signature(*func_index); - let signature = signatures.lookup_native(f.signature).unwrap(); + let signature = signatures + .lookup_native(unsafe { f.anyfunc.as_ref().type_index }) + .unwrap(); if signature != *import_signature { // TODO: If the difference is in the calling convention, // we could emit a wrapper function to fix it up. @@ -43,8 +45,8 @@ pub fn resolve_imports( ))); } function_imports.push(VMFunctionImport { - body: f.address, - vmctx: f.vmctx, + body: unsafe { f.anyfunc.as_ref().func_ptr }, + vmctx: unsafe { f.anyfunc.as_ref().vmctx }, }); } (EntityIndex::Function(_), Some(_)) => { @@ -169,16 +171,20 @@ fn is_global_compatible(exported: &Global, imported: &Global) -> bool { } let Global { + wasm_ty: exported_wasm_ty, ty: exported_ty, mutability: exported_mutability, initializer: _exported_initializer, } = exported; let Global { + wasm_ty: imported_wasm_ty, ty: imported_ty, mutability: imported_mutability, initializer: _imported_initializer, } = imported; - exported_ty == imported_ty && imported_mutability == exported_mutability + exported_wasm_ty == imported_wasm_ty + && exported_ty == imported_ty + && imported_mutability == exported_mutability } fn is_table_element_type_compatible( diff --git a/crates/runtime/src/export.rs b/crates/runtime/src/export.rs index 8cf42b8f8f24..fa7429eb4fbf 100644 --- a/crates/runtime/src/export.rs +++ b/crates/runtime/src/export.rs @@ -1,7 +1,7 @@ use crate::vmcontext::{ - VMContext, VMFunctionBody, VMGlobalDefinition, VMMemoryDefinition, VMSharedSignatureIndex, - VMTableDefinition, + VMCallerCheckedAnyfunc, VMContext, VMGlobalDefinition, VMMemoryDefinition, VMTableDefinition, }; +use std::ptr::NonNull; use wasmtime_environ::wasm::Global; use wasmtime_environ::{MemoryPlan, TablePlan}; @@ -24,14 +24,8 @@ pub enum Export { /// A function export value. #[derive(Debug, Clone)] pub struct ExportFunction { - /// The address of the native-code function. - pub address: *const VMFunctionBody, - /// Pointer to the containing `VMContext`. - pub vmctx: *mut VMContext, - /// The function signature declaration, used for compatibilty checking. - /// - /// Note that this indexes within the module associated with `vmctx`. - pub signature: VMSharedSignatureIndex, + /// The `VMCallerCheckedAnyfunc` for this exported function. + pub anyfunc: NonNull, } impl From for Export { diff --git a/crates/runtime/src/externref.rs b/crates/runtime/src/externref.rs index a085b2c4d697..a7bf80b6b376 100644 --- a/crates/runtime/src/externref.rs +++ b/crates/runtime/src/externref.rs @@ -1124,6 +1124,7 @@ mod tests { num_imported_tables: 0, num_imported_memories: 0, num_imported_globals: 0, + num_defined_functions: 0, num_defined_tables: 0, num_defined_memories: 0, num_defined_globals: 0, @@ -1150,6 +1151,7 @@ mod tests { num_imported_tables: 0, num_imported_memories: 0, num_imported_globals: 0, + num_defined_functions: 0, num_defined_tables: 0, num_defined_memories: 0, num_defined_globals: 0, diff --git a/crates/runtime/src/funcref.rs b/crates/runtime/src/funcref.rs deleted file mode 100644 index 54445a3641de..000000000000 --- a/crates/runtime/src/funcref.rs +++ /dev/null @@ -1,68 +0,0 @@ -use crate::{VMCallerCheckedAnyfunc, VMExternRef}; -use std::any::Any; -use std::convert::TryFrom; -use std::fmt; - -/// A `VMFuncRef` is a `funcref` as compiled Wasm code sees it. -/// -/// Internally, a `VMFuncRef` is a `VMExternRef` that always points to a -/// `VMCallerCheckedAnyfunc`. This means that it is managed by the same GC -/// discipline that `VMExternRef` is managed by. It also means that it is -/// represented with a single word (either an `r32` or an `r64` depending on the -/// target pointer width) and is therefore suitable for Wasm code to manipulate -/// on-stack and pass into libcalls (since Cranelift does not support compound -/// structures). -#[derive(Clone)] -#[repr(transparent)] -pub struct VMFuncRef { - inner: VMExternRef, -} - -impl VMFuncRef { - /// Create a new `funcref` from the given `VMCallerCheckedAnyfunc`. - pub fn new(func: VMCallerCheckedAnyfunc) -> Self { - VMFuncRef { - inner: VMExternRef::new(func), - } - } - - /// Get this `funcref`'s inner `VMCallerCheckedAnyfunc`. - pub fn as_func(&self) -> &VMCallerCheckedAnyfunc { - // We always control the inner `VMExternRef`, so we know it is always a - // `VMCallerCheckedAnyfunc`, and we can do an unsafe cast here. - debug_assert!(self.inner.is::()); - let p: *const dyn Any = &*self.inner as _; - let p = p as *const VMCallerCheckedAnyfunc; - unsafe { &*p } - } - - /// Get this `funcref`'s inner `VMExternRef`. - pub fn as_externref(&self) -> &VMExternRef { - &self.inner - } -} - -impl fmt::Debug for VMFuncRef { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let func = self.as_func(); - f.debug_struct("VMFuncRef").field("func", &func).finish() - } -} - -impl From for VMExternRef { - fn from(f: VMFuncRef) -> VMExternRef { - f.inner - } -} - -impl TryFrom for VMFuncRef { - type Error = VMExternRef; - - fn try_from(r: VMExternRef) -> Result { - if r.is::() { - Ok(VMFuncRef { inner: r }) - } else { - Err(r) - } - } -} diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index 85becbbe4715..dd1e151caed4 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -21,6 +21,7 @@ use std::any::Any; use std::cell::RefCell; use std::collections::HashMap; use std::convert::TryFrom; +use std::ptr::NonNull; use std::sync::Arc; use std::{mem, ptr, slice}; use thiserror::Error; @@ -55,7 +56,7 @@ pub(crate) struct Instance { /// Passive elements in this instantiation. As `elem.drop`s happen, these /// entries get removed. A missing entry is considered equivalent to an /// empty slice. - passive_elements: RefCell>>, + passive_elements: RefCell]>>>, /// Passive data segments from our module. As `data.drop`s happen, entries /// get removed. A missing entry is considered equivalent to an empty slice. @@ -74,6 +75,9 @@ pub(crate) struct Instance { /// interrupted. pub(crate) interrupts: Arc, + /// A canonical `ref.null func` value for this instance. + pub(crate) null_func_ref: Box, + /// Additional context used by compiled wasm code. This field is last, and /// represents a dynamically-sized array that extends beyond the nominal /// end of the struct (similar to a flexible array member). @@ -100,6 +104,11 @@ impl Instance { &self.module } + pub(crate) fn null_func_ref(&self) -> NonNull { + let p = &*self.null_func_ref as *const VMCallerCheckedAnyfunc as *mut _; + unsafe { NonNull::new_unchecked(p) } + } + /// Return a pointer to the `VMSharedSignatureIndex`s. fn signature_ids_ptr(&self) -> *mut VMSharedSignatureIndex { unsafe { self.vmctx_plus_offset(self.offsets.vmctx_signature_ids_begin()) } @@ -274,23 +283,10 @@ impl Instance { pub fn lookup_by_declaration(&self, export: &EntityIndex) -> Export { match export { EntityIndex::Function(index) => { - let signature = self.signature_id(self.module.local.functions[*index]); - let (address, vmctx) = - if let Some(def_index) = self.module.local.defined_func_index(*index) { - ( - self.finished_functions[def_index] as *const _, - self.vmctx_ptr(), - ) - } else { - let import = self.imported_function(*index); - (import.body, import.vmctx) - }; - ExportFunction { - address, - signature, - vmctx, - } - .into() + let anyfunc = self.get_caller_checked_anyfunc(*index).unwrap(); + let anyfunc = + NonNull::new(anyfunc as *const VMCallerCheckedAnyfunc as *mut _).unwrap(); + ExportFunction { anyfunc }.into() } EntityIndex::Table(index) => { let (definition, vmctx) = @@ -519,30 +515,27 @@ impl Instance { Layout::from_size_align(size, align).unwrap() } - /// Get a `VMCallerCheckedAnyfunc` for the given `FuncIndex`. - pub(crate) fn get_caller_checked_anyfunc(&self, index: FuncIndex) -> VMCallerCheckedAnyfunc { + /// Get a `&VMCallerCheckedAnyfunc` for the given `FuncIndex`. + /// + /// Returns `None` if the index is the reserved index value. + /// + /// The returned reference is a stable reference that won't be moved and can + /// be passed into JIT code. + pub(crate) fn get_caller_checked_anyfunc( + &self, + index: FuncIndex, + ) -> Option<&VMCallerCheckedAnyfunc> { if index == FuncIndex::reserved_value() { - return VMCallerCheckedAnyfunc::default(); + return None; } - let sig = self.module.local.functions[index]; - let type_index = self.signature_id(sig); + let anyfunc: &VMCallerCheckedAnyfunc = unsafe { &*self.anyfunc_ptr(index) }; + debug_assert!(!anyfunc.func_ptr.is_null()); + Some(anyfunc) + } - let (func_ptr, vmctx) = if let Some(def_index) = self.module.local.defined_func_index(index) - { - ( - self.finished_functions[def_index] as *const _, - self.vmctx_ptr(), - ) - } else { - let import = self.imported_function(index); - (import.body, import.vmctx) - }; - VMCallerCheckedAnyfunc { - func_ptr, - type_index, - vmctx, - } + unsafe fn anyfunc_ptr(&self, index: FuncIndex) -> *mut VMCallerCheckedAnyfunc { + self.vmctx_plus_offset(self.offsets.vmctx_anyfunc(index)) } /// The `table.init` operation: initializes a portion of a table with a @@ -580,7 +573,7 @@ impl Instance { // TODO(#983): investigate replacing this get/set loop with a `memcpy`. for (dst, src) in (dst..dst + len).zip(src..src + len) { table - .set(dst, TableElement::FuncRef(elem[src as usize].clone())) + .set(dst, TableElement::FuncRef(elem[src as usize])) .expect("should never panic because we already did the bounds check above"); } @@ -842,7 +835,12 @@ impl InstanceHandle { debug_assert!(!externref_activations_table.is_null()); debug_assert!(!stack_map_registry.is_null()); - let tables = create_tables(&module); + let null_func_ref = Box::new(VMCallerCheckedAnyfunc::null()); + + let tables = create_tables( + &module, + NonNull::new_unchecked(&*null_func_ref as *const VMCallerCheckedAnyfunc as *mut _), + ); let memories = create_memories(&module, mem_creator.unwrap_or(&DefaultMemoryCreator {}))?; let vmctx_tables = tables @@ -876,6 +874,7 @@ impl InstanceHandle { trampolines, host_state, interrupts, + null_func_ref, vmctx: VMContext {}, }; let layout = instance.alloc_layout(); @@ -938,6 +937,30 @@ impl InstanceHandle { *instance.externref_activations_table() = externref_activations_table; *instance.stack_map_registry() = stack_map_registry; + for (index, sig) in instance.module.local.functions.iter() { + let type_index = instance.signature_id(*sig); + + let (func_ptr, vmctx) = + if let Some(def_index) = instance.module.local.defined_func_index(index) { + ( + instance.finished_functions[def_index] as *const _, + instance.vmctx_ptr(), + ) + } else { + let import = instance.imported_function(index); + (import.body, import.vmctx) + }; + + ptr::write( + instance.anyfunc_ptr(index), + VMCallerCheckedAnyfunc { + func_ptr, + type_index, + vmctx, + }, + ); + } + // Perform infallible initialization in this constructor, while fallible // initialization is deferred to the `initialize` method. initialize_passive_elements(instance); @@ -1208,12 +1231,15 @@ fn check_memory_init_bounds( } /// Allocate memory for just the tables of the current module. -fn create_tables(module: &Module) -> BoxedSlice { +fn create_tables( + module: &Module, + null_func_ref: NonNull, +) -> BoxedSlice { let num_imports = module.local.num_imported_tables; let mut tables: PrimaryMap = PrimaryMap::with_capacity(module.local.table_plans.len() - num_imports); for table in &module.local.table_plans.values().as_slice()[num_imports..] { - tables.push(Table::new(table)); + tables.push(Table::new(table, null_func_ref)); } tables.into_boxed_slice() } @@ -1252,12 +1278,15 @@ fn initialize_tables(instance: &Instance) -> Result<(), InstantiationError> { } for (i, func_idx) in init.elements.iter().enumerate() { - let anyfunc = instance.get_caller_checked_anyfunc(*func_idx); + let anyfunc = instance.get_caller_checked_anyfunc(*func_idx).map_or( + instance.null_func_ref(), + |f: &VMCallerCheckedAnyfunc| { + let p = f as *const VMCallerCheckedAnyfunc as *mut _; + unsafe { NonNull::new_unchecked(p) } + }, + ); table - .set( - u32::try_from(start + i).unwrap(), - TableElement::FuncRef(anyfunc), - ) + .set(u32::try_from(start + i).unwrap(), anyfunc.into()) .unwrap(); } } @@ -1286,7 +1315,15 @@ fn initialize_passive_elements(instance: &Instance) { *idx, segments .iter() - .map(|s| instance.get_caller_checked_anyfunc(*s)) + .map(|s| { + instance.get_caller_checked_anyfunc(*s).map_or( + instance.null_func_ref(), + |f: &VMCallerCheckedAnyfunc| { + let p = f as *const _ as *mut _; + unsafe { NonNull::new_unchecked(p) } + }, + ) + }) .collect(), ) }), diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index 87f7be0f05df..c12021df2d0e 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -23,7 +23,6 @@ mod export; mod externref; -mod funcref; mod imports; mod instance; mod jit_int; @@ -39,7 +38,6 @@ pub mod libcalls; pub use crate::export::*; pub use crate::externref::*; -pub use crate::funcref::*; pub use crate::imports::Imports; pub use crate::instance::{InstanceHandle, InstantiationError, LinkError}; pub use crate::jit_int::GdbJitImageRegistration; @@ -58,3 +56,14 @@ pub use crate::vmcontext::{ /// Version number of this crate. pub const VERSION: &str = env!("CARGO_PKG_VERSION"); + +/// The Cranelift IR type used for reference types for this target architecture. +pub fn ref_type() -> wasmtime_environ::ir::Type { + if cfg!(target_pointer_width = "32") { + wasmtime_environ::ir::types::R32 + } else if cfg!(target_pointer_width = "64") { + wasmtime_environ::ir::types::R64 + } else { + unreachable!() + } +} diff --git a/crates/runtime/src/libcalls.rs b/crates/runtime/src/libcalls.rs index a2ebb3fb109f..b78fa0792ee4 100644 --- a/crates/runtime/src/libcalls.rs +++ b/crates/runtime/src/libcalls.rs @@ -57,14 +57,12 @@ //! ``` use crate::externref::VMExternRef; -use crate::funcref::VMFuncRef; use crate::table::Table; use crate::traphandlers::raise_lib_trap; use crate::vmcontext::{VMCallerCheckedAnyfunc, VMContext}; -use std::convert::TryFrom; -use wasmtime_environ::ir; +use std::ptr::NonNull; use wasmtime_environ::wasm::{ - DataIndex, DefinedMemoryIndex, ElemIndex, FuncIndex, MemoryIndex, TableElementType, TableIndex, + DataIndex, DefinedMemoryIndex, ElemIndex, MemoryIndex, TableElementType, TableIndex, }; /// Implementation of f32.ceil @@ -231,45 +229,36 @@ pub unsafe extern "C" fn wasmtime_imported_memory32_size( instance.imported_memory_size(memory_index) } -/// Implementation of `table.grow` for `externref`s. +/// Implementation of `table.grow`. pub unsafe extern "C" fn wasmtime_table_grow( vmctx: *mut VMContext, table_index: u32, delta: u32, + // NB: we don't know whether this is a pointer to a `VMCallerCheckedAnyfunc` + // or is a `VMExternRef` until we look at the table type. init_value: *mut u8, ) -> u32 { - let init_value = if init_value.is_null() { - None - } else { - Some(VMExternRef::clone_from_raw(init_value)) - }; - let instance = (&mut *vmctx).instance(); let table_index = TableIndex::from_u32(table_index); match instance.table_element_type(table_index) { TableElementType::Func => { - let func = match init_value { - None => VMCallerCheckedAnyfunc::default(), - Some(r) => VMFuncRef::try_from(r) - .expect("due to validation") - .as_func() - .clone(), - }; + let func = init_value as *mut VMCallerCheckedAnyfunc; + // Note: null func refs are a *non-null* pointer to a + // `VMCallerCheckedAnyfunc` whose `func_ptr` member is null. + let func = NonNull::new(func).expect("`funcref` pointers are always non-null"); instance .table_grow(table_index, delta, func.into()) .unwrap_or(-1_i32 as u32) } TableElementType::Val(ty) => { - assert_eq!( - ty, - if cfg!(target_pointer_width = "32") { - ir::types::R32 - } else if cfg!(target_pointer_width = "64") { - ir::types::R64 - } else { - unreachable!() - } - ); + debug_assert_eq!(ty, crate::ref_type()); + + let init_value = if init_value.is_null() { + None + } else { + Some(VMExternRef::clone_from_raw(init_value)) + }; + instance .table_grow(table_index, delta, init_value.into()) .unwrap_or(-1_i32 as u32) @@ -424,21 +413,3 @@ pub unsafe extern "C" fn wasmtime_data_drop(vmctx: *mut VMContext, data_index: u let instance = (&mut *vmctx).instance(); instance.data_drop(data_index) } - -/// Implementation of `ref.func`. -pub unsafe extern "C" fn wasmtime_ref_func(vmctx: *mut VMContext, func_index: u32) -> *mut u8 { - let instance = (&mut *vmctx).instance(); - - let func_index = FuncIndex::from_u32(func_index); - let func = instance.get_caller_checked_anyfunc(func_index); - - let funcref = VMFuncRef::new(func); - let externref: VMExternRef = funcref.into(); - let raw_ref = externref.as_raw(); - - let activations_table = &**instance.externref_activations_table(); - let registry = &**instance.stack_map_registry(); - activations_table.insert_with_gc(externref, registry); - - raw_ref -} diff --git a/crates/runtime/src/table.rs b/crates/runtime/src/table.rs index c4a268edf614..914cf87346e3 100644 --- a/crates/runtime/src/table.rs +++ b/crates/runtime/src/table.rs @@ -6,6 +6,7 @@ use crate::vmcontext::{VMCallerCheckedAnyfunc, VMTableDefinition}; use crate::{Trap, VMExternRef}; use std::cell::RefCell; use std::convert::{TryFrom, TryInto}; +use std::ptr::NonNull; use wasmtime_environ::wasm::TableElementType; use wasmtime_environ::{ir, TablePlan, TableStyle}; @@ -20,35 +21,30 @@ pub struct Table { #[derive(Clone, Debug)] pub enum TableElement { /// A `funcref`. - FuncRef(VMCallerCheckedAnyfunc), + FuncRef(NonNull), /// An `exrernref`. ExternRef(Option), } #[derive(Debug)] enum TableElements { - FuncRefs(Vec), + FuncRefs(Vec>), ExternRefs(Vec>), } impl Table { /// Create a new table instance with specified minimum and maximum number of elements. - pub fn new(plan: &TablePlan) -> Self { - let elements = - RefCell::new(match plan.table.ty { - TableElementType::Func => TableElements::FuncRefs(vec![ - VMCallerCheckedAnyfunc::default(); - usize::try_from(plan.table.minimum).unwrap() - ]), - TableElementType::Val(ty) - if (cfg!(target_pointer_width = "64") && ty == ir::types::R64) - || (cfg!(target_pointer_width = "32") && ty == ir::types::R32) => - { - let min = usize::try_from(plan.table.minimum).unwrap(); - TableElements::ExternRefs(vec![None; min]) - } - TableElementType::Val(ty) => unimplemented!("unsupported table type ({})", ty), - }); + pub fn new(plan: &TablePlan, null_func_ref: NonNull) -> Self { + debug_assert!(unsafe { (*null_func_ref.as_ptr()).func_ptr.is_null() }); + + let min = usize::try_from(plan.table.minimum).unwrap(); + let elements = RefCell::new(match plan.table.ty { + TableElementType::Func => TableElements::FuncRefs(vec![null_func_ref; min]), + TableElementType::Val(ty) => { + debug_assert_eq!(ty, crate::ref_type()); + TableElements::ExternRefs(vec![None; min]) + } + }); match plan.style { TableStyle::CallerChecksSignature => Self { elements, @@ -61,10 +57,7 @@ impl Table { pub fn element_type(&self) -> TableElementType { match &*self.elements.borrow() { TableElements::FuncRefs(_) => TableElementType::Func, - #[cfg(target_pointer_width = "32")] - TableElements::ExternRefs(_) => TableElementType::Val(ir::types::R32), - #[cfg(target_pointer_width = "64")] - TableElements::ExternRefs(_) => TableElementType::Val(ir::types::R64), + TableElements::ExternRefs(_) => TableElementType::Val(crate::ref_type()), } } @@ -210,7 +203,7 @@ impl Table { } } -impl TryFrom for VMCallerCheckedAnyfunc { +impl TryFrom for NonNull { type Error = TableElement; fn try_from(e: TableElement) -> Result { @@ -232,8 +225,8 @@ impl TryFrom for Option { } } -impl From for TableElement { - fn from(f: VMCallerCheckedAnyfunc) -> TableElement { +impl From> for TableElement { + fn from(f: NonNull) -> TableElement { TableElement::FuncRef(f) } } diff --git a/crates/runtime/src/vmcontext.rs b/crates/runtime/src/vmcontext.rs index 69cc7e74dd19..0427bd9d812a 100644 --- a/crates/runtime/src/vmcontext.rs +++ b/crates/runtime/src/vmcontext.rs @@ -3,8 +3,9 @@ use crate::instance::Instance; use std::any::Any; +use std::ptr; use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; -use std::{ptr, u32}; +use std::u32; use wasmtime_environ::BuiltinFunctionIndex; /// An imported function. @@ -475,6 +476,10 @@ impl Default for VMSharedSignatureIndex { #[repr(C)] pub struct VMCallerCheckedAnyfunc { /// Function body. + /// + /// This member is nullable. When null, this `VMCallerCheckedAnyfunc` + /// corresponds to `ref.null func`, and none of the other members of this + /// struct are valid. pub func_ptr: *const VMFunctionBody, /// Function signature id. pub type_index: VMSharedSignatureIndex, @@ -483,6 +488,17 @@ pub struct VMCallerCheckedAnyfunc { // If more elements are added here, remember to add offset_of tests below! } +impl VMCallerCheckedAnyfunc { + /// Create a null function reference value. + pub fn null() -> VMCallerCheckedAnyfunc { + VMCallerCheckedAnyfunc { + func_ptr: ptr::null_mut(), + type_index: VMSharedSignatureIndex::default(), + vmctx: ptr::null_mut(), + } + } +} + #[cfg(test)] mod test_vmcaller_checked_anyfunc { use super::VMCallerCheckedAnyfunc; @@ -513,16 +529,6 @@ mod test_vmcaller_checked_anyfunc { } } -impl Default for VMCallerCheckedAnyfunc { - fn default() -> Self { - Self { - func_ptr: ptr::null_mut(), - type_index: Default::default(), - vmctx: ptr::null_mut(), - } - } -} - /// An array that stores addresses of builtin functions. We translate code /// to use indirect calls. This way, we don't have to patch the code. #[repr(C)] @@ -549,7 +555,10 @@ impl VMBuiltinFunctionsArray { ptrs[BuiltinFunctionIndex::imported_memory32_size().index() as usize] = wasmtime_imported_memory32_size as usize; ptrs[BuiltinFunctionIndex::table_copy().index() as usize] = wasmtime_table_copy as usize; - ptrs[BuiltinFunctionIndex::table_grow().index() as usize] = wasmtime_table_grow as usize; + ptrs[BuiltinFunctionIndex::table_grow_funcref().index() as usize] = + wasmtime_table_grow as usize; + ptrs[BuiltinFunctionIndex::table_grow_externref().index() as usize] = + wasmtime_table_grow as usize; ptrs[BuiltinFunctionIndex::table_init().index() as usize] = wasmtime_table_init as usize; ptrs[BuiltinFunctionIndex::elem_drop().index() as usize] = wasmtime_elem_drop as usize; ptrs[BuiltinFunctionIndex::defined_memory_copy().index() as usize] = @@ -561,7 +570,6 @@ impl VMBuiltinFunctionsArray { wasmtime_imported_memory_fill as usize; ptrs[BuiltinFunctionIndex::memory_init().index() as usize] = wasmtime_memory_init as usize; ptrs[BuiltinFunctionIndex::data_drop().index() as usize] = wasmtime_data_drop as usize; - ptrs[BuiltinFunctionIndex::ref_func().index() as usize] = wasmtime_ref_func as usize; if cfg!(debug_assertions) { for i in 0..ptrs.len() { diff --git a/crates/wasmtime/src/externals.rs b/crates/wasmtime/src/externals.rs index 29faa28dd2a2..d4d1695315cc 100644 --- a/crates/wasmtime/src/externals.rs +++ b/crates/wasmtime/src/externals.rs @@ -7,6 +7,7 @@ use crate::{ ValType, }; use anyhow::{anyhow, bail, Result}; +use std::ptr::NonNull; use std::slice; use wasmtime_environ::wasm; use wasmtime_runtime::{self as runtime, InstanceHandle}; @@ -23,7 +24,7 @@ use wasmtime_runtime::{self as runtime, InstanceHandle}; #[derive(Clone)] pub enum Extern { /// A WebAssembly `func` which can be called. - Func(Func), + Func(Option), /// A WebAssembly `global` which acts like a `Cell` of sorts, supporting /// `get` and `set` operations. Global(Global), @@ -36,8 +37,11 @@ pub enum Extern { impl Extern { /// Returns the underlying `Func`, if this external is a function. /// + /// Returns `Some(None)` if this is the null reference to a function, + /// created in Wasm via the `ref.null func` instruction. + /// /// Returns `None` if this is not a function. - pub fn into_func(self) -> Option { + pub fn into_func(self) -> Option> { match self { Extern::Func(func) => Some(func), _ => None, @@ -77,16 +81,25 @@ impl Extern { /// Returns the type associated with this `Extern`. pub fn ty(&self) -> ExternType { match self { - Extern::Func(ft) => ExternType::Func(ft.ty()), + Extern::Func(None) => ExternType::Func(crate::types::FuncType::new( + Default::default(), + Default::default(), + )), + Extern::Func(Some(ft)) => ExternType::Func(ft.ty()), Extern::Memory(ft) => ExternType::Memory(ft.ty()), Extern::Table(tt) => ExternType::Table(tt.ty()), Extern::Global(gt) => ExternType::Global(gt.ty()), } } - pub(crate) fn get_wasmtime_export(&self) -> wasmtime_runtime::Export { + pub(crate) fn get_wasmtime_export(&self, store: &Store) -> wasmtime_runtime::Export { match self { - Extern::Func(f) => f.wasmtime_function().clone().into(), + Extern::Func(None) => { + wasmtime_runtime::Export::Function(wasmtime_runtime::ExportFunction { + anyfunc: store.null_func_ref(), + }) + } + Extern::Func(Some(f)) => f.wasmtime_function().clone().into(), Extern::Global(g) => g.wasmtime_export.clone().into(), Extern::Memory(m) => m.wasmtime_export.clone().into(), Extern::Table(t) => t.wasmtime_export.clone().into(), @@ -115,7 +128,8 @@ impl Extern { pub(crate) fn comes_from_same_store(&self, store: &Store) -> bool { let my_store = match self { - Extern::Func(f) => f.store(), + Extern::Func(None) => return true, + Extern::Func(Some(f)) => f.store(), Extern::Global(g) => &g.instance.store, Extern::Memory(m) => &m.instance.store, Extern::Table(t) => &t.instance.store, @@ -126,6 +140,12 @@ impl Extern { impl From for Extern { fn from(r: Func) -> Self { + Extern::Func(Some(r)) + } +} + +impl From> for Extern { + fn from(r: Option) -> Self { Extern::Func(r) } } @@ -205,7 +225,7 @@ impl Global { /// Returns the value type of this `global`. pub fn val_type(&self) -> ValType { - ValType::from_wasmtime_type(self.wasmtime_export.global.ty) + ValType::from_wasm_type(&self.wasmtime_export.global.wasm_ty) .expect("core wasm type should be supported") } @@ -298,14 +318,10 @@ fn set_table_item( instance: &InstanceHandle, table_index: wasm::DefinedTableIndex, item_index: u32, - item: wasmtime_runtime::VMCallerCheckedAnyfunc, + item: NonNull, ) -> Result<()> { instance - .table_set( - table_index, - item_index, - runtime::TableElement::FuncRef(item), - ) + .table_set(table_index, item_index, item.into()) .map_err(|()| anyhow!("table element index out of bounds")) } @@ -329,7 +345,7 @@ impl Table { let definition = unsafe { &*wasmtime_export.definition }; let index = instance.table_index(definition); for i in 0..definition.current_elements { - set_table_item(&instance, index, i, item.clone())?; + set_table_item(&instance, index, i, item)?; } Ok(Table { @@ -356,7 +372,7 @@ impl Table { let item = self.instance.table_get(table_index, index)?; match item { runtime::TableElement::FuncRef(f) => { - Some(from_checked_anyfunc(f, &self.instance.store)) + Some(unsafe { from_checked_anyfunc(f, &self.instance.store) }) } runtime::TableElement::ExternRef(None) => Some(Val::ExternRef(None)), runtime::TableElement::ExternRef(Some(x)) => Some(Val::ExternRef(Some(ExternRef { @@ -398,8 +414,7 @@ impl Table { let orig_size = match self.ty().element() { ValType::FuncRef => { let init = into_checked_anyfunc(init, &self.instance.store)?; - self.instance - .defined_table_grow(index, delta, runtime::TableElement::FuncRef(init)) + self.instance.defined_table_grow(index, delta, init.into()) } ValType::ExternRef => { let init = match init { @@ -1000,7 +1015,7 @@ impl<'instance> Export<'instance> { /// Consume this `Export` and return the contained `Func`, if it's a function, /// or `None` otherwise. - pub fn into_func(self) -> Option { + pub fn into_func(self) -> Option> { self.definition.into_func() } diff --git a/crates/wasmtime/src/func.rs b/crates/wasmtime/src/func.rs index 53c7e5ab01d7..b3afb1e80470 100644 --- a/crates/wasmtime/src/func.rs +++ b/crates/wasmtime/src/func.rs @@ -6,10 +6,11 @@ use std::cmp::max; use std::fmt; use std::mem; use std::panic::{self, AssertUnwindSafe}; -use std::ptr; +use std::ptr::{self, NonNull}; use std::rc::Weak; -use wasmtime_runtime::{raise_user_trap, ExportFunction, VMTrampoline}; -use wasmtime_runtime::{Export, InstanceHandle, VMContext, VMFunctionBody}; +use wasmtime_runtime::{ + raise_user_trap, Export, InstanceHandle, VMContext, VMFunctionBody, VMTrampoline, +}; /// A WebAssembly function which can be called. /// @@ -140,8 +141,8 @@ use wasmtime_runtime::{Export, InstanceHandle, VMContext, VMFunctionBody}; #[derive(Clone)] pub struct Func { instance: StoreInstanceHandle, - export: ExportFunction, trampoline: VMTrampoline, + export: wasmtime_runtime::ExportFunction, } macro_rules! getters { @@ -157,6 +158,8 @@ macro_rules! getters { $($args: WasmTy,)* R: WasmTy, { + debug_assert!(unsafe { !self.export.anyfunc.as_ref().func_ptr.is_null() }, "b"); + // Verify all the paramers match the expected parameters, and that // there are no extra parameters... let ty = self.ty(); @@ -175,10 +178,11 @@ macro_rules! getters { .context("Type mismatch in return type")?; ensure!(results.next().is_none(), "Type mismatch: too many return values (expected 1)"); - // Pass the instance into the closure so that we keep it live for the lifetime - // of the closure. Pass the export in so that we can call it. + // Pass the instance into the closure so that we keep it live for + // the lifetime of the closure. Pass the `anyfunc` in so that we can + // call it. let instance = self.instance.clone(); - let export = self.export.clone(); + let anyfunc = self.export.anyfunc; // ... and then once we've passed the typechecks we can hand out our // object since our `transmute` below should be safe! @@ -191,12 +195,12 @@ macro_rules! getters { *mut VMContext, $($args,)* ) -> R, - >(export.address); + >(anyfunc.as_ref().func_ptr); let mut ret = None; $(let $args = $args.into_abi();)* - invoke_wasm_and_catch_traps(export.vmctx, &instance.store, || { - ret = Some(fnptr(export.vmctx, ptr::null_mut(), $($args,)*)); + invoke_wasm_and_catch_traps(anyfunc.as_ref().vmctx, &instance.store, || { + ret = Some(fnptr(anyfunc.as_ref().vmctx, ptr::null_mut(), $($args,)*)); })?; Ok(ret.unwrap()) @@ -280,10 +284,11 @@ impl Func { }); let (instance, export, trampoline) = crate::trampoline::generate_func_export(&ty, func, store).expect("generated func"); + debug_assert!(unsafe { !export.anyfunc.as_ref().func_ptr.is_null() }, "c"); Func { instance, - export, trampoline, + export, } } @@ -488,7 +493,10 @@ impl Func { pub fn ty(&self) -> FuncType { // Signatures should always be registered in the store's registry of // shared signatures, so we should be able to unwrap safely here. - let sig = self.instance.store.lookup_signature(self.export.signature); + let sig = self + .instance + .store + .lookup_signature(unsafe { self.export.anyfunc.as_ref().type_index }); // This is only called with `Export::Function`, and since it's coming // from wasmtime_runtime itself we should support all the types coming @@ -498,13 +506,19 @@ impl Func { /// Returns the number of parameters that this function takes. pub fn param_arity(&self) -> usize { - let sig = self.instance.store.lookup_signature(self.export.signature); + let sig = self + .instance + .store + .lookup_signature(unsafe { self.export.anyfunc.as_ref().type_index }); sig.params.len() } /// Returns the number of results this function produces. pub fn result_arity(&self) -> usize { - let sig = self.instance.store.lookup_signature(self.export.signature); + let sig = self + .instance + .store + .lookup_signature(unsafe { self.export.anyfunc.as_ref().type_index }); sig.returns.len() } @@ -553,14 +567,21 @@ impl Func { } // Call the trampoline. - invoke_wasm_and_catch_traps(self.export.vmctx, &self.instance.store, || unsafe { - (self.trampoline)( - self.export.vmctx, - ptr::null_mut(), - self.export.address, - values_vec.as_mut_ptr(), - ) - })?; + unsafe { + debug_assert!(!self.export.anyfunc.as_ref().func_ptr.is_null(), "d"); + invoke_wasm_and_catch_traps( + self.export.anyfunc.as_ref().vmctx, + &self.instance.store, + || { + (self.trampoline)( + self.export.anyfunc.as_ref().vmctx, + ptr::null_mut(), + self.export.anyfunc.as_ref().func_ptr, + values_vec.as_mut_ptr(), + ) + }, + )?; + } // Load the return values out of `values_vec`. let mut results = Vec::with_capacity(my_ty.results().len()); @@ -578,22 +599,32 @@ impl Func { &self.export } + pub(crate) fn caller_checked_anyfunc( + &self, + ) -> NonNull { + self.export.anyfunc + } + pub(crate) fn from_wasmtime_function( export: wasmtime_runtime::ExportFunction, instance: StoreInstanceHandle, - ) -> Self { + ) -> Option { + if unsafe { export.anyfunc.as_ref().func_ptr.is_null() } { + return None; + } + // Each function signature in a module should have a trampoline stored // on that module as well, so unwrap the result here since otherwise // it's a bug in wasmtime. let trampoline = instance - .trampoline(export.signature) + .trampoline(unsafe { export.anyfunc.as_ref().type_index }) .expect("failed to retrieve trampoline from module"); - Func { + Some(Func { instance, export, trampoline, - } + }) } getters! { @@ -1189,6 +1220,7 @@ macro_rules! impl_into_func { ) .expect("failed to generate export") }; + debug_assert!(unsafe { !export.anyfunc.as_ref().func_ptr.is_null() }, "a"); Func { instance, export, diff --git a/crates/wasmtime/src/instance.rs b/crates/wasmtime/src/instance.rs index 3f5fa4186f33..7b3f1dfd162d 100644 --- a/crates/wasmtime/src/instance.rs +++ b/crates/wasmtime/src/instance.rs @@ -8,6 +8,7 @@ use wasmtime_jit::{CompiledModule, Resolver}; use wasmtime_runtime::{InstantiationError, VMContext, VMFunctionBody}; struct SimpleResolver<'a> { + store: Store, imports: &'a [Extern], } @@ -15,7 +16,7 @@ impl Resolver for SimpleResolver<'_> { fn resolve(&mut self, idx: u32, _name: &str, _field: &str) -> Option { self.imports .get(idx as usize) - .map(|i| i.get_wasmtime_export()) + .map(|i| i.get_wasmtime_export(&self.store)) } } @@ -41,7 +42,10 @@ fn instantiate( ); } - let mut resolver = SimpleResolver { imports }; + let mut resolver = SimpleResolver { + store: store.clone(), + imports, + }; let config = store.engine().config(); let instance = unsafe { let instance = compiled_module.instantiate( @@ -95,7 +99,7 @@ fn instantiate( mem::transmute::< *const VMFunctionBody, unsafe extern "C" fn(*mut VMContext, *mut VMContext), - >(f.address)(f.vmctx, vmctx_ptr) + >(f.anyfunc.as_ref().func_ptr)(f.anyfunc.as_ref().vmctx, vmctx_ptr) })?; } } @@ -236,7 +240,7 @@ impl Instance { /// Returns `None` if there was no export named `name`, or if there was but /// it wasn't a function. pub fn get_func(&self, name: &str) -> Option { - self.get_export(name)?.into_func() + self.get_export(name)?.into_func().and_then(|f| f) } /// Looks up an exported [`Table`] value by name. diff --git a/crates/wasmtime/src/linker.rs b/crates/wasmtime/src/linker.rs index dcf4bddb3c49..552661ba2eb5 100644 --- a/crates/wasmtime/src/linker.rs +++ b/crates/wasmtime/src/linker.rs @@ -371,7 +371,7 @@ impl Linker { /// "#; /// let module = Module::new(store.engine(), wat)?; /// linker.module("", &module)?; - /// let count = linker.get_one_by_name("", "run")?.into_func().unwrap().get0::()?()?; + /// let count = linker.get_one_by_name("", "run")?.into_func().unwrap().unwrap().get0::()?()?; /// assert_eq!(count, 0, "a Command should get a fresh instance on each invocation"); /// /// # Ok(()) @@ -384,7 +384,7 @@ impl Linker { let instance = self.instantiate(&module)?; if let Some(export) = instance.get_export("_initialize") { - if let Extern::Func(func) = export { + if let Extern::Func(Some(func)) = export { func.get0::<()>() .and_then(|f| f().map_err(Into::into)) .context("calling the Reactor initialization function")?; @@ -415,6 +415,7 @@ impl Linker { .unwrap() .into_func() .unwrap() + .unwrap() .call(params) .map_err(|error| error.downcast::().unwrap())?; @@ -427,7 +428,7 @@ impl Linker { Ok(()) }); - self.insert(module_name, export.name(), Extern::Func(func))?; + self.insert(module_name, export.name(), func.into())?; } else if export.name() == "memory" && export.ty().memory().is_some() { // Allow an exported "memory" memory for now. } else if export.name() == "__indirect_function_table" && export.ty().table().is_some() @@ -690,7 +691,7 @@ impl Linker { if items.next().is_some() { bail!("too many items named `` in `{}`", module); } - if let Extern::Func(func) = external { + if let Extern::Func(Some(func)) = external { return Ok(func.clone()); } bail!("default export in '{}' is not a function", module); @@ -702,7 +703,7 @@ impl Linker { if items.next().is_some() { bail!("too many items named `_start` in `{}`", module); } - if let Extern::Func(func) = external { + if let Extern::Func(Some(func)) = external { return Ok(func.clone()); } bail!("`_start` in '{}' is not a function", module); diff --git a/crates/wasmtime/src/runtime.rs b/crates/wasmtime/src/runtime.rs index d42860b9b923..d93a32851cab 100644 --- a/crates/wasmtime/src/runtime.rs +++ b/crates/wasmtime/src/runtime.rs @@ -11,6 +11,7 @@ use std::convert::TryFrom; use std::fmt; use std::hash::{Hash, Hasher}; use std::path::Path; +use std::ptr::NonNull; use std::rc::{Rc, Weak}; use std::sync::Arc; use wasmparser::{OperatorValidatorConfig, ValidatingParserConfig}; @@ -20,8 +21,8 @@ use wasmtime_jit::{native, CompilationStrategy, Compiler}; use wasmtime_profiling::{JitDumpAgent, NullProfilerAgent, ProfilingAgent, VTuneAgent}; use wasmtime_runtime::{ debug_builtins, InstanceHandle, RuntimeMemoryCreator, SignalHandler, SignatureRegistry, - StackMapRegistry, VMExternRef, VMExternRefActivationsTable, VMInterrupts, - VMSharedSignatureIndex, + StackMapRegistry, VMCallerCheckedAnyfunc, VMExternRef, VMExternRefActivationsTable, + VMInterrupts, VMSharedSignatureIndex, }; // Runtime Environment @@ -818,6 +819,7 @@ pub(crate) struct StoreInner { host_info: RefCell>>>, externref_activations_table: Rc, stack_map_registry: Rc, + null_func_ref: Box, } struct HostInfoKey(VMExternRef); @@ -860,6 +862,7 @@ impl Store { host_info: RefCell::new(HashMap::new()), externref_activations_table: Rc::new(VMExternRefActivationsTable::new()), stack_map_registry: Rc::new(StackMapRegistry::default()), + null_func_ref: Box::new(VMCallerCheckedAnyfunc::null()), }), } } @@ -882,6 +885,11 @@ impl Store { .map(|x| x as _) } + pub(crate) fn null_func_ref(&self) -> NonNull { + let p = &*self.inner.null_func_ref as *const VMCallerCheckedAnyfunc as *mut _; + unsafe { NonNull::new_unchecked(p) } + } + pub(crate) fn lookup_signature(&self, sig_index: VMSharedSignatureIndex) -> wasm::WasmFuncType { self.inner .signatures diff --git a/crates/wasmtime/src/trampoline/global.rs b/crates/wasmtime/src/trampoline/global.rs index 4fa13e42bdf0..db09d4b81eef 100644 --- a/crates/wasmtime/src/trampoline/global.rs +++ b/crates/wasmtime/src/trampoline/global.rs @@ -1,13 +1,13 @@ use super::create_handle::create_handle; use crate::trampoline::StoreInstanceHandle; -use crate::Store; -use crate::{GlobalType, Mutability, Val}; +use crate::{GlobalType, Mutability, Store, Val}; use anyhow::{bail, Result}; use wasmtime_environ::entity::PrimaryMap; use wasmtime_environ::{wasm, EntityIndex, Module}; pub fn create_global(store: &Store, gt: &GlobalType, val: Val) -> Result { let global = wasm::Global { + wasm_ty: gt.content().to_wasm_type(), ty: match gt.content().get_wasmtime_type() { Some(t) => t, None => bail!("cannot support {:?} as a wasm global type", gt.content()), diff --git a/crates/wasmtime/src/types.rs b/crates/wasmtime/src/types.rs index ad1905963a6b..d91e6fef5e89 100644 --- a/crates/wasmtime/src/types.rs +++ b/crates/wasmtime/src/types.rs @@ -106,25 +106,7 @@ impl ValType { ValType::F32 => Some(ir::types::F32), ValType::F64 => Some(ir::types::F64), ValType::V128 => Some(ir::types::I8X16), - #[cfg(target_pointer_width = "64")] - ValType::ExternRef => Some(ir::types::R64), - #[cfg(target_pointer_width = "32")] - ValType::ExternRef => Some(ir::types::R32), - _ => None, - } - } - - pub(crate) fn from_wasmtime_type(ty: ir::Type) -> Option { - match ty { - ir::types::I32 => Some(ValType::I32), - ir::types::I64 => Some(ValType::I64), - ir::types::F32 => Some(ValType::F32), - ir::types::F64 => Some(ValType::F64), - ir::types::I8X16 => Some(ValType::V128), - #[cfg(target_pointer_width = "64")] - ir::types::R64 => Some(ValType::ExternRef), - #[cfg(target_pointer_width = "32")] - ir::types::R32 => Some(ValType::ExternRef), + ValType::ExternRef => Some(wasmtime_runtime::ref_type()), _ => None, } } @@ -353,7 +335,7 @@ impl GlobalType { /// Returns `None` if the wasmtime global has a type that we can't /// represent, but that should only very rarely happen and indicate a bug. pub(crate) fn from_wasmtime_global(global: &wasm::Global) -> Option { - let ty = ValType::from_wasmtime_type(global.ty)?; + let ty = ValType::from_wasm_type(&global.wasm_ty)?; let mutability = if global.mutability { Mutability::Var } else { diff --git a/crates/wasmtime/src/values.rs b/crates/wasmtime/src/values.rs index d12813bf3643..9cd79c59036b 100644 --- a/crates/wasmtime/src/values.rs +++ b/crates/wasmtime/src/values.rs @@ -1,8 +1,7 @@ use crate::r#ref::ExternRef; use crate::{Func, Store, ValType}; use anyhow::{bail, Result}; -use std::convert::TryFrom; -use std::ptr; +use std::ptr::{self, NonNull}; use wasmtime_runtime::{self as runtime, VMExternRef}; /// Possible runtime values that a WebAssembly module can either consume or @@ -27,11 +26,18 @@ pub enum Val { /// `f64::from_bits` to create an `f64` value. F64(u64), - /// An `externref` value which can hold opaque data to the wasm instance itself. + /// An `externref` value which can hold opaque data to the Wasm instance + /// itself. + /// + /// `ExternRef(None)` is the null function reference, created by `ref.null + /// extern` in Wasm. ExternRef(Option), /// A first-class reference to a WebAssembly function. - FuncRef(Func), + /// + /// `FuncRef(None)` is the null function reference, created by `ref.null + /// func` in Wasm. + FuncRef(Option), /// A 128-bit number V128(u128), @@ -95,19 +101,12 @@ impl Val { .insert_with_gc(x.inner, store.stack_map_registry()); ptr::write(p as *mut *mut u8, externref_ptr) } - Val::FuncRef(f) => { - let f = f.wasmtime_function(); - let funcref = runtime::VMFuncRef::new(runtime::VMCallerCheckedAnyfunc { - func_ptr: f.address, - type_index: f.signature, - vmctx: f.vmctx, - }); - let externref: VMExternRef = funcref.into(); - let externref_ptr = externref.as_raw(); - store - .externref_activations_table() - .insert_with_gc(externref, store.stack_map_registry()); - ptr::write(p as *mut *mut u8, externref_ptr); + Val::FuncRef(None) => ptr::write(p, 0), + Val::FuncRef(Some(f)) => { + ptr::write( + p as *mut NonNull, + f.caller_checked_anyfunc(), + ); } } } @@ -131,15 +130,12 @@ impl Val { } } ValType::FuncRef => { - let raw = ptr::read(p as *const *mut u8); - let func = if raw.is_null() { - runtime::VMCallerCheckedAnyfunc::default() + let func = ptr::read(p as *const *mut runtime::VMCallerCheckedAnyfunc); + if let Some(f) = NonNull::new(func) { + from_checked_anyfunc(f, store) } else { - let externref = VMExternRef::clone_from_raw(raw); - let funcref = runtime::VMFuncRef::try_from(externref).unwrap(); - funcref.as_func().clone() - }; - from_checked_anyfunc(func, store) + Val::FuncRef(None) + } } } } @@ -150,7 +146,7 @@ impl Val { (I64(i64) i64 unwrap_i64 *e) (F32(f32) f32 unwrap_f32 f32::from_bits(*e)) (F64(f64) f64 unwrap_f64 f64::from_bits(*e)) - (FuncRef(&Func) funcref unwrap_funcref e) + (FuncRef(Option<&Func>) funcref unwrap_funcref e.as_ref()) (V128(u128) v128 unwrap_v128 *e) } @@ -184,7 +180,8 @@ impl Val { pub(crate) fn comes_from_same_store(&self, store: &Store) -> bool { match self { - Val::FuncRef(f) => Store::same(store, f.store()), + Val::FuncRef(Some(f)) => Store::same(store, f.store()), + Val::FuncRef(None) => true, // TODO: need to implement this once we actually finalize what // `externref` will look like and it's actually implemented to pass it @@ -235,51 +232,47 @@ impl From> for Val { } } +impl From> for Val { + fn from(val: Option) -> Val { + Val::FuncRef(val) + } +} + impl From for Val { fn from(val: Func) -> Val { - Val::FuncRef(val) + Val::FuncRef(Some(val)) } } pub(crate) fn into_checked_anyfunc( val: Val, store: &Store, -) -> Result { +) -> Result> { if !val.comes_from_same_store(store) { bail!("cross-`Store` values are not supported"); } Ok(match val { - Val::ExternRef(None) => wasmtime_runtime::VMCallerCheckedAnyfunc { - func_ptr: ptr::null(), - type_index: wasmtime_runtime::VMSharedSignatureIndex::default(), - vmctx: ptr::null_mut(), - }, - Val::FuncRef(f) => { - let f = f.wasmtime_function(); - wasmtime_runtime::VMCallerCheckedAnyfunc { - func_ptr: f.address, - type_index: f.signature, - vmctx: f.vmctx, - } - } + Val::FuncRef(None) => store.null_func_ref(), + Val::FuncRef(Some(f)) => f.caller_checked_anyfunc(), _ => bail!("val is not funcref"), }) } -pub(crate) fn from_checked_anyfunc( - item: wasmtime_runtime::VMCallerCheckedAnyfunc, +pub(crate) unsafe fn from_checked_anyfunc( + anyfunc: NonNull, store: &Store, ) -> Val { - if item.type_index == wasmtime_runtime::VMSharedSignatureIndex::default() { - return Val::ExternRef(None); + if anyfunc.as_ref().func_ptr.is_null() { + return Val::FuncRef(None); } - let instance_handle = unsafe { wasmtime_runtime::InstanceHandle::from_vmctx(item.vmctx) }; - let export = wasmtime_runtime::ExportFunction { - address: item.func_ptr, - signature: item.type_index, - vmctx: item.vmctx, - }; + + debug_assert!( + anyfunc.as_ref().type_index != wasmtime_runtime::VMSharedSignatureIndex::default() + ); + + let instance_handle = wasmtime_runtime::InstanceHandle::from_vmctx(anyfunc.as_ref().vmctx); + let export = wasmtime_runtime::ExportFunction { anyfunc }; let instance = store.existing_instance_handle(instance_handle); - let f = Func::from_wasmtime_function(export, instance); - Val::FuncRef(f) + let f = Func::from_wasmtime_function(export, instance).unwrap(); + Val::FuncRef(Some(f)) } diff --git a/crates/wast/src/spectest.rs b/crates/wast/src/spectest.rs index f49125dbe8cb..ef744fa42d50 100644 --- a/crates/wast/src/spectest.rs +++ b/crates/wast/src/spectest.rs @@ -35,7 +35,7 @@ pub fn link_spectest(linker: &mut Linker) -> Result<()> { linker.define("spectest", "global_f64", g)?; let ty = TableType::new(ValType::FuncRef, Limits::new(10, Some(20))); - let table = Table::new(linker.store(), ty, Val::ExternRef(None))?; + let table = Table::new(linker.store(), ty, Val::FuncRef(None))?; linker.define("spectest", "table", table)?; let ty = MemoryType::new(Limits::new(1, Some(2))); diff --git a/crates/wast/src/wast.rs b/crates/wast/src/wast.rs index 56d34f6dad1b..0cb12a779c81 100644 --- a/crates/wast/src/wast.rs +++ b/crates/wast/src/wast.rs @@ -162,7 +162,8 @@ impl WastContext { let func = self .get_export(instance_name, field)? .into_func() - .ok_or_else(|| anyhow!("no function named `{}`", field))?; + .ok_or_else(|| anyhow!("no function named `{}`", field))? + .ok_or_else(|| anyhow!("attempted to call `ref.null func`"))?; Ok(match func.call(args) { Ok(result) => Outcome::Ok(result.into()), Err(e) => Outcome::Trap(e.downcast()?), diff --git a/src/commands/run.rs b/src/commands/run.rs index 359686031e2f..bcbe377ea24e 100644 --- a/src/commands/run.rs +++ b/src/commands/run.rs @@ -263,7 +263,8 @@ impl RunCommand { fn invoke_export(&self, linker: &Linker, name: &str) -> Result<()> { let func = match linker.get_one_by_name("", name)?.into_func() { - Some(func) => func, + Some(Some(func)) => func, + Some(None) => bail!("attempt to call null func ref"), None => bail!("export of `{}` wasn't a function", name), }; self.invoke_func(func, Some(name)) diff --git a/tests/all/custom_signal_handler.rs b/tests/all/custom_signal_handler.rs index ff381d6e00f1..bc21afb67996 100644 --- a/tests/all/custom_signal_handler.rs +++ b/tests/all/custom_signal_handler.rs @@ -103,7 +103,7 @@ mod tests { i32::from_le_bytes(memory[0..4].try_into().unwrap()) } }); - wasmtime::Extern::Func(func) + wasmtime::Extern::Func(Some(func)) }) .collect::>() } diff --git a/tests/all/externals.rs b/tests/all/externals.rs index 05ef48c63271..5d842abbfd59 100644 --- a/tests/all/externals.rs +++ b/tests/all/externals.rs @@ -28,27 +28,27 @@ fn bad_tables() { // get out of bounds let ty = TableType::new(ValType::FuncRef, Limits::new(0, Some(1))); - let t = Table::new(&Store::default(), ty.clone(), Val::ExternRef(None)).unwrap(); + let t = Table::new(&Store::default(), ty.clone(), Val::FuncRef(None)).unwrap(); assert!(t.get(0).is_none()); assert!(t.get(u32::max_value()).is_none()); // set out of bounds or wrong type let ty = TableType::new(ValType::FuncRef, Limits::new(1, Some(1))); - let t = Table::new(&Store::default(), ty.clone(), Val::ExternRef(None)).unwrap(); + let t = Table::new(&Store::default(), ty.clone(), Val::FuncRef(None)).unwrap(); assert!(t.set(0, Val::I32(0)).is_err()); - assert!(t.set(0, Val::ExternRef(None)).is_ok()); - assert!(t.set(1, Val::ExternRef(None)).is_err()); + assert!(t.set(0, Val::FuncRef(None)).is_ok()); + assert!(t.set(1, Val::FuncRef(None)).is_err()); // grow beyond max let ty = TableType::new(ValType::FuncRef, Limits::new(1, Some(1))); - let t = Table::new(&Store::default(), ty.clone(), Val::ExternRef(None)).unwrap(); - assert!(t.grow(0, Val::ExternRef(None)).is_ok()); - assert!(t.grow(1, Val::ExternRef(None)).is_err()); + let t = Table::new(&Store::default(), ty.clone(), Val::FuncRef(None)).unwrap(); + assert!(t.grow(0, Val::FuncRef(None)).is_ok()); + assert!(t.grow(1, Val::FuncRef(None)).is_err()); assert_eq!(t.size(), 1); // grow wrong type let ty = TableType::new(ValType::FuncRef, Limits::new(1, Some(2))); - let t = Table::new(&Store::default(), ty.clone(), Val::ExternRef(None)).unwrap(); + let t = Table::new(&Store::default(), ty.clone(), Val::FuncRef(None)).unwrap(); assert!(t.grow(1, Val::I32(0)).is_err()); assert_eq!(t.size(), 1); } @@ -69,7 +69,7 @@ fn cross_store() -> anyhow::Result<()> { let ty = MemoryType::new(Limits::new(1, None)); let memory = Memory::new(&store2, ty); let ty = TableType::new(ValType::FuncRef, Limits::new(1, None)); - let table = Table::new(&store2, ty, Val::ExternRef(None))?; + let table = Table::new(&store2, ty, Val::FuncRef(None))?; let need_func = Module::new(&engine, r#"(module (import "" "" (func)))"#)?; assert!(Instance::new(&store1, &need_func, &[func.into()]).is_err()); @@ -85,8 +85,8 @@ fn cross_store() -> anyhow::Result<()> { // ============ Cross-store globals ============== - let store1val = Val::FuncRef(Func::wrap(&store1, || {})); - let store2val = Val::FuncRef(Func::wrap(&store2, || {})); + let store1val = Val::FuncRef(Some(Func::wrap(&store1, || {}))); + let store2val = Val::FuncRef(Some(Func::wrap(&store2, || {}))); let ty = GlobalType::new(ValType::FuncRef, Mutability::Var); assert!(Global::new(&store2, ty.clone(), store1val.clone()).is_err()); diff --git a/tests/all/funcref.rs b/tests/all/funcref.rs index 0a8d48666f06..aa6230ec1da2 100644 --- a/tests/all/funcref.rs +++ b/tests/all/funcref.rs @@ -27,11 +27,11 @@ fn pass_funcref_in_and_out_of_wasm() -> anyhow::Result<()> { // Do it a bunch of times to make sure we exercise GC with `funcref`s in the // activations table. for _ in 0..1024 { - let results = func.call(&[Val::FuncRef(func.clone())])?; + let results = func.call(&[Val::FuncRef(Some(func.clone()))])?; assert_eq!(results.len(), 1); // Can't compare `Func` for equality, so this is the best we can do here. - let result_func = results[0].unwrap_funcref(); + let result_func = results[0].unwrap_funcref().unwrap(); assert_eq!(func.ty(), result_func.ty()); } diff --git a/tests/all/invoke_func_via_table.rs b/tests/all/invoke_func_via_table.rs index a73f7c1c404d..cc77a028c7d0 100644 --- a/tests/all/invoke_func_via_table.rs +++ b/tests/all/invoke_func_via_table.rs @@ -23,6 +23,7 @@ fn test_invoke_func_via_table() -> Result<()> { .unwrap() .funcref() .unwrap() + .unwrap() .clone(); let result = f.call(&[]).unwrap(); assert_eq!(result[0].unwrap_i64(), 42); diff --git a/tests/all/linker.rs b/tests/all/linker.rs index 0db288aedacc..5a79ba998310 100644 --- a/tests/all/linker.rs +++ b/tests/all/linker.rs @@ -59,11 +59,11 @@ fn link_twice_bad() -> Result<()> { // tables let ty = TableType::new(ValType::FuncRef, Limits::new(1, None)); - let table = Table::new(&store, ty, Val::ExternRef(None))?; + let table = Table::new(&store, ty, Val::FuncRef(None))?; linker.define("", "", table.clone())?; assert!(linker.define("", "", table.clone()).is_err()); let ty = TableType::new(ValType::FuncRef, Limits::new(2, None)); - let table = Table::new(&store, ty, Val::ExternRef(None))?; + let table = Table::new(&store, ty, Val::FuncRef(None))?; assert!(linker.define("", "", table.clone()).is_err()); Ok(()) } @@ -93,7 +93,12 @@ fn function_interposition() -> Result<()> { )?; } let instance = linker.instantiate(&module)?; - let func = instance.get_export("green").unwrap().into_func().unwrap(); + let func = instance + .get_export("green") + .unwrap() + .into_func() + .unwrap() + .unwrap(); let func = func.get0::()?; assert_eq!(func()?, 112); Ok(()) @@ -155,7 +160,12 @@ fn module_interposition() -> Result<()> { )?; } let instance = linker.instantiate(&module)?; - let func = instance.get_export("export").unwrap().into_func().unwrap(); + let func = instance + .get_export("export") + .unwrap() + .into_func() + .unwrap() + .unwrap(); let func = func.get0::()?; assert_eq!(func()?, 112); Ok(()) diff --git a/tests/all/table.rs b/tests/all/table.rs index 9bf7cb4cc484..26830c4ea9bf 100644 --- a/tests/all/table.rs +++ b/tests/all/table.rs @@ -4,9 +4,9 @@ use wasmtime::*; fn get_none() { let store = Store::default(); let ty = TableType::new(ValType::FuncRef, Limits::new(1, None)); - let table = Table::new(&store, ty, Val::ExternRef(None)).unwrap(); + let table = Table::new(&store, ty, Val::FuncRef(None)).unwrap(); match table.get(0) { - Some(Val::ExternRef(None)) => {} + Some(Val::FuncRef(None)) => {} _ => panic!(), } assert!(table.get(1).is_none()); diff --git a/tests/all/use_after_drop.rs b/tests/all/use_after_drop.rs index 625bac897fed..293dacafd023 100644 --- a/tests/all/use_after_drop.rs +++ b/tests/all/use_after_drop.rs @@ -11,10 +11,10 @@ fn use_func_after_drop() -> Result<()> { assert_eq!(closed_over_data, "abcd"); }); let ty = TableType::new(ValType::FuncRef, Limits::new(1, None)); - table = Table::new(&store, ty, Val::ExternRef(None))?; + table = Table::new(&store, ty, Val::FuncRef(None))?; table.set(0, func.into())?; } - let func = table.get(0).unwrap().funcref().unwrap().clone(); + let func = table.get(0).unwrap().funcref().unwrap().unwrap().clone(); let func = func.get0::<()>()?; func()?; Ok(())