Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Interrupt blocking instructions #1921

99 changes: 53 additions & 46 deletions core/iwasm/interpreter/wasm_interp_classic.c
Original file line number Diff line number Diff line change
Expand Up @@ -4183,79 +4183,86 @@ wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMExecEnv *exec_env,

wasm_exec_env_set_cur_frame(exec_env, frame);

if (function->is_import_func) {
sigjmp_buf *context = os_create_stack_context();
if (!os_is_returning_from_signal(exec_env->handle, context)) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

how about resource cleanup i mentioned in #1910 (comment) ?

if (function->is_import_func) {
#if WASM_ENABLE_MULTI_MODULE != 0
if (function->import_module_inst) {
wasm_interp_call_func_import(module_inst, exec_env, function,
frame);
}
else
if (function->import_module_inst) {
wasm_interp_call_func_import(module_inst, exec_env, function,
frame);
}
else
#endif
{
/* it is a native function */
wasm_interp_call_func_native(module_inst, exec_env, function,
frame);
{
/* it is a native function */
wasm_interp_call_func_native(module_inst, exec_env, function,
frame);
}
}
}
else {
else {
#if WASM_ENABLE_LAZY_JIT != 0

/* Fast JIT to LLVM JIT tier-up is enabled */
/* Fast JIT to LLVM JIT tier-up is enabled */
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0
/* Fast JIT and LLVM JIT are both enabled, call llvm jit function
if it is compiled, else call fast jit function */
uint32 func_idx = (uint32)(function - module_inst->e->functions);
if (module_inst->module->func_ptrs_compiled
[func_idx - module_inst->module->import_function_count]) {
/* Fast JIT and LLVM JIT are both enabled, call llvm jit function
if it is compiled, else call fast jit function */
uint32 func_idx = (uint32)(function - module_inst->e->functions);
if (module_inst->module->func_ptrs_compiled
[func_idx - module_inst->module->import_function_count]) {
llvm_jit_call_func_bytecode(module_inst, exec_env, function,
argc, argv);
/* For llvm jit, the results have been stored in argv,
no need to copy them from stack frame again */
copy_argv_from_frame = false;
}
else {
fast_jit_call_func_bytecode(module_inst, exec_env, function,
frame);
}
#elif WASM_ENABLE_JIT != 0
/* Only LLVM JIT is enabled */
llvm_jit_call_func_bytecode(module_inst, exec_env, function, argc,
argv);
/* For llvm jit, the results have been stored in argv,
no need to copy them from stack frame again */
copy_argv_from_frame = false;
}
else {
fast_jit_call_func_bytecode(module_inst, exec_env, function, frame);
}
#elif WASM_ENABLE_JIT != 0
/* Only LLVM JIT is enabled */
llvm_jit_call_func_bytecode(module_inst, exec_env, function, argc,
argv);
/* For llvm jit, the results have been stored in argv,
no need to copy them from stack frame again */
copy_argv_from_frame = false;
#elif WASM_ENABLE_FAST_JIT != 0
/* Only Fast JIT is enabled */
fast_jit_call_func_bytecode(module_inst, exec_env, function, frame);
/* Only Fast JIT is enabled */
fast_jit_call_func_bytecode(module_inst, exec_env, function, frame);
#else
/* Both Fast JIT and LLVM JIT are disabled */
wasm_interp_call_func_bytecode(module_inst, exec_env, function, frame);
/* Both Fast JIT and LLVM JIT are disabled */
wasm_interp_call_func_bytecode(module_inst, exec_env, function,
frame);
#endif

#else /* else of WASM_ENABLE_LAZY_JIT != 0 */

/* Fast JIT to LLVM JIT tier-up is enabled */
/* Fast JIT to LLVM JIT tier-up is enabled */
#if WASM_ENABLE_JIT != 0
/* LLVM JIT is enabled */
llvm_jit_call_func_bytecode(module_inst, exec_env, function, argc,
argv);
/* For llvm jit, the results have been stored in argv,
no need to copy them from stack frame again */
copy_argv_from_frame = false;
/* LLVM JIT is enabled */
llvm_jit_call_func_bytecode(module_inst, exec_env, function, argc,
argv);
/* For llvm jit, the results have been stored in argv,
no need to copy them from stack frame again */
copy_argv_from_frame = false;
#elif WASM_ENABLE_FAST_JIT != 0
/* Fast JIT is enabled */
fast_jit_call_func_bytecode(module_inst, exec_env, function, frame);
/* Fast JIT is enabled */
fast_jit_call_func_bytecode(module_inst, exec_env, function, frame);
#else
/* Both Fast JIT and LLVM JIT are disabled */
wasm_interp_call_func_bytecode(module_inst, exec_env, function, frame);
/* Both Fast JIT and LLVM JIT are disabled */
wasm_interp_call_func_bytecode(module_inst, exec_env, function,
frame);
#endif

#endif /* end of WASM_ENABLE_LAZY_JIT != 0 */

(void)wasm_interp_call_func_bytecode;
(void)wasm_interp_call_func_bytecode;
#if WASM_ENABLE_FAST_JIT != 0
(void)fast_jit_call_func_bytecode;
(void)fast_jit_call_func_bytecode;
#endif
}
}
os_remove_stack_context(exec_env->handle, context);

/* Output the return value to the caller */
if (!wasm_get_exception(module_inst)) {
Expand Down
12 changes: 0 additions & 12 deletions core/iwasm/libraries/lib-pthread/lib_pthread_wrapper.c
Original file line number Diff line number Diff line change
Expand Up @@ -150,18 +150,6 @@ static uint32 handle_id = 1;
static void
lib_pthread_destroy_callback(WASMCluster *cluster);

static uint32
thread_handle_hash(void *handle)
{
return (uint32)(uintptr_t)handle;
}

static bool
thread_handle_equal(void *h1, void *h2)
{
return (uint32)(uintptr_t)h1 == (uint32)(uintptr_t)h2 ? true : false;
}

static void
thread_info_destroy(void *node)
{
Expand Down
4 changes: 4 additions & 0 deletions core/iwasm/libraries/thread-mgr/thread_manager.c
Original file line number Diff line number Diff line change
Expand Up @@ -1059,6 +1059,10 @@ set_exception_visitor(void *node, void *user_data)
bh_memcpy_s(curr_wasm_inst->cur_exception,
sizeof(curr_wasm_inst->cur_exception),
wasm_inst->cur_exception, sizeof(wasm_inst->cur_exception));

if (curr_exec_env->handle)
os_thread_signal(curr_exec_env->handle, SIGUSR1);

/* Terminate the thread so it can exit from dead loops */
set_thread_cancel_flags(curr_exec_env);
}
Expand Down
171 changes: 171 additions & 0 deletions core/shared/platform/common/posix/posix_thread.c
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,13 @@
#include "platform_api_vmcore.h"
#include "platform_api_extension.h"

#include "bh_hashmap.h"
#include <setjmp.h>

#define STACK_CTX_SIG SIGUSR1
static os_thread_local_attribute sigset_t newmask, oldmask;
bh_stack_context_handler_t stack_context_handler;

typedef struct {
thread_start_routine_t start;
void *arg;
Expand Down Expand Up @@ -47,6 +54,164 @@ os_thread_wrapper(void *arg)
return NULL;
}

static void
stack_context_sig_handler(int sig)
{
assert(sig == STACK_CTX_SIG);
korp_tid self = pthread_self();

/* Get latest stack context (first in the list) for current thread */
bh_list *context_list = bh_hash_map_find(stack_context_handler.contexts,
(void *)(uintptr_t)self);
assert(context_list);
sigjmp_buf *context = bh_list_first_elem(context_list);
assert(context);

siglongjmp(*context, 1);
}

sigjmp_buf *
os_create_stack_context(korp_tid handle)
{
return BH_MALLOC(sizeof(sigjmp_buf));
}

static void
free_stack_context_list(void *stack_context_list)
{
bh_list *context_list = (bh_list *)stack_context_list;

sigjmp_buf *context = bh_list_first_elem(context_list);
while (context) {
sigjmp_buf *next = bh_list_elem_next(context);
BH_FREE(context);
context = next;
};

BH_FREE(context_list);
}

static inline void
stack_ctx_block_sig_and_lock()
{
sigemptyset(&newmask);
sigaddset(&newmask, STACK_CTX_SIG);
pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);

os_mutex_lock(&stack_context_handler.lock);
}

static inline void
stack_ctx_unlock_and_unblock_sig()
{
os_mutex_unlock(&stack_context_handler.lock);
pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
}

bool
os_stack_contexts_init()
{
struct sigaction act;
memset(&act, 0, sizeof(act));
act.sa_handler = stack_context_sig_handler;
sigfillset(&act.sa_mask);
if (sigaction(STACK_CTX_SIG, &act, NULL) < 0) {
LOG_ERROR("failed to set signal handler");
return false;
}

if (os_mutex_init(&stack_context_handler.lock) != 0) {
LOG_ERROR("failed to init context mutex");
return false;
}

stack_context_handler.contexts = bh_hash_map_create(
32, false, (HashFunc)thread_handle_hash,
(KeyEqualFunc)thread_handle_equal, NULL, free_stack_context_list);
if (stack_context_handler.contexts == NULL) {
LOG_ERROR("failed to init stack contexts");
return false;
}

return true;
}

void
os_stack_contexts_destroy()
{
if (!bh_hash_map_destroy(stack_context_handler.contexts))
LOG_ERROR("failed to destroy stack context map");
stack_context_handler.contexts = NULL;

if (os_mutex_destroy(&stack_context_handler.lock) != BHT_OK)
LOG_ERROR("failed to destroy stack contexts");
}

void
os_remove_stack_context(korp_tid handle, sigjmp_buf *context)
{
korp_tid self = pthread_self();

stack_ctx_block_sig_and_lock();
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why don't you make it thread-local?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point, otherwise it would be shared between threads (and would then require access protection).

Actually, I wanted to avoid globals and return the oldmask from stack_ctx_block_sig_and_lock(), but returning anything from that function was corrupting the stack when used inside os_is_returning_from_signal(), I'm still not sure I understand why


bh_list *context_list = bh_hash_map_find(stack_context_handler.contexts,
(void *)(uintptr_t)self);
assert(context_list);
int ret = bh_list_remove(context_list, context);
assert(ret == BH_LIST_SUCCESS);
BH_FREE(context);

stack_ctx_unlock_and_unblock_sig();
}

bool
os_is_returning_from_signal(korp_tid handle, sigjmp_buf *context)
{
if (sigsetjmp(*context, 1))
return true;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this looks unsafe to me.
when we return here with a longjmp, the call frame of os_is_returning_from_signal is not valid anymore.

to make this work, os_is_returning_from_signal should always be inlined.
maybe it's more straightforward to use a macro instead.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

thanks, that might explain some problems that I was seeing when trying to use it with AOT. In any case, I realized that I missed this PR #1309 that is basically doing something very similar to what I want to do. The approach is better because we don't need to keep a map <thread_id, stack contexts> if we use tls. I'll try to apply a similar approach.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@eloparco Yes, in the memory access boundary check with hardware trap feature, both interpreter and AOT use the setjmp/longjmp mechanism in posix like platforms. The local variable jmpbuf is pushed into exec_env's jmpbuf stack before setjmp, runtime doesn't allocate memory for it.
I think that the principle of this PR is similar to the hw bound check feature: firstly register a signal handler, then setjmp before call, and when running wasm function, if there is a signal raised, the signal handler is trigger and the handler longjmp into the place of setjmp.

I think there may be two ways to implement this PR: one is to extend the hw bound check feature, e.g. reuse the current signal handler, the current setjmp and longjmp process, but add another macro to control them respectively (for example, OS_ENABLE_XXX and OS_ENABLE_HW_BOUND_CHECK). The other is to register new signal handler, add new setjmp/longjmp process, but I suggest not to add setjmp in wasm_interp_class.c, had better add in wasm_runtime.c, before calling wasm function:
https://github.com/bytecodealliance/wasm-micro-runtime/blob/main/core/iwasm/interpreter/wasm_runtime.c#L2163
https://github.com/bytecodealliance/wasm-micro-runtime/blob/main/core/iwasm/interpreter/wasm_runtime.c#LL2459C3-L2459C3

And note that setjmp is only supported by posix platforms, there should be macro to control the feature, and had better use os_setjmp/os_longjmp.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

And could you submit this PR to main branch since the main branch also has the issue?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://github.com/bytecodealliance/wasm-micro-runtime/blob/main/core/iwasm/interpreter/wasm_runtime.c#L2163

yes that works, for aot I'm not sure what to wrap with os_setjmp since there are 2 calls, I need to check

ret = invoke_native_internal(exec_env, function->u.func.func_ptr,

ret = invoke_native_internal(exec_env, function->u.func.func_ptr,

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

And could you submit this PR to main branch since the main branch also has the issue?

I'll try to have it working here first since I'm testing it with wasi threads, then I'll create a PR in main to merge it there

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm closing this PR in favour of #1930, where I rely on what's already implemented for hardware bound checks.


stack_ctx_block_sig_and_lock();

/* Get or create list of stack contexts for current thread */
bh_list *context_list = bh_hash_map_find(stack_context_handler.contexts,
(void *)(uintptr_t)handle);
if (!context_list) {
context_list = BH_MALLOC(sizeof(bh_list));
if (!context_list) {
LOG_ERROR("failed to allocate stack context list");
goto unlock;
}

int ret_list = bh_list_init(context_list);
if (ret_list != BH_LIST_SUCCESS) {
LOG_ERROR("failed to initialize stack context list");
goto free_list_and_unlock;
}

bool ret_map =
bh_hash_map_insert(stack_context_handler.contexts,
(void *)(uintptr_t)handle, context_list);
if (!ret_map) {
LOG_ERROR("failed to insert stack context list into map");
goto free_list_and_unlock;
}
}

/* Add stack context to stack context list */
int bh_res = bh_list_insert(context_list, context);
if (bh_res != BH_LIST_SUCCESS) {
LOG_ERROR("failed to insert stack context into list");
goto free_list_and_unlock;
}
goto unlock;

free_list_and_unlock:
BH_FREE(context_list);
unlock:
stack_ctx_unlock_and_unblock_sig();
return false;
}

int
os_thread_create_with_prio(korp_tid *tid, thread_start_routine_t start,
void *arg, unsigned int stack_size, int prio)
Expand Down Expand Up @@ -89,6 +254,12 @@ os_thread_create_with_prio(korp_tid *tid, thread_start_routine_t start,
return BHT_OK;
}

int
os_thread_signal(korp_tid tid, int sig)
{
return pthread_kill(tid, sig);
}

int
os_thread_create(korp_tid *tid, thread_start_routine_t start, void *arg,
unsigned int stack_size)
Expand Down
Loading