Add exception stack system to runtime

Runtime
-------

* An exception stack type, `jl_exc_stack_t` and associated manipulation functions
  has been added. Conceptually this stores a stack of (exception,backtrace)
  pairs. It's stored in a contiguous buffer so we can avoid reallocating when
  throwing and catching the same exception or set of exceptions repeatedly. Space
  for the exception and backtrace is allocated on demand inside `throw_internal`,
  after changing GC mode. Several variations were tried for allocating this
  sgorage, including allocating up front with malloc on the thread local state
  and copying during task switching. Keeping everything on the task seemed the
  simplest as it involves the least copying and keeps track of the exception
  stack buffers in a unified way.

* The exception in transit is no longer a single pointer in the thread local
  storage. It's stored in `ptls->current_task->exc_stack` instead, along with
  associated backtrace.

* `jl_current_exception()` is now the method to retreive the current exception
  from within a `JL_CATCH` dynamic context.

* Several places where manual restoration of the exception and backtrace was
  done are no longer necessary because the stack system handles these
  automatically. This code is removed, including
  `jl_apply_with_saved_exception_state`.

* `jl_eh_restore_state` has become non-inline. It seemed good to get this out
  of the public header.

* `jl_sig_throw` is now used with `jl_throw_in_ctx` from signal handlers, to
  make the special circumstances clear and to avoid conflation with rethrow
  which now simply rethrows the existing exception stack.

* Make rethrow outside a catch block into an error.

* Use `ptls->previous_exception` to support `jl_exception_occurred` in
  embedding API.

* finally lowering no longer includes a `foreigncall`, so expressions
  using finally can now be interpreted.

Interpreter / Codegen
---------------------

Mostly small changes here, to track the `:enter` and `:pop_exc` association
with the SSAValue token. The token SSAValue slot is ideal here for storing the
state of the exception stack at the `:enter`.

GC
--

Integrate exception and raw backtrace scanning as a special case into GC mark
loop. This is necessary now that Task objects can refer to exception and
backtrace data in raw form.

src/ast.c

@@ -889,7 +889,9 @@ jl_value_t *jl_parse_eval_all(const char *fname,
         form = jl_pchar_to_string(fname, len);
         result = jl_box_long(jl_lineno);
         err = 1;
+        goto finally; // skip jl_restore_exc_stack
     }
+finally:
     jl_get_ptls_states()->world_age = last_age;
     jl_lineno = last_lineno;
     jl_filename = last_filename;
@@ -901,7 +903,7 @@ jl_value_t *jl_parse_eval_all(const char *fname,
             jl_rethrow();
         else
             jl_rethrow_other(jl_new_struct(jl_loaderror_type, form, result,
-                                           ptls->exception_in_transit));
+                                           jl_current_exception()));
     }
     JL_GC_POP();
     return result;
@@ -1044,7 +1046,7 @@ static jl_value_t *jl_invoke_julia_macro(jl_array_t *args, jl_module_t *inmodule
                 margs[0] = jl_cstr_to_string("<macrocall>");
             margs[1] = jl_fieldref(lno, 0); // extract and allocate line number
             jl_rethrow_other(jl_new_struct(jl_loaderror_type, margs[0], margs[1],
-                                           ptls->exception_in_transit));
+                                           jl_current_exception()));
         }
     }
     ptls->world_age = last_age;

src/cgutils.cpp

@@ -2568,14 +2568,6 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg
     }
 }
 
-static Value *emit_exc_in_transit(jl_codectx_t &ctx)
-{
-    Value *pexc_in_transit = emit_bitcast(ctx, ctx.ptlsStates, T_pprjlvalue);
-    Constant *offset = ConstantInt::getSigned(T_int32,
-        offsetof(jl_tls_states_t, exception_in_transit) / sizeof(void*));
-    return ctx.builder.CreateInBoundsGEP(pexc_in_transit, ArrayRef<Value*>(offset), "jl_exception_in_transit");
-}
-
 static void emit_signal_fence(jl_codectx_t &ctx)
 {
 #if defined(_CPU_ARM_) || defined(_CPU_AARCH64_)

src/codegen.cpp

@@ -287,7 +287,10 @@ static Function *jlgetfield_func;
 static Function *jlmethod_func;
 static Function *jlgenericfunction_func;
 static Function *jlenter_func;
+static Function *jlcurrent_exception_func;
 static Function *jlleave_func;
+static Function *jl_restore_exc_stack_func;
+static Function *jl_exc_stack_state_func;
 static Function *jlegal_func;
 static Function *jl_alloc_obj_func;
 static Function *jl_newbits_func;
@@ -782,9 +785,8 @@ static void emit_write_barrier(jl_codectx_t&, Value*, Value*);
 
 static void jl_rethrow_with_add(const char *fmt, ...)
 {
-    jl_ptls_t ptls = jl_get_ptls_states();
-    if (jl_typeis(ptls->exception_in_transit, jl_errorexception_type)) {
-        char *str = jl_string_data(jl_fieldref(ptls->exception_in_transit,0));
+    if (jl_typeis(jl_current_exception(), jl_errorexception_type)) {
+        char *str = jl_string_data(jl_fieldref(jl_current_exception(),0));
         char buf[1024];
         va_list args;
         va_start(args, fmt);
@@ -1772,7 +1774,13 @@ static jl_value_t *static_apply_type(jl_codectx_t &ctx, const jl_cgval_t *args,
     size_t last_age = jl_get_ptls_states()->world_age;
     // call apply_type, but ignore errors. we know that will work in world 1.
     jl_get_ptls_states()->world_age = 1;
-    jl_value_t *result = jl_apply_with_saved_exception_state(v, nargs, 1);
+    jl_value_t *result;
+    JL_TRY {
+        result = jl_apply(v, nargs);
+    }
+    JL_CATCH {
+        result = NULL;
+    }
     jl_get_ptls_states()->world_age = last_age;
     return result;
 }
@@ -1854,7 +1862,13 @@ static jl_value_t *static_eval(jl_codectx_t &ctx, jl_value_t *ex, int sparams=tr
                     size_t last_age = jl_get_ptls_states()->world_age;
                     // here we know we're calling specific builtin functions that work in world 1.
                     jl_get_ptls_states()->world_age = 1;
-                    jl_value_t *result = jl_apply_with_saved_exception_state(v, n+1, 1);
+                    jl_value_t *result;
+                    JL_TRY {
+                        result = jl_apply(v, n+1);
+                    }
+                    JL_CATCH {
+                        result = NULL;
+                    }
                     jl_get_ptls_states()->world_age = last_age;
                     JL_GC_POP();
                     return result;
@@ -3761,7 +3775,9 @@ static void emit_stmtpos(jl_codectx_t &ctx, jl_value_t *expr, int ssaval_result)
                            ConstantInt::get(T_int32, jl_unbox_long(args[0])));
     }
     else if (head == pop_exc_sym) {
-        // FIXME
+        jl_cgval_t exc_stack_state = emit_expr(ctx, jl_exprarg(expr, 0));
+        assert(exc_stack_state.V && exc_stack_state.V->getType() == T_size);
+        ctx.builder.CreateCall(prepare_call(jl_restore_exc_stack_func), exc_stack_state.V);
         return;
     }
     else {
@@ -3916,8 +3932,7 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaval)
                 bnd = jl_get_binding_for_method_def(mod, (jl_sym_t*)mn);
             }
             JL_CATCH {
-                jl_ptls_t ptls = jl_get_ptls_states();
-                jl_value_t *e = ptls->exception_in_transit;
+                jl_value_t *e = jl_current_exception();
                 // errors. boo. root it somehow :(
                 bnd = jl_get_binding_wr(ctx.module, (jl_sym_t*)jl_gensym(), 1);
                 bnd->value = e;
@@ -3986,9 +4001,9 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaval)
         Value *val = emit_jlcall(ctx, jlnew_func, typ, &argv[1], nargs - 1);
         return mark_julia_type(ctx, val, true, ty);
     }
-    else if (head == exc_sym) { // *ptls->exception_in_transit
+    else if (head == exc_sym) {
         return mark_julia_type(ctx,
-                ctx.builder.CreateLoad(emit_exc_in_transit(ctx), /*isvolatile*/true),
+                ctx.builder.CreateCall(prepare_call(jlcurrent_exception_func)),
                 true, jl_any_type);
     }
     else if (head == copyast_sym) {
@@ -6179,6 +6194,14 @@ static std::unique_ptr<Module> emit_function(
 
             assert(jl_is_long(args[0]));
             int lname = jl_unbox_long(args[0]);
+            // Save exception stack depth at enter for use in pop_exc
+
+            Value *exc_stack_state =
+                ctx.builder.CreateCall(prepare_call(jl_exc_stack_state_func));
+            assert(!ctx.ssavalue_assigned.at(cursor));
+            ctx.SAvalues.at(cursor) = jl_cgval_t(exc_stack_state, NULL, false,
+                                                 (jl_value_t*)jl_ulong_type, NULL);
+            ctx.ssavalue_assigned.at(cursor) = true;
             CallInst *sj = ctx.builder.CreateCall(prepare_call(except_enter_func));
             // We need to mark this on the call site as well. See issue #6757
             sj->setCanReturnTwice();
@@ -6333,6 +6356,7 @@ static std::unique_ptr<Module> emit_function(
                     }
                 }
                 assert(found);
+                (void)found;
             }
             else {
                 terminator->removeFromParent();
@@ -7079,6 +7103,12 @@ static void init_julia_llvm_env(Module *m)
                          "jl_enter_handler", m);
     add_named_global(jlenter_func, &jl_enter_handler);
 
+    jlcurrent_exception_func =
+        Function::Create(FunctionType::get(T_prjlvalue, false),
+                         Function::ExternalLinkage,
+                         "jl_current_exception", m);
+    add_named_global(jlcurrent_exception_func, &jl_current_exception);
+
 #ifdef _OS_WINDOWS_
 #if defined(_CPU_X86_64_)
     juliapersonality_func = Function::Create(FunctionType::get(T_int32, true),
@@ -7118,6 +7148,18 @@ static void init_julia_llvm_env(Module *m)
                          "jl_pop_handler", m);
     add_named_global(jlleave_func, &jl_pop_handler);
 
+    jl_restore_exc_stack_func =
+        Function::Create(FunctionType::get(T_void, T_size, false),
+                         Function::ExternalLinkage,
+                         "jl_restore_exc_stack", m);
+    add_named_global(jl_restore_exc_stack_func, &jl_restore_exc_stack);
+
+    jl_exc_stack_state_func =
+        Function::Create(FunctionType::get(T_size, false),
+                         Function::ExternalLinkage,
+                         "jl_exc_stack_state", m);
+    add_named_global(jl_exc_stack_state_func, &jl_exc_stack_state);
+
     std::vector<Type *> args_2vals_callee_rooted(0);
     args_2vals_callee_rooted.push_back(PointerType::get(T_jlvalue, AddressSpace::CalleeRooted));
     args_2vals_callee_rooted.push_back(PointerType::get(T_jlvalue, AddressSpace::CalleeRooted));

src/dump.c

@@ -2337,7 +2337,6 @@ static void jl_finalize_serializer(jl_serializer_state *s)
 void jl_typemap_rehash(jl_typemap_t *ml, int8_t offs);
 static void jl_reinit_item(jl_value_t *v, int how, arraylist_t *tracee_list)
 {
-    jl_ptls_t ptls = jl_get_ptls_states();
     JL_TRY {
         switch (how) {
             case 1: { // rehash IdDict
@@ -2390,7 +2389,7 @@ static void jl_reinit_item(jl_value_t *v, int how, arraylist_t *tracee_list)
         jl_printf(JL_STDERR, "WARNING: error while reinitializing value ");
         jl_static_show(JL_STDERR, v);
         jl_printf(JL_STDERR, ":\n");
-        jl_static_show(JL_STDERR, ptls->exception_in_transit);
+        jl_static_show(JL_STDERR, jl_current_exception());
         jl_printf(JL_STDERR, "\n");
     }
 }

src/gc.c

@@ -234,7 +234,7 @@ static void run_finalizer(jl_ptls_t ptls, jl_value_t *o, jl_value_t *ff)
     }
     JL_CATCH {
         jl_printf(JL_STDERR, "error in running finalizer: ");
-        jl_static_show(JL_STDERR, ptls->exception_in_transit);
+        jl_static_show(JL_STDERR, jl_current_exception());
         jl_printf(JL_STDERR, "\n");
     }
 }
@@ -313,27 +313,11 @@ static void jl_gc_run_finalizers_in_list(jl_ptls_t ptls, arraylist_t *list)
     jl_value_t **items = (jl_value_t**)list->items;
     size_t len = list->len;
     JL_UNLOCK_NOGC(&finalizers_lock);
-    // from jl_apply_with_saved_exception_state; to hoist state saving out of the loop
-    jl_value_t *exc = ptls->exception_in_transit;
-    jl_array_t *bt = NULL;
-    jl_array_t *bt2 = NULL;
-    JL_GC_PUSH3(&exc, &bt, &bt2);
-    if (ptls->bt_size > 0) {
-        jl_get_backtrace(&bt, &bt2);
-        ptls->bt_size = 0;
-    }
     // run finalizers in reverse order they were added, so lower-level finalizers run last
     for (size_t i = len-4; i >= 2; i -= 2)
         run_finalizer(ptls, items[i], items[i + 1]);
     // first entries were moved last to make room for GC frame metadata
     run_finalizer(ptls, items[len-2], items[len-1]);
-    ptls->exception_in_transit = exc;
-    if (bt != NULL) {
-        // This is sufficient because bt2 roots the managed values
-        memcpy(ptls->bt_data, bt->data, jl_array_len(bt) * sizeof(void*));
-        ptls->bt_size = jl_array_len(bt);
-    }
-    JL_GC_POP();
     // matches the jl_gc_push_arraylist above
     JL_GC_POP();
 }
@@ -1844,6 +1828,8 @@ STATIC_INLINE int gc_mark_scan_obj32(jl_ptls_t ptls, jl_gc_mark_sp_t *sp, gc_mar
             goto obj32;                         \
         case GC_MARK_L_stack:                   \
             goto stack;                         \
+        case GC_MARK_L_excstack:                \
+            goto excstack;                      \
         case GC_MARK_L_module_binding:          \
             goto module_binding;                \
         default:                                \
@@ -1929,6 +1915,7 @@ JL_EXTENSION NOINLINE void gc_mark_loop(jl_ptls_t ptls, jl_gc_mark_sp_t sp)
         gc_mark_label_addrs[GC_MARK_L_obj16] = gc_mark_laddr(obj16);
         gc_mark_label_addrs[GC_MARK_L_obj32] = gc_mark_laddr(obj32);
         gc_mark_label_addrs[GC_MARK_L_stack] = gc_mark_laddr(stack);
+        gc_mark_label_addrs[GC_MARK_L_excstack] = gc_mark_laddr(excstack);
         gc_mark_label_addrs[GC_MARK_L_module_binding] = gc_mark_laddr(module_binding);
         return;
     }
@@ -2081,6 +2068,46 @@ stack: {
         }
     }
 
+excstack: {
+        // Scan an exception stack
+        gc_mark_exc_stack_t *stackitr = gc_pop_markdata(&sp, gc_mark_exc_stack_t);
+        jl_exc_stack_t *exc_stack = stackitr->s;
+        size_t itr = stackitr->itr;
+        size_t i = stackitr->i;
+        while (itr > 0) {
+            size_t bt_size = jl_exc_stack_bt_size(exc_stack, itr);
+            uintptr_t *bt_data = jl_exc_stack_bt_data(exc_stack, itr);
+            while (i+2 < bt_size) {
+                if (bt_data[i] != (uintptr_t)-1) {
+                    i++;
+                    continue;
+                }
+                // found an interpreter frame to mark
+                new_obj = (jl_value_t*)bt_data[i+1];
+                uintptr_t nptr = 0;
+                if (gc_try_setmark(new_obj, &nptr, &tag, &bits)) {
+                    stackitr->i = i + 3;
+                    stackitr->itr = itr;
+                    gc_repush_markdata(&sp, gc_mark_exc_stack_t);
+                    goto mark;
+                }
+                i += 3;
+            }
+            // mark the exception
+            new_obj = jl_exc_stack_exception(exc_stack, itr);
+            itr = jl_exc_stack_next(exc_stack, itr);
+            i = 0;
+            uintptr_t nptr = 0;
+            if (gc_try_setmark(new_obj, &nptr, &tag, &bits)) {
+                stackitr->i = i;
+                stackitr->itr = itr;
+                gc_repush_markdata(&sp, gc_mark_exc_stack_t);
+                goto mark;
+            }
+        }
+        goto pop;
+    }
+
 module_binding: {
         // Scan a module. see `gc_mark_binding_t`
         // Other fields of the module will be scanned after the bindings are scanned
@@ -2340,13 +2367,22 @@ mark: {
                 gc_mark_stack_push(&ptls->gc_cache, &sp, gc_mark_laddr(stack),
                                    &stackdata, sizeof(stackdata), 1);
             }
+            if (ta->exc_stack) {
+                gc_setmark_buf_(ptls, ta->exc_stack, bits, sizeof(jl_exc_stack_t) +
+                                sizeof(uintptr_t)*ta->exc_stack->reserved_size);
+                gc_mark_exc_stack_t stackdata = {ta->exc_stack, ta->exc_stack->top, 0};
+                gc_mark_stack_push(&ptls->gc_cache, &sp, gc_mark_laddr(excstack),
+                                   &stackdata, sizeof(stackdata), 1);
+            }
             const jl_datatype_layout_t *layout = jl_task_type->layout;
             assert(layout->fielddesc_type == 0);
             size_t nfields = layout->nfields;
             assert(nfields > 0);
             obj8_begin = (jl_fielddesc8_t*)jl_dt_layout_fields(layout);
             obj8_end = obj8_begin + nfields;
-            gc_mark_obj8_t markdata = {new_obj, obj8_begin, obj8_end, (9 << 2) | 1 | bits};
+            // assume tasks always reference young objects: set lowest bit
+            uintptr_t nptr = (9 << 2) | 1 | bits;
+            gc_mark_obj8_t markdata = {new_obj, obj8_begin, obj8_end, nptr};
             gc_mark_stack_push(&ptls->gc_cache, &sp, gc_mark_laddr(obj8),
                                &markdata, sizeof(markdata), 0);
             obj8 = (gc_mark_obj8_t*)sp.data;
@@ -2441,7 +2477,8 @@ static void jl_gc_queue_thread_local(jl_gc_mark_cache_t *gc_cache, jl_gc_mark_sp
 {
     gc_mark_queue_obj(gc_cache, sp, ptls2->current_task);
     gc_mark_queue_obj(gc_cache, sp, ptls2->root_task);
-    gc_mark_queue_obj(gc_cache, sp, ptls2->exception_in_transit);
+    if (ptls2->previous_exception)
+        gc_mark_queue_obj(gc_cache, sp, ptls2->previous_exception);
 }
 
 // mark the initial root set

src/gc.h

@@ -84,19 +84,21 @@ enum {
     GC_MARK_L_obj16,
     GC_MARK_L_obj32,
     GC_MARK_L_stack,
+    GC_MARK_L_excstack,
     GC_MARK_L_module_binding,
     _GC_MARK_L_MAX
 };
 
-/**
- * The `nptr` member of marking data records the number of pointers slots referenced by
- * an object to be used in the full collection heuristics as well as whether the object
- * references young objects.
- * `nptr >> 2` is the number of pointers fields referenced by the object.
- * The lowest bit of `nptr` is set if the object references young object.
- * The 2nd lowest bit of `nptr` is the GC old bits of the object after marking.
- * A `0x3` in the low bits means that the object needs to be in the remset.
- */
+// The following structs (`gc_mark_*_t`) contain iterator state used for the
+// scanning of various object types.
+//
+// The `nptr` member records the number of pointers slots referenced by
+// an object to be used in the full collection heuristics as well as whether the object
+// references young objects.
+// `nptr >> 2` is the number of pointers fields referenced by the object.
+// The lowest bit of `nptr` is set if the object references young object.
+// The 2nd lowest bit of `nptr` is the GC old bits of the object after marking.
+// A `0x3` in the low bits means that the object needs to be in the remset.
 
 // An generic object that's marked and needs to be scanned
 // The metadata might need update too (depend on the PC)
@@ -149,6 +151,13 @@ typedef struct {
     uintptr_t ub;
 } gc_mark_stackframe_t;
 
+// Exception stack data
+typedef struct {
+    jl_exc_stack_t *s; // Stack of exceptions
+    size_t itr;        // Iterator into exception stack
+    size_t i;          // Iterator into backtrace data for exception
+} gc_mark_exc_stack_t;
+
 // Module bindings. This is also the beginning of module scanning.
 // The loop will start marking other references in a module after the bindings are marked
 typedef struct {
@@ -176,6 +185,7 @@ union _jl_gc_mark_data {
     gc_mark_obj16_t obj16;
     gc_mark_obj32_t obj32;
     gc_mark_stackframe_t stackframe;
+    gc_mark_exc_stack_t excstackframe;
     gc_mark_binding_t binding;
     gc_mark_finlist_t finlist;
 };