Fix some issues with precompilation with threads on (#53697)

This doesn't fix all issues, but should make some of them more correct
than before. These fixes may also enable exiting julia with threads
running safer in the future.

src/scheduler.c

@@ -129,7 +129,7 @@ void jl_parallel_gc_threadfun(void *arg)
 
     // initialize this thread (set tid and create heap)
     jl_ptls_t ptls = jl_init_threadtls(targ->tid);
-
+    (void)jl_atomic_fetch_add_relaxed(&nrunning, -1);
     // wait for all threads
     jl_gc_state_set(ptls, JL_GC_STATE_WAITING, JL_GC_STATE_UNSAFE);
     uv_barrier_wait(targ->barrier);
@@ -158,7 +158,7 @@ void jl_concurrent_gc_threadfun(void *arg)
 
     // initialize this thread (set tid and create heap)
     jl_ptls_t ptls = jl_init_threadtls(targ->tid);
-
+    (void)jl_atomic_fetch_add_relaxed(&nrunning, -1);
     // wait for all threads
     jl_gc_state_set(ptls, JL_GC_STATE_WAITING, JL_GC_STATE_UNSAFE);
     uv_barrier_wait(targ->barrier);
@@ -183,8 +183,6 @@ void jl_threadfun(void *arg)
     jl_init_stack_limits(0, &stack_lo, &stack_hi);
     // warning: this changes `jl_current_task`, so be careful not to call that from this function
     jl_task_t *ct = jl_init_root_task(ptls, stack_lo, stack_hi);
-    int wasrunning = jl_atomic_fetch_add_relaxed(&nrunning, 1);
-    assert(wasrunning); (void)wasrunning;
     JL_GC_PROMISE_ROOTED(ct);
 
     // wait for all threads
@@ -438,6 +436,7 @@ JL_DLLEXPORT jl_task_t *jl_task_get_next(jl_value_t *trypoptask, jl_value_t *q,
         jl_ptls_t ptls = ct->ptls;
         if (sleep_check_after_threshold(&start_cycles) || (ptls->tid == jl_atomic_load_relaxed(&io_loop_tid) && (!jl_atomic_load_relaxed(&_threadedregion) || wait_empty))) {
             // acquire sleep-check lock
+            assert(jl_atomic_load_relaxed(&ptls->sleep_check_state) == not_sleeping);
             jl_atomic_store_relaxed(&ptls->sleep_check_state, sleeping);
             jl_fence(); // [^store_buffering_1]
             JL_PROBE_RT_SLEEP_CHECK_SLEEP(ptls);
@@ -544,10 +543,12 @@ JL_DLLEXPORT jl_task_t *jl_task_get_next(jl_value_t *trypoptask, jl_value_t *q,
                 // so make sure io_loop_tid is notified to check wait_empty
                 // TODO: this also might be a good time to check again that
                 // libuv's queue is truly empty, instead of during delete_thread
-                if (ptls->tid != jl_atomic_load_relaxed(&io_loop_tid)) {
-                    uv_mutex_lock(&ptls->sleep_lock);
-                    uv_cond_wait(&ptls->wake_signal, &ptls->sleep_lock);
-                    uv_mutex_unlock(&ptls->sleep_lock);
+                int16_t tid2 = 0;
+                if (ptls->tid != tid2) {
+                    jl_ptls_t ptls2 = jl_atomic_load_relaxed(&jl_all_tls_states)[tid2];
+                    uv_mutex_lock(&ptls2->sleep_lock);
+                    uv_cond_signal(&ptls2->wake_signal);
+                    uv_mutex_unlock(&ptls2->sleep_lock);
                 }
             }
 
@@ -556,16 +557,19 @@ JL_DLLEXPORT jl_task_t *jl_task_get_next(jl_value_t *trypoptask, jl_value_t *q,
             int8_t gc_state = jl_gc_safe_enter(ptls);
             uv_mutex_lock(&ptls->sleep_lock);
             while (may_sleep(ptls)) {
-                task = wait_empty;
-                if (ptls->tid == 0 && task && jl_atomic_load_relaxed(&nrunning) == 0) {
-                    wasrunning = jl_atomic_fetch_add_relaxed(&nrunning, 1);
-                    assert(!wasrunning);
-                    wasrunning = !set_not_sleeping(ptls);
-                    assert(!wasrunning);
-                    JL_PROBE_RT_SLEEP_CHECK_TASK_WAKE(ptls);
-                    if (!ptls->finalizers_inhibited)
-                        ptls->finalizers_inhibited++; // this annoyingly is rather sticky (we should like to reset it at the end of jl_task_wait_empty)
-                    break;
+                if (ptls->tid == 0) {
+                    task = wait_empty;
+                    if (task && jl_atomic_load_relaxed(&nrunning) == 0) {
+                        wasrunning = jl_atomic_fetch_add_relaxed(&nrunning, 1);
+                        assert(!wasrunning);
+                        wasrunning = !set_not_sleeping(ptls);
+                        assert(!wasrunning);
+                        JL_PROBE_RT_SLEEP_CHECK_TASK_WAKE(ptls);
+                        if (!ptls->finalizers_inhibited)
+                            ptls->finalizers_inhibited++; // this annoyingly is rather sticky (we should like to reset it at the end of jl_task_wait_empty)
+                        break;
+                    }
+                    task = NULL;
                 }
                 // else should we warn the user of certain deadlock here if tid == 0 && nrunning == 0?
                 uv_cond_wait(&ptls->wake_signal, &ptls->sleep_lock);