Don't suspend schedulers if terminating and reset steal_attempts on w…

…ake (#2447)

Prior to this commit, a scheduler thread could potentially suspend even
if it was supposed to be terminating. This commit makes it so that it
only goes down the suspend path if the thread is not terminating.

Additionally, we now set `steal_attempts = 0` on waking from a suspend
to ensure that the scheduler thread will try to steal from all other
threads prior to suspending again.

src/libponyrt/sched/scheduler.c

@@ -416,9 +416,11 @@ static pony_actor_t* steal(scheduler_t* sched)
       {
         // if we're the highest active scheduler thread
         // and there are more active schedulers than the minimum requested
+        // and we're not terminating
         if ((sched == &scheduler[current_active_scheduler_count - 1])
-          && (current_active_scheduler_count > min_scheduler_count) &&
-          !atomic_exchange_explicit(&scheduler_count_changing, true,
+          && (current_active_scheduler_count > min_scheduler_count)
+          && (!sched->terminate)
+          && !atomic_exchange_explicit(&scheduler_count_changing, true,
             memory_order_acquire))
         {
           // let sched 0 know we're suspending
@@ -457,6 +459,10 @@ static pony_actor_t* steal(scheduler_t* sched)
 
           // dtrace resume notification
           DTRACE1(THREAD_RESUME, (uintptr_t)sched);
+
+          // reset steal_attempts so we try to steal from all other schedulers
+          // prior to suspending again
+          steal_attempts = 0;
         }
         else if(!sched->asio_noisy)
         {