Mitigate timer job submission failures (#4852)

This deals with the (unlikely) possibility that the send queue is not full when the timer servicing action is submitted, but becomes full while submitting the user jobs. Now we catch the failure and re-add (single-expiration) jobs to the start of the priority queue.

This is the missing piece to #4846.

This is an incremental change, so that we don't have to touch the happy path. A rewrite would be justified to collapse gathering and self-sends.

There is an optimisation realised in `@prune`.

src/prelude/internals.mo

@@ -544,7 +544,7 @@ func @prune(n : ?@Node) : ?@Node = switch n {
     if (n.expire[0] == 0) {
       @prune(n.post) // by corollary
     } else {
-      ?{ n with pre = @prune(n.pre); post = @prune(n.post) }
+      ?{ n with pre = @prune(n.pre) }
     }
   }
 };
@@ -612,9 +612,25 @@ func @timer_helper() : async () {
   ignore (prim "global_timer_set" : Nat64 -> Nat64) exp;
   if (exp == 0) @timers := null;
 
+  var failed : Nat64 = 0;
+  func reinsert(job : () -> async ()) {
+    if (failed == 0) {
+      @timers := @prune @timers;
+      ignore (prim "global_timer_set" : Nat64 -> Nat64) 1
+    };
+    failed += 1;
+    @timers := ?(switch @timers {
+      case (?{ id = 0; pre; post; job = j; expire; delay })
+        // push top node's contents into pre
+        ({ expire = [var failed]; id = 0; delay; job; post
+         ; pre = ?{ id = 0; expire; pre; post = null; delay; job = j } });
+      case _ ({ expire = [var failed]; id = 0; delay = null; job; pre = null; post = @timers })
+    })
+  };
+
   for (o in thunks.vals()) {
     switch o {
-      case (?thunk) ignore thunk();
+      case (?thunk) try ignore thunk() catch _ reinsert thunk;
       case _ return
     }
   }

test/fail/ok/illegal-await.tc.ok

@@ -24,14 +24,14 @@ illegal-await.mo:24.11: info, start of scope [email protected] mentioned in err
 illegal-await.mo:26.5: info, end of scope [email protected] mentioned in error at illegal-await.mo:25.7-25.14
 illegal-await.mo:22.10: info, start of scope [email protected] mentioned in error at illegal-await.mo:25.7-25.14
 illegal-await.mo:27.3: info, end of scope [email protected] mentioned in error at illegal-await.mo:25.7-25.14
-illegal-await.mo:35.11-35.12: type error [M0087], ill-scoped await: expected async type from current scope $Rec, found async type from other scope $__15
+illegal-await.mo:35.11-35.12: type error [M0087], ill-scoped await: expected async type from current scope $Rec, found async type from other scope $__19
   scope $Rec is illegal-await.mo:33.44-40.2
-  scope $__15 is illegal-await.mo:33.1-40.2
+  scope $__19 is illegal-await.mo:33.1-40.2
 illegal-await.mo:33.44: info, start of scope $Rec mentioned in error at illegal-await.mo:35.5-35.12
 illegal-await.mo:40.1: info, end of scope $Rec mentioned in error at illegal-await.mo:35.5-35.12
-illegal-await.mo:33.1: info, start of scope $__15 mentioned in error at illegal-await.mo:35.5-35.12
-illegal-await.mo:40.1: info, end of scope $__15 mentioned in error at illegal-await.mo:35.5-35.12
+illegal-await.mo:33.1: info, start of scope $__19 mentioned in error at illegal-await.mo:35.5-35.12
+illegal-await.mo:40.1: info, end of scope $__19 mentioned in error at illegal-await.mo:35.5-35.12
 illegal-await.mo:38.20-38.21: type error [M0096], expression of type
-  async<$__15> ()
+  async<$__19> ()
 cannot produce expected type
   async<$Rec> ()