From af743f31f4ada61d160daeb3ef275b51e3798d87 Mon Sep 17 00:00:00 2001
From: Gabor Greif <gabor@dfinity.org>
Date: Fri, 17 Jan 2025 15:49:38 +0100
Subject: [PATCH] Mitigate timer job submission failures (#4852)

This deals with the (unlikely) possibility that the send queue is not full when the timer servicing action is submitted, but becomes full while submitting the user jobs. Now we catch the failure and re-add (single-expiration) jobs to the start of the priority queue.

This is the missing piece to #4846.

This is an incremental change, so that we don't have to touch the happy path. A rewrite would be justified to collapse gathering and self-sends.

There is an optimisation realised in `@prune`.
---
 src/prelude/internals.mo         | 20 ++++++++++++++++++--
 test/fail/ok/illegal-await.tc.ok | 10 +++++-----
 2 files changed, 23 insertions(+), 7 deletions(-)

diff --git a/src/prelude/internals.mo b/src/prelude/internals.mo
index aed9fe51f7c..70da6648924 100644
--- a/src/prelude/internals.mo
+++ b/src/prelude/internals.mo
@@ -544,7 +544,7 @@ func @prune(n : ?@Node) : ?@Node = switch n {
     if (n.expire[0] == 0) {
       @prune(n.post) // by corollary
     } else {
-      ?{ n with pre = @prune(n.pre); post = @prune(n.post) }
+      ?{ n with pre = @prune(n.pre) }
     }
   }
 };
@@ -612,9 +612,25 @@ func @timer_helper() : async () {
   ignore (prim "global_timer_set" : Nat64 -> Nat64) exp;
   if (exp == 0) @timers := null;
 
+  var failed : Nat64 = 0;
+  func reinsert(job : () -> async ()) {
+    if (failed == 0) {
+      @timers := @prune @timers;
+      ignore (prim "global_timer_set" : Nat64 -> Nat64) 1
+    };
+    failed += 1;
+    @timers := ?(switch @timers {
+      case (?{ id = 0; pre; post; job = j; expire; delay })
+        // push top node's contents into pre
+        ({ expire = [var failed]; id = 0; delay; job; post
+         ; pre = ?{ id = 0; expire; pre; post = null; delay; job = j } });
+      case _ ({ expire = [var failed]; id = 0; delay = null; job; pre = null; post = @timers })
+    })
+  };
+
   for (o in thunks.vals()) {
     switch o {
-      case (?thunk) ignore thunk();
+      case (?thunk) try ignore thunk() catch _ reinsert thunk;
       case _ return
     }
   }
diff --git a/test/fail/ok/illegal-await.tc.ok b/test/fail/ok/illegal-await.tc.ok
index 3e32e84fd34..0e695b1bc02 100644
--- a/test/fail/ok/illegal-await.tc.ok
+++ b/test/fail/ok/illegal-await.tc.ok
@@ -24,14 +24,14 @@ illegal-await.mo:24.11: info, start of scope $@anon-async-24.11 mentioned in err
 illegal-await.mo:26.5: info, end of scope $@anon-async-24.11 mentioned in error at illegal-await.mo:25.7-25.14
 illegal-await.mo:22.10: info, start of scope $@anon-async-22.10 mentioned in error at illegal-await.mo:25.7-25.14
 illegal-await.mo:27.3: info, end of scope $@anon-async-22.10 mentioned in error at illegal-await.mo:25.7-25.14
-illegal-await.mo:35.11-35.12: type error [M0087], ill-scoped await: expected async type from current scope $Rec, found async type from other scope $__15
+illegal-await.mo:35.11-35.12: type error [M0087], ill-scoped await: expected async type from current scope $Rec, found async type from other scope $__19
   scope $Rec is illegal-await.mo:33.44-40.2
-  scope $__15 is illegal-await.mo:33.1-40.2
+  scope $__19 is illegal-await.mo:33.1-40.2
 illegal-await.mo:33.44: info, start of scope $Rec mentioned in error at illegal-await.mo:35.5-35.12
 illegal-await.mo:40.1: info, end of scope $Rec mentioned in error at illegal-await.mo:35.5-35.12
-illegal-await.mo:33.1: info, start of scope $__15 mentioned in error at illegal-await.mo:35.5-35.12
-illegal-await.mo:40.1: info, end of scope $__15 mentioned in error at illegal-await.mo:35.5-35.12
+illegal-await.mo:33.1: info, start of scope $__19 mentioned in error at illegal-await.mo:35.5-35.12
+illegal-await.mo:40.1: info, end of scope $__19 mentioned in error at illegal-await.mo:35.5-35.12
 illegal-await.mo:38.20-38.21: type error [M0096], expression of type
-  async<$__15> ()
+  async<$__19> ()
 cannot produce expected type
   async<$Rec> ()