Time-limited join for watcher webservice

mountaineer/cli.py

@@ -16,7 +16,6 @@
 from traceback import format_exception
 from typing import Any, Callable, MutableMapping
 
-from click import secho
 from fastapi import Request
 from rich.traceback import install as rich_traceback_install
 
@@ -263,9 +262,8 @@ def stop(self, hard_timeout: float = 5.0):
             hard_timeout -= 1
 
         if hard_timeout == 0:
-            secho(
-                f"Server shutdown reached hard timeout deadline: {self.is_alive()}",
-                fg="red",
+            CONSOLE.print(
+                f"[bold red]Server shutdown reached hard timeout deadline: {self.is_alive()}",
             )
 
         # As a last resort we send a hard termination signal
@@ -413,7 +411,7 @@ def graceful_shutdown(signum, frame):
             current_process.stop()
         if watcher_webservice is not None:
             watcher_webservice.stop()
-        secho("Services shutdown, now exiting...", fg="yellow")
+        CONSOLE.print("[yellow]Services shutdown, now exiting...")
         exit(0)
 
     signal(SIGINT, graceful_shutdown)

mountaineer/watch_server.py

@@ -71,6 +71,7 @@ def start(self):
         if self.has_started:
             raise Exception("WatcherWebservice has already started")
 
+        # UvicornThreads are daemon threads by default
         self.webservice_thread = UvicornThread(
             app=self.app,
             port=self.port,
@@ -80,16 +81,35 @@ def start(self):
         LOGGER.debug("Starting WatcherWebservice on port %d", self.port)
         self.webservice_thread.start()
 
-        self.monitor_build_thread = Thread(target=self.monitor_builds)
+        self.monitor_build_thread = Thread(target=self.monitor_builds, daemon=True)
         self.monitor_build_thread.start()
 
         self.has_started = True
 
-    def stop(self):
+    def stop(self, wait_for_completion: int = 1) -> bool:
+        """
+        Attempts to stop the separate WatcherWebservice threads. We will send a termination
+        signal to the threads and wait the desired interval for full completion. If the threads
+        haven't exited after the interval, we will return False. Clients can then decide whether
+        to send a harder termination signal to terminate the threads on the OS level.
+
+        """
+        success: bool = True
         if self.webservice_thread is not None:
             self.webservice_thread.stop()
-            self.webservice_thread.join()
+            self.webservice_thread.join(wait_for_completion)
         if self.monitor_build_thread is not None:
             self.notification_queue.put(None)
-            self.monitor_build_thread.join()
-        LOGGER.info("WatcherWebservice has stopped")
+            self.monitor_build_thread.join(wait_for_completion)
+
+        if (self.webservice_thread and self.webservice_thread.is_alive()) or (
+            self.monitor_build_thread and self.monitor_build_thread.is_alive()
+        ):
+            success = False
+            LOGGER.info(
+                f"WatcherWebservice still has outstanding threads: {self.webservice_thread} {self.monitor_build_thread}"
+            )
+        else:
+            LOGGER.info("WatcherWebservice has fully stopped")
+
+        return success