Making services more debuggable (#1690)

* fixes an issue with missing logs from container * updated error message * maked debugging easier when container fails * triggering ci agian * fix previous Co-authored-by: Andrei Neagu <[email protected]> Co-authored-by: Odei Maiz <[email protected]>
ITISFoundation · Aug 11, 2020 · abfced4 · abfced4
1 parent e23b47d
commit abfced4
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 4 deletions.
diff --git a/services/sidecar/src/simcore_service_sidecar/executor.py b/services/sidecar/src/simcore_service_sidecar/executor.py
@@ -74,8 +74,8 @@ async def run(self):
             await self._post_messages(
                 LogType.LOG, "[sidecar]...task completed successfully."
             )
-        except exceptions.SidecarException:
-            await self._post_messages(LogType.LOG, "[sidecar]...task failed.")
+        except exceptions.SidecarException as e:
+            await self._post_messages(LogType.LOG, f"[sidecar]...task failed: {str(e)}")
             raise
 
     async def preprocess(self):

diff --git a/services/sidecar/src/simcore_service_sidecar/log_parser.py b/services/sidecar/src/simcore_service_sidecar/log_parser.py
@@ -3,6 +3,7 @@
 import re
 from enum import Enum
 from pathlib import Path
+import aiodocker
 from typing import Awaitable, Callable, Tuple, Union
 
 import aiofiles
@@ -78,8 +79,14 @@ async def monitor_logs_task(
 async def _monitor_docker_container(
     container: DockerContainer, log_cb: Awaitable[Callable[[LogType, str], None]]
 ) -> None:
-    async for line in container.log(stdout=True, stderr=True, follow=True):
-        log_type, parsed_line = await parse_line(line)
+    try:
+        async for line in container.log(stdout=True, stderr=True, follow=True):
+            log_type, parsed_line = await parse_line(line)
+    except aiodocker.exceptions.DockerError as e:
+        log_type, parsed_line = await parse_line(
+            f"Could not recover logs because: {str(e)}"
+        )
+    finally:
         await log_cb(log_type, parsed_line)