remote: add support for pystuck

To allow debugging in production, we need a way to introspect objects in
the coordinator and exporter without stopping the process. This is
possible with pystuck.

We add it as a dev dependency only, as it's only needed in special
circumstances and can be installed manually in that case. It is only
enabled when the --pystuck option is passed on startup.

Signed-off-by: Jan Luebbe <[email protected]>

labgrid/remote/coordinator.py

@@ -1002,6 +1002,9 @@ async def serve(listen, cleanup) -> None:
     logging.debug("Starting server")
     await server.start()
 
+    if inspect:
+        inspect.coordinator = coordinator
+
     async def server_graceful_shutdown():
         logging.info("Starting graceful shutdown...")
         # Shuts down the server with 0 seconds of grace period. During the
@@ -1025,6 +1028,8 @@ def main():
         help="coordinator listening host and port",
     )
     parser.add_argument("-d", "--debug", action="store_true", default=False, help="enable debug mode")
+    parser.add_argument("--pystuck", action="store_true", help="enable pystuck")
+    parser.add_argument("--pystuck-port", metavar="PORT", type=int, default=6666, help="use a different pystuck port than 6666")
 
     args = parser.parse_args()
 
@@ -1033,6 +1038,17 @@ def main():
     loop = asyncio.new_event_loop()
     asyncio.set_event_loop(loop)
 
+    global inspect
+    if args.pystuck:
+        from types import SimpleNamespace
+        inspect = SimpleNamespace()
+        inspect.loop = loop
+
+        import pystuck
+        pystuck.run_server(port=args.pystuck_port)
+    else:
+        inspect = None
+
     cleanup = []
     loop.set_debug(True)
     try:

labgrid/remote/exporter.py

@@ -1005,6 +1005,10 @@ async def update_resource(self, group_name, resource_name):
 
 async def amain(config) -> bool:
     exporter = Exporter(config)
+
+    if inspect:
+        inspect.exporter = exporter
+
     await exporter.run()
 
 
@@ -1044,6 +1048,8 @@ def main():
         default=False,
         help="enable isolated mode (always request SSH forwards)",
     )
+    parser.add_argument("--pystuck", action="store_true", help="enable pystuck")
+    parser.add_argument("--pystuck-port", metavar="PORT", type=int, default=6667, help="use a different pystuck port than 6667")
     parser.add_argument("resources", metavar="RESOURCES", type=str, help="resource config file name")
 
     args = parser.parse_args()
@@ -1065,6 +1071,17 @@ def main():
     loop = asyncio.new_event_loop()
     asyncio.set_event_loop(loop)
 
+    global inspect
+    if args.pystuck:
+        from types import SimpleNamespace
+        inspect = SimpleNamespace()
+        inspect.loop = loop
+
+        import pystuck
+        pystuck.run_server(port=args.pystuck_port)
+    else:
+        inspect = None
+
     asyncio.run(amain(config), debug=bool(args.debug))
 
     if reexec:

pyproject.toml

@@ -88,6 +88,7 @@ dev = [
     "pytest-mock>=3.6.1",
     "pylint>=3.0.0",
     "ruff>=0.5.7",
+    "pystuck",
 
     # GRPC Channelz support
     "grpcio-channelz>=1.64.1, <2.0.0",