From 2eee758aab896d09b48c820a13595bd29f657a40 Mon Sep 17 00:00:00 2001 From: "Mark E. Haase" Date: Fri, 24 Jan 2020 15:38:50 -0500 Subject: [PATCH 1/3] Add new trio_loop config flag that runs Trio on main thread This flag causes Trio to run on the main thread, and puts ZMQ and other stuff on a background thread. When trio runs on the main thread, async cells are executed as tasks inside a global nursery. A global nursery named GLOBAL_NURSERY is exported in builtins so that background tasks can continue to run even after a cell finishes executing. Exceptions in background tasks are caught and displayed in the current cell. This implicitly runs `%autoawait trio` when the kernel starts and then disables the %autoawait magic so that users can't switch to a different loop while in Trio loop mode. Co-authored-by: Brian Mackintosh --- ipykernel/kernelapp.py | 31 +++++++++++++++++++++++++++---- ipykernel/trio_runner.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 4 deletions(-) create mode 100644 ipykernel/trio_runner.py diff --git a/ipykernel/kernelapp.py b/ipykernel/kernelapp.py index d683e8263..afd1dbdc4 100644 --- a/ipykernel/kernelapp.py +++ b/ipykernel/kernelapp.py @@ -12,6 +12,7 @@ import signal import traceback import logging +import threading from tornado import ioloop import zmq @@ -72,6 +73,10 @@ {'IPKernelApp' : {'pylab' : 'auto'}}, """Pre-load matplotlib and numpy for interactive use with the default matplotlib backend."""), + 'trio-loop' : ( + {'InteractiveShell' : {'trio_loop' : False}}, + 'Enable Trio as main event loop.' + ), }) # inherit flags&aliases for any IPython shell apps @@ -147,6 +152,7 @@ def abs_connection_file(self): # streams, etc. no_stdout = Bool(False, help="redirect stdout to the null device").tag(config=True) no_stderr = Bool(False, help="redirect stderr to the null device").tag(config=True) + trio_loop = Bool(False, help="Set main event loop.").tag(config=True) quiet = Bool(True, help="Only send stdout/stderr to output stream").tag(config=True) outstream_class = DottedObjectName('ipykernel.iostream.OutStream', help="The importstring for the OutStream factory").tag(config=True) @@ -579,10 +585,27 @@ def start(self): self.poller.start() self.kernel.start() self.io_loop = ioloop.IOLoop.current() - try: - self.io_loop.start() - except KeyboardInterrupt: - pass + if self.trio_loop: + from ipykernel.trio_runner import TrioRunner + import warnings + tr = TrioRunner() + self.kernel.shell.set_trio_runner(tr) + self.kernel.shell.run_line_magic('autoawait', 'trio') + self.kernel.shell.magics_manager.magics['line']['autoawait'] = \ + lambda _: warnings.warn("Autoawait isn't allowed in Trio " + "background loop mode.") + bg_thread = threading.Thread(target=self.io_loop.start, daemon=True, + name='TornadoBackground') + bg_thread.start() + try: + tr.run() + except KeyboardInterrupt: + pass + else: + try: + self.io_loop.start() + except KeyboardInterrupt: + pass launch_new_instance = IPKernelApp.launch_instance diff --git a/ipykernel/trio_runner.py b/ipykernel/trio_runner.py new file mode 100644 index 000000000..443fb0409 --- /dev/null +++ b/ipykernel/trio_runner.py @@ -0,0 +1,38 @@ +import builtins +import logging +import traceback + +import trio + + +class TrioRunner: + def __init__(self): + self._trio_token = None + + def run(self): + def log_nursery_exc(exc): + exc = '\n'.join(traceback.format_exception(type(exc), exc, + exc.__traceback__)) + logging.error('An exception occurred in a global nursery task.\n%s', + exc) + + async def trio_main(): + self._trio_token = trio.hazmat.current_trio_token() + async with trio.open_nursery() as nursery: + # TODO This hack prevents the nursery from cancelling all child + # tasks when an uncaught exception occurs, but it's ugly. + nursery._add_exc = log_nursery_exc + builtins.GLOBAL_NURSERY = nursery + await trio.sleep_forever() + + trio.run(trio_main) + + def __call__(self, async_fn): + async def loc(coro): + """ + We need the dummy no-op async def to protect from + trio's internal. See https://github.com/python-trio/trio/issues/89 + """ + return await coro + + return trio.from_thread.run(loc, async_fn, trio_token=self._trio_token) From 9c246c8260d2308287cfc261ca09bd0b2c92e5d8 Mon Sep 17 00:00:00 2001 From: "Mark E. Haase" Date: Fri, 24 Jan 2020 15:49:46 -0500 Subject: [PATCH 2/3] Move a bit more of the implementation into trio_runner.py This minimizes the amount of change in kernelapp.py. Co-authored-by: Brian Mackintosh --- ipykernel/kernelapp.py | 11 +---------- ipykernel/trio_runner.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/ipykernel/kernelapp.py b/ipykernel/kernelapp.py index afd1dbdc4..75d4d5e64 100644 --- a/ipykernel/kernelapp.py +++ b/ipykernel/kernelapp.py @@ -12,7 +12,6 @@ import signal import traceback import logging -import threading from tornado import ioloop import zmq @@ -587,16 +586,8 @@ def start(self): self.io_loop = ioloop.IOLoop.current() if self.trio_loop: from ipykernel.trio_runner import TrioRunner - import warnings tr = TrioRunner() - self.kernel.shell.set_trio_runner(tr) - self.kernel.shell.run_line_magic('autoawait', 'trio') - self.kernel.shell.magics_manager.magics['line']['autoawait'] = \ - lambda _: warnings.warn("Autoawait isn't allowed in Trio " - "background loop mode.") - bg_thread = threading.Thread(target=self.io_loop.start, daemon=True, - name='TornadoBackground') - bg_thread.start() + tr.initialize(self.kernel, self.io_loop) try: tr.run() except KeyboardInterrupt: diff --git a/ipykernel/trio_runner.py b/ipykernel/trio_runner.py index 443fb0409..ad4c30e72 100644 --- a/ipykernel/trio_runner.py +++ b/ipykernel/trio_runner.py @@ -1,6 +1,8 @@ import builtins import logging +import threading import traceback +import warnings import trio @@ -9,6 +11,16 @@ class TrioRunner: def __init__(self): self._trio_token = None + def initialize(self, kernel, io_loop): + kernel.shell.set_trio_runner(self) + kernel.shell.run_line_magic('autoawait', 'trio') + kernel.shell.magics_manager.magics['line']['autoawait'] = \ + lambda _: warnings.warn("Autoawait isn't allowed in Trio " + "background loop mode.") + bg_thread = threading.Thread(target=io_loop.start, daemon=True, + name='TornadoBackground') + bg_thread.start() + def run(self): def log_nursery_exc(exc): exc = '\n'.join(traceback.format_exception(type(exc), exc, From 3bb1bf518d5b103ac02babdaf8143a8c9e84dc9f Mon Sep 17 00:00:00 2001 From: "Mark E. Haase" Date: Fri, 31 Jan 2020 09:32:17 -0500 Subject: [PATCH 3/3] Add support for kernel interrupts The trio runner didn't support kernel interrupts: it would just continue to hang. I'm not sure why, but kernelapp.py ignores SIGINT. So in this commit, the Trio runner registers a new SIGINT handler that uses Trio cancel scopes to cancel the currently executing cell. The downside to this approach is that if a cell hangs in a loop that never reaches a Trio checkpoint, then the cancellation will have no effect. We could have sent SIGINT to the main thread (i.e. SIG_DFL), but that would have the effect of potentially raising SIGINT on one of the background tasks, which would likely force you to restart the kernel. --- ipykernel/trio_runner.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/ipykernel/trio_runner.py b/ipykernel/trio_runner.py index ad4c30e72..4ba1da196 100644 --- a/ipykernel/trio_runner.py +++ b/ipykernel/trio_runner.py @@ -1,5 +1,6 @@ import builtins import logging +import signal import threading import traceback import warnings @@ -9,6 +10,7 @@ class TrioRunner: def __init__(self): + self._cell_cancel_scope = None self._trio_token = None def initialize(self, kernel, io_loop): @@ -21,7 +23,15 @@ def initialize(self, kernel, io_loop): name='TornadoBackground') bg_thread.start() + def interrupt(self, signum, frame): + if self._cell_cancel_scope: + self._cell_cancel_scope.cancel() + else: + raise Exception('Kernel interrupted but no cell is running') + def run(self): + old_sig = signal.signal(signal.SIGINT, self.interrupt) + def log_nursery_exc(exc): exc = '\n'.join(traceback.format_exception(type(exc), exc, exc.__traceback__)) @@ -38,13 +48,13 @@ async def trio_main(): await trio.sleep_forever() trio.run(trio_main) + signal.signal(signal.SIGINT, old_sig) def __call__(self, async_fn): async def loc(coro): - """ - We need the dummy no-op async def to protect from - trio's internal. See https://github.com/python-trio/trio/issues/89 - """ - return await coro + self._cell_cancel_scope = trio.CancelScope() + with self._cell_cancel_scope: + return await coro + self._cell_cancel_scope = None return trio.from_thread.run(loc, async_fn, trio_token=self._trio_token)