From 0c5b8194fa2560f2b23780c64e9df548717cf754 Mon Sep 17 00:00:00 2001 From: Lukasz Mitusinski Date: Thu, 19 Apr 2018 15:13:03 +0200 Subject: [PATCH 1/2] #7130 groovy magic epanded to other languages --- beakerx/beakerx/install.py | 9 +- beakerx/beakerx_magics/__init__.py | 7 +- beakerx/beakerx_magics/clojure_magic.py | 39 +++++ beakerx/beakerx_magics/groovy_magic.py | 65 +------- beakerx/beakerx_magics/java_magic.py | 39 +++++ beakerx/beakerx_magics/kernel_magic.py | 110 +++++++++++++ beakerx/beakerx_magics/kotlin_magic.py | 39 +++++ beakerx/beakerx_magics/magic_kernel_comm.py | 42 ----- beakerx/beakerx_magics/scala_magic.py | 39 +++++ beakerx/beakerx_magics/sql_magic.py | 39 +++++ doc/python/KernelsMagics.ipynb | 167 ++++++++++++++++++++ 11 files changed, 489 insertions(+), 106 deletions(-) create mode 100644 beakerx/beakerx_magics/clojure_magic.py create mode 100644 beakerx/beakerx_magics/java_magic.py create mode 100644 beakerx/beakerx_magics/kernel_magic.py create mode 100644 beakerx/beakerx_magics/kotlin_magic.py delete mode 100644 beakerx/beakerx_magics/magic_kernel_comm.py create mode 100644 beakerx/beakerx_magics/scala_magic.py create mode 100644 beakerx/beakerx_magics/sql_magic.py create mode 100644 doc/python/KernelsMagics.ipynb diff --git a/beakerx/beakerx/install.py b/beakerx/beakerx/install.py index 169aa2fa8c..5077ecad59 100644 --- a/beakerx/beakerx/install.py +++ b/beakerx/beakerx/install.py @@ -129,8 +129,13 @@ def _install_magics(): with open(os.path.join(dir_path, 'ipython_config.py'), 'w+') as ipython_config: ipython_config.write("c = get_config()\n") ipython_config.write("c.InteractiveShellApp.extensions = [" - "'beakerx_magics.magic_kernel_comm',\n" - "'beakerx_magics.groovy_magic'\n" + "'beakerx_magics.kernel_magic',\n" + "'beakerx_magics.groovy_magic',\n" + "'beakerx_magics.clojure_magic',\n" + "'beakerx_magics.kotlin_magic',\n" + "'beakerx_magics.scala_magic',\n" + "'beakerx_magics.sql_magic',\n" + "'beakerx_magics.java_magic'\n" "]\n") def _set_conf_privileges(): diff --git a/beakerx/beakerx_magics/__init__.py b/beakerx/beakerx_magics/__init__.py index 48931fb16c..363cc4a6dc 100644 --- a/beakerx/beakerx_magics/__init__.py +++ b/beakerx/beakerx_magics/__init__.py @@ -12,5 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +from .kernel_magic import * +from .clojure_magic import * from .groovy_magic import * -from .magic_kernel_comm import * +from .java_magic import * +from .kotlin_magic import * +from .scala_magic import * +from .sql_magic import * diff --git a/beakerx/beakerx_magics/clojure_magic.py b/beakerx/beakerx_magics/clojure_magic.py new file mode 100644 index 0000000000..64df628390 --- /dev/null +++ b/beakerx/beakerx_magics/clojure_magic.py @@ -0,0 +1,39 @@ +# Copyright 2017 TWO SIGMA OPEN SOURCE, LLC # +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from IPython import get_ipython +from IPython.core.magic import (magics_class, cell_magic) +from .kernel_magic import KernelMagics + + +@magics_class +class ClojureMagics(KernelMagics): + + def __init__(self, shell): + super(ClojureMagics, self).__init__(shell) + + def start(self): + super(ClojureMagics, self).start('clojure') + + @cell_magic + def clojure(self, line, cell): + return self.run_cell(line, cell) + + +def load_ipython_extension(ipython): + ipython.register_magics(ClojureMagics) + + +if __name__ == '__main__': + ip = get_ipython() + ip.register_magics(ClojureMagics) diff --git a/beakerx/beakerx_magics/groovy_magic.py b/beakerx/beakerx_magics/groovy_magic.py index 779e55a340..60f7c74616 100644 --- a/beakerx/beakerx_magics/groovy_magic.py +++ b/beakerx/beakerx_magics/groovy_magic.py @@ -11,76 +11,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -from queue import Empty from IPython import get_ipython -from IPython.core.magic import (Magics, magics_class, cell_magic) -from jupyter_client.manager import KernelManager -import atexit +from IPython.core.magic import (magics_class, cell_magic) +from .kernel_magic import KernelMagics @magics_class -class GroovyMagics(Magics): - _execution_count = 1 - - def stop_kernel(self): - self.kc.stop_channels() - self.km.shutdown_kernel(now=True) +class GroovyMagics(KernelMagics): def __init__(self, shell): super(GroovyMagics, self).__init__(shell) - self.km = None - self.kc = None - self.comms = [] def start(self): - self.km = KernelManager() - self.km.kernel_name = 'groovy' - self.km.start_kernel() - atexit.register(self.stop_kernel) - self.kc = self.km.client() - self.kc.start_channels() - try: - self.kc.wait_for_ready() - print("Groovy started successfully\n") - except AttributeError: - self._wait_for_ready_backport() - - def run_cell(self, line, code): - if not self.km: - self.start() - self.kc.execute(code, allow_stdin=True) - reply = self.kc.get_shell_msg() - self._handle_iopub_messages() - - def _handle_iopub_messages(self): - while True: - try: - msg = self.kc.get_iopub_msg(timeout=1) - except Empty: - break - comm_id = msg['content'].get('comm_id') - if comm_id and comm_id not in self.comms: - self.comms.append(comm_id) - self.shell.kernel.session.send(self.shell.kernel.iopub_socket, msg['msg_type'], - msg['content'], - metadata=msg['metadata'], - parent=self.shell.kernel._parent_header, - ident=msg.get('comm_id'), - buffers=msg['buffers'], - ) - - def pass_message(self, msg_raw): - comm_id = msg_raw['content'].get('comm_id') - if comm_id in self.comms: - content = msg_raw['content'] - msg = self.kc.session.msg(msg_raw['msg_type'], content) - self.kc.shell_channel.send(msg) - self._handle_iopub_messages() - else: - self.log.warn("No such comm: %s", comm_id) - if self.log.isEnabledFor(logging.DEBUG): - # don't create the list of keys if debug messages aren't enabled - self.log.debug("Current comms: %s", list(self.comms.keys())) + super(GroovyMagics, self).start('groovy') @cell_magic def groovy(self, line, cell): diff --git a/beakerx/beakerx_magics/java_magic.py b/beakerx/beakerx_magics/java_magic.py new file mode 100644 index 0000000000..86c45a4b5c --- /dev/null +++ b/beakerx/beakerx_magics/java_magic.py @@ -0,0 +1,39 @@ +# Copyright 2017 TWO SIGMA OPEN SOURCE, LLC # +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from IPython import get_ipython +from IPython.core.magic import (magics_class, cell_magic) +from .kernel_magic import KernelMagics + + +@magics_class +class JavaMagics(KernelMagics): + + def __init__(self, shell): + super(JavaMagics, self).__init__(shell) + + def start(self): + super(JavaMagics, self).start('java') + + @cell_magic + def java(self, line, cell): + return self.run_cell(line, cell) + + +def load_ipython_extension(ipython): + ipython.register_magics(JavaMagics) + + +if __name__ == '__main__': + ip = get_ipython() + ip.register_magics(JavaMagics) diff --git a/beakerx/beakerx_magics/kernel_magic.py b/beakerx/beakerx_magics/kernel_magic.py new file mode 100644 index 0000000000..8398ce395b --- /dev/null +++ b/beakerx/beakerx_magics/kernel_magic.py @@ -0,0 +1,110 @@ +# Copyright 2017 TWO SIGMA OPEN SOURCE, LLC # +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from queue import Empty +from IPython import get_ipython +from IPython.core.magic import (Magics, magics_class, cell_magic) +from jupyter_client.manager import KernelManager +import atexit +import logging + +@magics_class +class KernelMagics(Magics): + _execution_count = 1 + + def stop_kernel(self): + self.kc.stop_channels() + self.km.shutdown_kernel(now=True) + + def __init__(self, shell): + super(KernelMagics, self).__init__(shell) + self.km = None + self.kc = None + self.comms = [] + + def start(self, kernel_name): + self.km = KernelManager() + self.km.kernel_name = kernel_name + self.km.start_kernel() + atexit.register(self.stop_kernel) + self.kc = self.km.client() + self.kc.start_channels() + try: + self.kc.wait_for_ready() + print("{} started successfully\n".format(kernel_name.capitalize())) + except AttributeError: + self._wait_for_ready_backport() + + def run_cell(self, line, code): + if not self.km: + self.start() + self.kc.execute(code, allow_stdin=True) + reply = self.kc.get_shell_msg() + self._handle_iopub_messages() + + def _handle_iopub_messages(self): + while True: + try: + msg = self.kc.get_iopub_msg(timeout=1) + except Empty: + break + comm_id = msg['content'].get('comm_id') + if comm_id and comm_id not in self.comms: + self.comms.append(comm_id) + self.shell.kernel.session.send(self.shell.kernel.iopub_socket, msg['msg_type'], + msg['content'], + metadata=msg['metadata'], + parent=self.shell.kernel._parent_header, + ident=msg.get('comm_id'), + buffers=msg['buffers'], + ) + + def pass_message(self, msg_raw): + comm_id = msg_raw['content'].get('comm_id') + if comm_id in self.comms: + content = msg_raw['content'] + msg = self.kc.session.msg(msg_raw['msg_type'], content) + self.kc.shell_channel.send(msg) + self._handle_iopub_messages() + else: + self.log.warn("No such comm: %s", comm_id) + if self.log.isEnabledFor(logging.DEBUG): + # don't create the list of keys if debug messages aren't enabled + self.log.debug("Current comms: %s", list(self.comms.keys())) + +def comm_msg(stream, ident, msg): + content = msg['content'] + comm_id = content['comm_id'] + comm_manager = get_ipython().kernel.comm_manager + comm = comm_manager.comms.get('comm_id') + if comm is None: + magic_registry = comm_manager.kernel.shell.magics_manager.registry + for magic in magic_registry.values(): + if (hasattr(magic, 'pass_message') and comm_id in magic.comms): + try: + magic.pass_message(msg) + return + except Exception: + comm_manager.log.error('Exception in comm_msg for %s', comm_id, exc_info=True) + comm_manager.log.warn("No such comm: %s", comm_id) + if comm_manager.log.isEnabledFor(logging.DEBUG): + comm_manager.log.debug("Current comms: %s", list(comm_manager.comms.keys())) + else: + try: + comm.handle_msg(msg) + except Exception: + comm_manager.log.error('Exception in comm_msg for %s', comm_id, exc_info=True) + + +def load_ipython_extension(ipython): + ipython.kernel.shell_handlers['comm_msg'] = comm_msg diff --git a/beakerx/beakerx_magics/kotlin_magic.py b/beakerx/beakerx_magics/kotlin_magic.py new file mode 100644 index 0000000000..db7596cf87 --- /dev/null +++ b/beakerx/beakerx_magics/kotlin_magic.py @@ -0,0 +1,39 @@ +# Copyright 2017 TWO SIGMA OPEN SOURCE, LLC # +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from IPython import get_ipython +from IPython.core.magic import (magics_class, cell_magic) +from .kernel_magic import KernelMagics + + +@magics_class +class KotlinMagics(KernelMagics): + + def __init__(self, shell): + super(KotlinMagics, self).__init__(shell) + + def start(self): + super(KotlinMagics, self).start('kotlin') + + @cell_magic + def kotlin(self, line, cell): + return self.run_cell(line, cell) + + +def load_ipython_extension(ipython): + ipython.register_magics(KotlinMagics) + + +if __name__ == '__main__': + ip = get_ipython() + ip.register_magics(KotlinMagics) diff --git a/beakerx/beakerx_magics/magic_kernel_comm.py b/beakerx/beakerx_magics/magic_kernel_comm.py deleted file mode 100644 index 07f2e0c95d..0000000000 --- a/beakerx/beakerx_magics/magic_kernel_comm.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2017 TWO SIGMA OPEN SOURCE, LLC # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from IPython import get_ipython -import logging - -def comm_msg(stream, ident, msg): - content = msg['content'] - comm_id = content['comm_id'] - comm_manager = get_ipython().kernel.comm_manager - comm = comm_manager.comms.get('comm_id') - if comm is None: - magic_registry = comm_manager.kernel.shell.magics_manager.registry - for magic in magic_registry.values(): - if (hasattr(magic, 'pass_message') and comm_id in magic.comms): - try: - magic.pass_message(msg) - return - except Exception: - comm_manager.log.error('Exception in comm_msg for %s', comm_id, exc_info=True) - comm_manager.log.warn("No such comm: %s", comm_id) - if comm_manager.log.isEnabledFor(logging.DEBUG): - comm_manager.log.debug("Current comms: %s", list(comm_manager.comms.keys())) - else: - try: - comm.handle_msg(msg) - except Exception: - comm_manager.log.error('Exception in comm_msg for %s', comm_id, exc_info=True) - - -def load_ipython_extension(ipython): - ipython.kernel.shell_handlers['comm_msg'] = comm_msg diff --git a/beakerx/beakerx_magics/scala_magic.py b/beakerx/beakerx_magics/scala_magic.py new file mode 100644 index 0000000000..b7eecbe47d --- /dev/null +++ b/beakerx/beakerx_magics/scala_magic.py @@ -0,0 +1,39 @@ +# Copyright 2017 TWO SIGMA OPEN SOURCE, LLC # +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from IPython import get_ipython +from IPython.core.magic import (magics_class, cell_magic) +from .kernel_magic import KernelMagics + + +@magics_class +class ScalaMagics(KernelMagics): + + def __init__(self, shell): + super(ScalaMagics, self).__init__(shell) + + def start(self): + super(ScalaMagics, self).start('scala') + + @cell_magic + def scala(self, line, cell): + return self.run_cell(line, cell) + + +def load_ipython_extension(ipython): + ipython.register_magics(ScalaMagics) + + +if __name__ == '__main__': + ip = get_ipython() + ip.register_magics(ScalaMagics) diff --git a/beakerx/beakerx_magics/sql_magic.py b/beakerx/beakerx_magics/sql_magic.py new file mode 100644 index 0000000000..05ccf1f779 --- /dev/null +++ b/beakerx/beakerx_magics/sql_magic.py @@ -0,0 +1,39 @@ +# Copyright 2017 TWO SIGMA OPEN SOURCE, LLC # +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from IPython import get_ipython +from IPython.core.magic import (magics_class, cell_magic) +from .kernel_magic import KernelMagics + + +@magics_class +class SqlMagics(KernelMagics): + + def __init__(self, shell): + super(SqlMagics, self).__init__(shell) + + def start(self): + super(SqlMagics, self).start('SQL') + + @cell_magic + def sql(self, line, cell): + return self.run_cell(line, cell) + + +def load_ipython_extension(ipython): + ipython.register_magics(SqlMagics) + + +if __name__ == '__main__': + ip = get_ipython() + ip.register_magics(SqlMagics) diff --git a/doc/python/KernelsMagics.ipynb b/doc/python/KernelsMagics.ipynb new file mode 100644 index 0000000000..f6d4737b6d --- /dev/null +++ b/doc/python/KernelsMagics.ipynb @@ -0,0 +1,167 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%clojure\n", + "(doto (Plot.)\n", + " (.setTitle \"Title\")\n", + " (.setXLabel \"Horizontal\")\n", + " (.setYLabel \"Vertical\")\n", + " (.add (doto (Line.)\n", + " (.setX [0, 1, 2, 3, 4, 5])\n", + " (.setY [0, 1, 6, 5, 2, 8]))))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%groovy\n", + "def p = new Plot(title : 'Title', xLabel: 'Horizontal', yLabel: 'Vertical');\n", + "p << new Line(x: [0, 1, 2, 3, 4, 5], y: [0, 1, 6, 5, 2, 8])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%java\n", + "import java.util.List;\n", + "import com.twosigma.beakerx.chart.xychart.Plot;\n", + "import java.util.Arrays;\n", + "\n", + "Plot p = new Plot();\n", + "\n", + "p.setTitle(\"Title\");\n", + "p.setXLabel(\"Horizontal\");\n", + "p.setYLabel(\"Vertical\");\n", + "\n", + "Bars b = new Bars();\n", + "\n", + "List x = Arrays.asList(0, 1, 2, 3, 4, 5);\n", + "List y = Arrays.asList(0, 1, 6, 5, 2, 8);\n", + "Line line = new Line();\n", + "line.setX(x);\n", + "line.setY(y);\n", + "p.add(line);\n", + " \n", + "return p;" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%kotlin\n", + "val x: MutableList = mutableListOf(0, 1, 2, 3, 4, 5)\n", + "val y: MutableList = mutableListOf(0, 1, 6, 5, 2, 8)\n", + "val line = Line()\n", + "line.setX(x)\n", + "line.setY(y)\n", + "\n", + "val plot = Plot()\n", + "plot.setTitle(\"Title\")\n", + "plot.setXLabel(\"Horizontal\")\n", + "plot.setYLabel(\"Vertical\")\n", + "plot.add(line)\n", + "plot" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%scala\n", + "val plot = new Plot { title = \"Title\"; xLabel=\"Horizontal\"; yLabel=\"Vertical\" }\n", + "val line = new Line {x = Seq(0, 1, 2, 3, 4, 5); y = Seq(0, 1, 6, 5, 2, 8)}\n", + "plot.add(line)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "%defaultDatasource jdbc:h2:mem:db\n", + "DROP TABLE IF EXISTS cities;\n", + "CREATE TABLE cities(\n", + " zip_code varchar(5),\n", + " latitude float,\n", + " longitude float,\n", + " city varchar(100),\n", + " state varchar(2),\n", + " county varchar(100),\n", + " PRIMARY KEY (zip_code),\n", + ") AS SELECT\n", + " zip_code,\n", + " latitude,\n", + " longitude,\n", + " city,\n", + " state,\n", + " county\n", + "FROM CSVREAD('../resources/data/UScity.csv')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "SELECT * FROM cities WHERE state = 'NY'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%defaultDatasource jdbc:h2:mem:db" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 2d7e0b3a06018b4b5c5245a5d7d733f9a2c91796 Mon Sep 17 00:00:00 2001 From: Lukasz Mitusinski Date: Thu, 19 Apr 2018 17:07:32 +0200 Subject: [PATCH 2/2] #7205 fixed broken comm in python --- beakerx/beakerx_magics/kernel_magic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beakerx/beakerx_magics/kernel_magic.py b/beakerx/beakerx_magics/kernel_magic.py index 8398ce395b..da8df71749 100644 --- a/beakerx/beakerx_magics/kernel_magic.py +++ b/beakerx/beakerx_magics/kernel_magic.py @@ -86,7 +86,7 @@ def comm_msg(stream, ident, msg): content = msg['content'] comm_id = content['comm_id'] comm_manager = get_ipython().kernel.comm_manager - comm = comm_manager.comms.get('comm_id') + comm = comm_manager.comms.get(comm_id) if comm is None: magic_registry = comm_manager.kernel.shell.magics_manager.registry for magic in magic_registry.values():