diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..df0c9b1 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/.github/dependabot/" + schedule: + interval: "daily" diff --git a/.github/dependabot/constraints.txt b/.github/dependabot/constraints.txt new file mode 100644 index 0000000..9a9113b --- /dev/null +++ b/.github/dependabot/constraints.txt @@ -0,0 +1,20 @@ +backcall==0.2.0 +decorator==4.4.2 +ipykernel==5.3.4 +ipython==7.19.0 +ipython-genutils==0.2.0 +jedi==0.17.2 +jupyter-client==6.1.7 +jupyter-core==4.7.0 +parso==0.7.1 +pexpect==4.8.0 +pickleshare==0.7.5 +prompt-toolkit==3.0.8 +ptyprocess==0.6.0 +Pygments==2.7.2 +python-dateutil==2.8.1 +pyzmq==20.0.0 +six==1.15.0 +tornado==6.1 +traitlets==5.0.5 +wcwidth==0.2.5 diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml new file mode 100644 index 0000000..1433d21 --- /dev/null +++ b/.github/workflows/python-publish.yml @@ -0,0 +1,34 @@ +# This workflows will upload a Python Package using Twine when a release is created +# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries + +name: Upload Python Package + +on: + release: + types: [created] + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.8' + + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip + python3 -m pip install setuptools wheel twine + + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python3 -m setup.py sdist bdist_wheel + python3 -m twine upload dist/* diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..b80c3dd --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,48 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Tests + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + tests: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ 3.6, 3.7, 3.8, 3.9 ] + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('**/setup.py') }}-${{ hashFiles('.github/dependabot/constraints.txt') }} + + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip + + if [ ! ${{ matrix.python-version }} == "3.6" ]; then + # Our constraints can't be applied to python 3.6 + python3 -m pip install ".[test]" --constraint .github/dependabot/constraints.txt + else + python3 -m pip install ".[test]" + fi + + - name: Test with pytest + run: | + python3 -m pytest -s --verbose --cov=vip_ipykernel + + - uses: codecov/codecov-action@v1 diff --git a/.gitignore b/.gitignore index 1296d08..06e6339 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,11 @@ .venv .venv/ + +*.egg-info +**/__pycache__/** + +**/.ipynb_checkpoints + +.coverage + +.vscode diff --git a/README.md b/README.md new file mode 100644 index 0000000..f292695 --- /dev/null +++ b/README.md @@ -0,0 +1,132 @@ +[![Lifecycle](https://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/#active) +[![Release](https://img.shields.io/github/release/robertrosca/vip-ipykernel.svg)](https://github.com/robertrosca/vip-ipykernel/releases/latest) + +[![Tests](https://github.com/RobertRosca/vip-ipykernel/workflows/Tests/badge.svg)](https://github.com/RobertRosca/vip-ipykernel/actions?query=workflow%3ATests) +[![Codecov](https://codecov.io/gh/RobertRosca/vip-ipykernel/branch/main/graph/badge.svg)](https://codecov.io/gh/RobertRosca/vip-ipykernel) + +# ViP IPykernel + +Venv in Parent IPykernel + +- [ViP IPykernel](#vip-ipykernel) + - [Overview](#overview) + - [How it Works](#how-it-works) + - [Caveats and Gotchas](#caveats-and-gotchas) + - [VSCode Jupyter Notebook Integration](#vscode-jupyter-notebook-integration) + - [Venv Names](#venv-names) + - [Acknowledgements](#acknowledgements) + - [Todo](#todo) + +## Overview + +Do you use `venv`'s for all of your environments? Do you run Jupyter out of a +system/user installed location or via JupyterHub? Are you bored of making a +kernel for every single venv? Then this is the package for you! + +vip-ipykernel overwrites the default `python3` kernel and replaces it with one +which will traverse directories upwards until it finds a `.venv` directory, if +it finds one then it will start the kernel with python out of that directory, if +it does not find a venv then it will carry on with the default python3. + +NOTE: Your venv **must have ipykernel installed in it**, as this 'kernel' just +searches for and launches ipykernel out of the local venv. If ipykernel is not +available inside the venv then it will fail to start. + +This only needs to be installed once, you can do this with `pip install +vip-ipykernel --user` to install it into your local user environment. + +Once the package is installed, run `python3 -m vip_ipykernel.kernelspec --user` +to install the kernel, now when you run a notebook with the default `python3` +kernel it will instead use the venv in a parent directory. + +If you want to revert the changes, run `python3 -m ipykernel install --user`, +this will re-install the default `python3` kernel. + +Alternatively, if you don't want to overwrite the default kernel, then you can +pass a name (`python3 -m vip_ipykernel.kernelspec --user --name venv-kernel`) to +so that the kernel appears separately in the list of kernels and the default +behaviour is not modified. + +## How it Works + +The standard python3 kernel is: + +``` +{ + "argv": [ + "/usr/bin/python3", + "-m", + "ipykernel_launcher", + "-f", + "{connection_file}" + ], + "display_name": "Python 3", + "language": "python" +} +``` + +This just says "Run using `python3` to run `ipykernel_launcher` with an argument +`-f {connection_file}`". When you install the vip ipykernel this is replace by: + +``` +{ + "argv": [ + "/usr/bin/python3", + "-m", + "vip_ipykernel_launcher", + "-m", + "ipykernel_launcher", + "-f", + "{connection_file}" + ], + "display_name": "Python 3", + "language": "python" +} +``` + +Which will instead run the `vip_ipykernel.vip_ipykernel_launcher` module, +passing it the arguments `-m ipykernel_launcher -f {connection_file}`. The +module runs a function `venv_search` which looks in the current directory, and +upwards to any parent directories, until it finds a `.venv` or `venv` directory +containing `bin/python3`. + +If it finds a venv with python3 in it, it passes the arguments `-m +ipykernel_launcher -f {connection_file}` to that python executable, which starts +and connects the kernel from that venv to your current session, in the same way +that a kernel installed for that specific venv would. + +If it does not find a venv, then it will default to the system python executable +and behave like the standard `python3` kernel. + +## Caveats and Gotchas + +### VSCode Jupyter Notebook Integration + +VSCode manages kernels for its notebooks with its own system, so it will not use +the vip-ipykernel. + +### Venv Names + +Currently only venv's named `.venv` or `venv` are searched for, if your venv has +a different name it won't be found, and if you have multiple venv's available +then the first one (sorted alphanumerically, so `.venv` takes priority over +`venv`) will be used. + +## Acknowledgements + +The kernel implementation and tests are largely copy-and-paste'd directly from +the [ipykernel project](https://github.com/ipython/ipykernel) with some minor +modifications made to search for a venv and launch python out of it if possible. + +## Todo + +- [ ] Expand tests to different versions of ipykernel/jupyter_core +- [ ] Look at ways to show kernel errors +- [ ] Support for other environments: + - [ ] Poetry-created venvs (`poetry env info --path`) + - [ ] Pipenv-created venvs + - [ ] Pyenv-created venvs + - [ ] Conda-created environments + - [ ] User-configured venvs + - [ ] Reading from vscode configuration? + - [ ] etc... diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..6a76a9f --- /dev/null +++ b/setup.py @@ -0,0 +1,45 @@ +from setuptools import setup +import codecs +import os.path + + +def read(rel_path): + here = os.path.abspath(os.path.dirname(__file__)) + with codecs.open(os.path.join(here, rel_path), 'r') as fp: + return fp.read() + + +def get_version(rel_path): + for line in read(rel_path).splitlines(): + if line.startswith('__version__'): + delim = '"' if '"' in line else "'" + return line.split(delim)[1] + else: + raise RuntimeError("Unable to find version string.") + + +setup( + name='vip-ipykernel', + url='https://github.com/RobertRosca/vip-ipykernel', + long_description=read("README.md"), + long_description_content_type='text/markdown', + version=get_version("./src/vip_ipykernel/__init__.py"), + python_requires='==3.*,>=3.6.0', + author='Robert Rosca', + author_email='32569096+RobertRosca@users.noreply.github.com', + py_modules=['vip_ipykernel_launcher'], + packages=['vip_ipykernel'], + package_dir={"": "src"}, + install_requires=[ + 'jupyter-client>=4.2', + 'ipykernel>=4.4', + ], + extras_require={ + 'test': [ + 'pytest==5.*,>=5.2.0', + 'pytest-cov==2.*,>=2.10.1', + 'nose==1.*,>=1.3.7', + 'nbval==0.9.*,>=0.9.6', + ], + }, +) diff --git a/src/vip_ipykernel/__init__.py b/src/vip_ipykernel/__init__.py new file mode 100644 index 0000000..0dc4a4e --- /dev/null +++ b/src/vip_ipykernel/__init__.py @@ -0,0 +1,30 @@ +__version__ = '1.0.0' + +import sys +from pathlib import Path + +VENV_NAMES = [ + '.venv', + 'venv', +] + +ANCHOR = Path(Path.cwd().anchor) + + +def venv_search(prefix: Path = Path('.')) -> Path: + prefix = prefix.absolute() + + found_venvs = [] + + for venv in VENV_NAMES: + path = prefix / venv / 'bin' / 'python3' + if path.is_file(): + found_venvs.append(path.absolute()) + + if any(found_venvs): + # If there are multiple venvs just return the first one + return found_venvs[0].absolute() + elif prefix == ANCHOR: + return Path(sys.executable).resolve() + else: + return venv_search(prefix=prefix.parent) diff --git a/src/vip_ipykernel/kernelspec.py b/src/vip_ipykernel/kernelspec.py new file mode 100644 index 0000000..ad01c6d --- /dev/null +++ b/src/vip_ipykernel/kernelspec.py @@ -0,0 +1,62 @@ +import sys + +import ipykernel.kernelspec +from ipykernel.kernelspec import ( + KERNEL_NAME, + RESOURCES, + InstallIPythonKernelSpecApp, + get_kernel_dict, + install, + make_ipkernel_cmd, +) + + +def make_vip_ipkernel_cmd( + mod='ipykernel_launcher', executable=None, extra_arguments=None, **kw +): + """Build Popen command list for launching an ViP-IPython kernel. + + Parameters + ---------- + mod : str, optional (default 'ipykernel_launcher') + A string of an IPython module whose __main__ starts an IPython kernel + + executable : str, optional (default sys.executable) + The Python executable to use for the kernel process. + + extra_arguments : list, optional + A list of extra arguments to pass when executing the launch code. + + Returns + ------- + + A Popen command list + """ + + # Copyright (c) IPython Development Team. + # Distributed under the terms of the Modified BSD License. + + if executable is None: + executable = sys.executable + extra_arguments = extra_arguments or [] + # When installing the ViP IPykernel, the first `-m` module call points to + # our `vip_ipykernel_launcher`, and the second module call points to the + # desired ipykernel launcher module + arguments = [ + executable, + '-m', + 'vip_ipykernel_launcher', + '-m', + mod, + '-f', + '{connection_file}', + ] + arguments.extend(extra_arguments) + + return arguments + + +ipykernel.kernelspec.make_ipkernel_cmd = make_vip_ipkernel_cmd + +if __name__ == '__main__': + InstallIPythonKernelSpecApp.launch_instance() diff --git a/src/vip_ipykernel_launcher.py b/src/vip_ipykernel_launcher.py new file mode 100644 index 0000000..216bf8c --- /dev/null +++ b/src/vip_ipykernel_launcher.py @@ -0,0 +1,26 @@ +"""Taken and modified from +https://github.com/ipython/ipykernel/blob/master/ipykernel_launcher.py + +Entry point for launching an ViP-IPython kernel. +""" + +import os +import sys + +if __name__ == '__main__': + # Remove the CWD from sys.path while we load stuff. + # This is added back by InteractiveShellApp.init_path() + if sys.path[0] == '': + del sys.path[0] + + from vip_ipykernel import venv_search + + args = sys.argv.copy() + + args[0] = str(venv_search()) + + # TODO: I want to use the jupyter logger to print this off but can't figure + # out how to do that, @takluyver can you help with this? + print(f"Starting venv kernel with args: {args}") + + os.execv(args[0], args) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/notebook-std.ipynb b/tests/notebook-std.ipynb new file mode 100644 index 0000000..bb4efb8 --- /dev/null +++ b/tests/notebook-std.ipynb @@ -0,0 +1,54 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import sys" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "not \".venv\" in sys.executable" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.6" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/tests/notebook-vip.ipynb b/tests/notebook-vip.ipynb new file mode 100644 index 0000000..a39618a --- /dev/null +++ b/tests/notebook-vip.ipynb @@ -0,0 +1,54 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import sys" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\".venv\" in sys.executable" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.6" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/tests/test_kernelspec_install.py b/tests/test_kernelspec_install.py new file mode 100644 index 0000000..e35f5b4 --- /dev/null +++ b/tests/test_kernelspec_install.py @@ -0,0 +1,93 @@ +"""Taken and modified from + +https://github.com/ipython/ipykernel/blob/master/ipykernel/tests/test_kernelspec.py +""" + +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. + +import io +import json +import os +from unittest import mock + +import nose.tools as nt +from jupyter_core.paths import jupyter_data_dir + +from vip_ipykernel.kernelspec import ( + KERNEL_NAME, + RESOURCES, + InstallIPythonKernelSpecApp, + install, +) + + +def assert_is_spec(path): + for fname in os.listdir(RESOURCES): + dst = os.path.join(path, fname) + assert os.path.exists(dst) + kernel_json = os.path.join(path, "kernel.json") + assert os.path.exists(kernel_json) + with io.open(kernel_json, encoding="utf8") as f: + json.load(f) + + +def test_install_kernelspec(tmp_path): + InstallIPythonKernelSpecApp.launch_instance( + argv=["--prefix", str(tmp_path)] # `launch_instance` does not like `Path` + ) + + assert_is_spec(os.path.join(tmp_path, "share", "jupyter", "kernels", KERNEL_NAME)) + + +def test_install_user(tmp_path): + with mock.patch.dict(os.environ, {"HOME": str(tmp_path)}): + install(user=True) + data_dir = jupyter_data_dir() + + assert_is_spec(os.path.join(data_dir, "kernels", KERNEL_NAME)) + + +def test_install(tmp_path): + with mock.patch("jupyter_client.kernelspec.SYSTEM_JUPYTER_PATH", [tmp_path]): + install() + + assert_is_spec(os.path.join(tmp_path, "kernels", KERNEL_NAME)) + + +def test_install_profile(tmp_path): + with mock.patch("jupyter_client.kernelspec.SYSTEM_JUPYTER_PATH", [tmp_path]): + install(profile="Test") + + spec = os.path.join(tmp_path, "kernels", KERNEL_NAME, "kernel.json") + + with open(spec) as f: + spec = json.load(f) + + assert spec["display_name"].endswith(" [profile=Test]") + + nt.assert_equal(spec["argv"][-2:], ["--profile", "Test"]) + + +def test_install_display_name_overrides_profile(tmp_path): + with mock.patch("jupyter_client.kernelspec.SYSTEM_JUPYTER_PATH", [tmp_path]): + install(display_name="Display", profile="Test") + + spec = os.path.join(tmp_path, "kernels", KERNEL_NAME, "kernel.json") + + with open(spec) as f: + spec = json.load(f) + + assert spec["display_name"] == "Display" + + +def test_install_uses_vip_ipykernel(tmp_path): + with mock.patch("jupyter_client.kernelspec.SYSTEM_JUPYTER_PATH", [tmp_path]): + install() + + spec = os.path.join(tmp_path, "kernels", KERNEL_NAME, "kernel.json") + + with open(spec) as f: + spec = json.load(f) + + assert "vip_ipykernel" in spec["argv"][2] diff --git a/tests/test_nb.py b/tests/test_nb.py new file mode 100644 index 0000000..52ee42f --- /dev/null +++ b/tests/test_nb.py @@ -0,0 +1,97 @@ +import json +import os +import pathlib +import shutil +import subprocess +import sys +import venv +from unittest import mock + +import jupyter_client.kernelspec +import pytest + +PROJECT_ROOT = pathlib.Path(__file__).parent.absolute().parent +NB_FILE_STD = pathlib.Path(__file__).parent.absolute() / 'notebook-std.ipynb' +NB_FILE_VIP = pathlib.Path(__file__).parent.absolute() / 'notebook-vip.ipynb' + + +def test_nb_vip_no_venv(tmp_path, monkeypatch): + import vip_ipykernel.kernelspec + from vip_ipykernel.kernelspec import install as install_vip + + monkeypatch.chdir(tmp_path) + + shutil.copy(NB_FILE_STD, ".") + + # Install the custom kernel + with mock.patch.dict(os.environ, {"HOME": str(tmp_path)}): + defaults = list(vip_ipykernel.kernelspec.make_vip_ipkernel_cmd.__defaults__) + defaults[1] = str(sys.executable) # Set the python executable path to python used for pytest + defaults = tuple(defaults) + with mock.patch.object(vip_ipykernel.kernelspec.make_vip_ipkernel_cmd, "__defaults__", defaults): + dest = install_vip(user=True) + + with open(str(dest) + "/kernel.json") as f: + spec = json.load(f) + + # The kernel should be installed in tmp_path + assert str(tmp_path) in jupyter_client.kernelspec.find_kernel_specs()['python3'] + + # The kernel should be running with the same python that is running the tests + assert sys.executable in spec["argv"][0] + + # Should be using the vip_ipykernel module + assert "vip_ipykernel" in spec["argv"][2] + + # Test the ViP kernel falling back to base python + assert pytest.main([ + "--verbose", + "--nbval", + "notebook-std.ipynb", + ]) == 0 + + +def test_nb_vip_venv(tmp_path, monkeypatch): + import vip_ipykernel.kernelspec + from vip_ipykernel.kernelspec import install as install_vip + + monkeypatch.chdir(tmp_path) + + shutil.copy(NB_FILE_VIP, ".") + + # Create venv in the temporary directory + venv.create(".venv", with_pip=True) + + # Install ipykernel in it + subprocess.run([ + ".venv/bin/python3", + "-m", + "pip", + "install", + "ipykernel", + "jupyter_client", + ]) + + # Install the custom kernel + with mock.patch.dict(os.environ, {"HOME": str(tmp_path)}): + defaults = list(vip_ipykernel.kernelspec.make_vip_ipkernel_cmd.__defaults__) + defaults[1] = str(sys.executable) # Set the python executable path to python used for pytest + defaults = tuple(defaults) + with mock.patch.object(vip_ipykernel.kernelspec.make_vip_ipkernel_cmd, "__defaults__", defaults): + dest = install_vip(user=True) + + with open(str(dest) + "/kernel.json") as f: + spec = json.load(f) + + # The kernel should be installed in tmp_path + assert str(tmp_path) in jupyter_client.kernelspec.find_kernel_specs()['python3'] + + # The kernel should be running with the same python that is running the tests + assert sys.executable in spec["argv"][0] + + # Test the ViP kernel notbook + assert pytest.main([ + "--verbose", + "--nbval", + "notebook-vip.ipynb", + ]) == 0 diff --git a/tests/test_venv_search.py b/tests/test_venv_search.py new file mode 100644 index 0000000..3f5d193 --- /dev/null +++ b/tests/test_venv_search.py @@ -0,0 +1,25 @@ +import os +import sys +from pathlib import Path + +from vip_ipykernel import venv_search + + +def test_venv_search(tmp_path): + os.chdir(tmp_path) + + venv_bin = tmp_path / '.venv' / 'bin' + venv_bin.mkdir(parents=True) + + venv_executable = venv_bin / 'python3' + venv_executable.touch() + + # Use 'in' instead of '==' as venv_search may return python3.x instead of 3 + assert str(venv_executable) in str(venv_search()) + + +def test_venv_search_missing(tmp_path): + os.chdir(tmp_path) + + # Use 'in' instead of '==' as venv_search may return python3.x instead of 3 + assert str(Path(sys.executable).resolve()) in str(venv_search())