From 43c9e9fb0c147cfb14a292b99c6b0aa6f5535619 Mon Sep 17 00:00:00 2001 From: Zsailer Date: Wed, 10 Jun 2020 15:10:39 -0700 Subject: [PATCH 01/30] add utils module in extension --- jupyter_server/extension/utils.py | 111 ++++++++++++++++++++++++++++++ tests/extension/test_utils.py | 55 +++++++++++++++ 2 files changed, 166 insertions(+) create mode 100644 jupyter_server/extension/utils.py create mode 100644 tests/extension/test_utils.py diff --git a/jupyter_server/extension/utils.py b/jupyter_server/extension/utils.py new file mode 100644 index 0000000000..a47c63e09d --- /dev/null +++ b/jupyter_server/extension/utils.py @@ -0,0 +1,111 @@ +import pathlib +import pkgutil + +from jupyter_core.paths import jupyter_config_path +from traitlets.utils.importstring import import_item +from traitlets.config.loader import ( + JSONFileConfigLoader, + PyFileConfigLoader, + Config +) + + +class JupyterServerExtensionPathsMissing(Exception): + """""" + + +def list_extensions_from_configd( + configd_prefix="jupyter_server", + config_paths=None +): + """Get a dictionary of all jpserver_extensions found in the + config directories list. + + Parameters + ---------- + config_paths : list + List of config directories to search for the + `jupyter_server_config.d` directory. + """ + # Build directory name for `config.d` directory. + configd = "_".join([configd_prefix, "config.d"]) + + if not config_paths: + config_paths = jupyter_config_path() + + # Leverage pathlib for path management. + config_paths = [pathlib.Path(p) for p in config_paths] + + extensions = {} + for path in config_paths: + py_files = path.joinpath(configd).glob("*.py") + json_files = path.joinpath(configd).glob("*.json") + + for f in py_files: + pyc = PyFileConfigLoader( + filename=str(f.name), + path=str(f.parent) + ) + c = pyc.load_config() + items = c.get("ServerApp", {}).get("jpserver_extensions", {}) + extensions.update(items) + + for f in json_files: + jsc = JSONFileConfigLoader( + filename=str(f.name), + path=str(f.parent) + ) + c = jsc.load_config() + items = c.get("ServerApp", {}).get("jpserver_extensions", {}) + extensions.update(items) + + return extensions + + +def _list_extensions_from_entrypoints(): + pass + + + + + + +def _get_server_extension_metadata(module): + """Load server extension metadata from a module. + + Returns a tuple of ( + the package as loaded + a list of server extension specs: [ + { + "module": "import.path.to.extension" + } + ] + ) + + Parameters + ---------- + module : str + Importable Python module exposing the + magic-named `_jupyter_server_extension_paths` function + """ + m = import_item(module) + if not hasattr(m, '_jupyter_server_extension_paths'): + raise JupyterServerExtensionPathsMissing( + 'The Python module {} does not include ' + 'any valid server extensions'.format(module) + ) + return m, m._jupyter_server_extension_paths() + + + +def _get_load_jupyter_server_extension(obj): + """Looks for load_jupyter_server_extension as an attribute + of the object or module. + """ + try: + func = getattr(obj, '_load_jupyter_server_extension') + except AttributeError: + func = getattr(obj, 'load_jupyter_server_extension') + except: + raise ExtensionLoadingError("_load_jupyter_server_extension function was not found.") + return func \ No newline at end of file diff --git a/tests/extension/test_utils.py b/tests/extension/test_utils.py new file mode 100644 index 0000000000..34e3e5f455 --- /dev/null +++ b/tests/extension/test_utils.py @@ -0,0 +1,55 @@ +import pytest +from jupyter_server.extension.utils import list_extensions_from_configd + + +@pytest.fixture +def config_path(tmp_path): + root = tmp_path.joinpath('config') + root.mkdir() + return root + +@pytest.fixture +def configd(config_path): + configd = config_path.joinpath('jupyter_server_config.d') + configd.mkdir() + return configd + + +ext1_json_config = """\ +{ + "ServerApp": { + "jpserver_extensions": { + "ext1_config": true + } + } +} +""" + + +@pytest.fixture +def ext1_config(configd): + config = configd.joinpath("ext1_config.json") + config.write_text(ext1_json_config) + + +ext2_py_config = """\ +c.ServerApp.jpserver_extensions = { + "ext2_config": True +} +""" + + +@pytest.fixture +def ext2_config(configd): + config = configd.joinpath("ext2_config.py") + config.write_text(ext2_py_config) + + +def test_list_extension_from_configd(config_path, ext1_config, ext2_config): + extensions = list_extensions_from_configd( + config_paths=[config_path] + ) + assert "ext2_config" in extensions + assert "ext1_config" in extensions + assert extensions["ext2_config"] + assert extensions["ext1_config"] From d8ad89186fcf2cb2ee71fc301566246d729b9f67 Mon Sep 17 00:00:00 2001 From: Zsailer Date: Thu, 11 Jun 2020 12:55:22 -0700 Subject: [PATCH 02/30] wip --- jupyter_server/extension/utils.py | 53 ++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/jupyter_server/extension/utils.py b/jupyter_server/extension/utils.py index a47c63e09d..88316a9d15 100644 --- a/jupyter_server/extension/utils.py +++ b/jupyter_server/extension/utils.py @@ -14,6 +14,30 @@ class JupyterServerExtensionPathsMissing(Exception): """""" + + +def _x(): + + for f in py_files: + pyc = PyFileConfigLoader( + filename=str(f.name), + path=str(f.parent) + ) + c = pyc.load_config() + items = c.get("ServerApp", {}).get("jpserver_extensions", {}) + extensions.update(items) + + for f in json_files: + jsc = JSONFileConfigLoader( + filename=str(f.name), + path=str(f.parent) + ) + c = jsc.load_config() + items = c.get("ServerApp", {}).get("jpserver_extensions", {}) + extensions.update(items) + + + def list_extensions_from_configd( configd_prefix="jupyter_server", config_paths=None @@ -40,32 +64,25 @@ def list_extensions_from_configd( for path in config_paths: py_files = path.joinpath(configd).glob("*.py") json_files = path.joinpath(configd).glob("*.json") + if + - for f in py_files: - pyc = PyFileConfigLoader( - filename=str(f.name), - path=str(f.parent) - ) - c = pyc.load_config() - items = c.get("ServerApp", {}).get("jpserver_extensions", {}) - extensions.update(items) - - for f in json_files: - jsc = JSONFileConfigLoader( - filename=str(f.name), - path=str(f.parent) - ) - c = jsc.load_config() - items = c.get("ServerApp", {}).get("jpserver_extensions", {}) - extensions.update(items) return extensions -def _list_extensions_from_entrypoints(): +def list_extensions_from_entrypoints(): pass +def validate_extension(): + + + + + + + From 5eb75f9a38732fbfc7de50edfc7ee8de23519c83 Mon Sep 17 00:00:00 2001 From: Zsailer Date: Thu, 11 Jun 2020 14:00:29 -0700 Subject: [PATCH 03/30] wip --- jupyter_server/extension/utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/jupyter_server/extension/utils.py b/jupyter_server/extension/utils.py index 88316a9d15..dd3e3f8be3 100644 --- a/jupyter_server/extension/utils.py +++ b/jupyter_server/extension/utils.py @@ -38,7 +38,7 @@ def _x(): -def list_extensions_from_configd( +def list_extensions_in_configd( configd_prefix="jupyter_server", config_paths=None ): @@ -60,13 +60,13 @@ def list_extensions_from_configd( # Leverage pathlib for path management. config_paths = [pathlib.Path(p) for p in config_paths] - extensions = {} + extensions = [] for path in config_paths: - py_files = path.joinpath(configd).glob("*.py") json_files = path.joinpath(configd).glob("*.json") - if - - + for file in json_files: + # The extension name is the file name (minus file suffix) + extension_name = file.stem + extensions.append(extension_name) return extensions From 28ac3616a4c9da026cf56058de6f899dc4d7d7b7 Mon Sep 17 00:00:00 2001 From: Zsailer Date: Fri, 12 Jun 2020 11:49:47 -0700 Subject: [PATCH 04/30] add extension manager --- jupyter_server/extension/application.py | 25 ++- jupyter_server/extension/utils.py | 202 +++++++++++++++++------- jupyter_server/serverapp.py | 2 +- tests/extension/test_utils.py | 104 +++++++++--- 4 files changed, 246 insertions(+), 87 deletions(-) diff --git a/jupyter_server/extension/application.py b/jupyter_server/extension/application.py index 33ef2e7424..6b86123d4f 100644 --- a/jupyter_server/extension/application.py +++ b/jupyter_server/extension/application.py @@ -144,6 +144,9 @@ class method. This method can be set as a entry_point in # side-by-side when launched directly. load_other_extensions = True + # Pass se + server_config = {} + # The extension name used to name the jupyter config # file, jupyter_{name}_config. # This should also match the jupyter subcommand used to launch @@ -291,18 +294,12 @@ def initialize_server(cls, argv=[], load_other_extensions=True, **kwargs): # The ExtensionApp needs to add itself as enabled extension # to the jpserver_extensions trait, so that the ServerApp # initializes it. - config = Config({ - "ServerApp": { - "jpserver_extensions": {cls.name: True}, - "open_browser": cls.open_browser, - "default_url": cls.extension_url - } - }) + config = Config(cls._jupyter_server_config()) serverapp = ServerApp.instance(**kwargs, argv=[], config=config) serverapp.initialize(argv=argv, find_extensions=load_other_extensions) return serverapp - def link_to_serverapp(self, serverapp): + def _link_jupyter_server_extension(self, serverapp): """Link the ExtensionApp to an initialized ServerApp. The ServerApp is stored as an attribute and config @@ -369,6 +366,18 @@ def stop(self): self.serverapp.stop() self.serverapp.clear_instance() + @classmethod + def _jupyter_server_config(cls): + base_config = { + "ServerApp": { + "jpserver_extensions": {cls.name: True}, + "open_browser": True, + "default_url": cls.extension_url + } + } + base_config["ServerApp"].update(cls.server_config) + return base_config + @classmethod def _load_jupyter_server_extension(cls, serverapp): """Initialize and configure this extension, then add the extension's diff --git a/jupyter_server/extension/utils.py b/jupyter_server/extension/utils.py index dd3e3f8be3..af1d156830 100644 --- a/jupyter_server/extension/utils.py +++ b/jupyter_server/extension/utils.py @@ -1,8 +1,7 @@ import pathlib -import pkgutil +import importlib from jupyter_core.paths import jupyter_config_path -from traitlets.utils.importstring import import_item from traitlets.config.loader import ( JSONFileConfigLoader, PyFileConfigLoader, @@ -10,33 +9,89 @@ ) -class JupyterServerExtensionPathsMissing(Exception): - """""" +def configd_path( + config_dir=None, + configd_prefix="jupyter_server", +): + # Build directory name for `config.d` directory. + configd = "_".join([configd_prefix, "config.d"]) + if not config_dir: + config_dir = jupyter_config_path() + # Leverage pathlib for path management. + return [pathlib.Path(path).joinpath(configd) for path in config_dir] +def configd_files( + config_dir=None, + configd_prefix="jupyter_server", +): + """Lists (only) JSON files found in a Jupyter config.d folder. + """ + paths = configd_path( + config_dir=config_dir, + configd_prefix=configd_prefix + ) + files = [] + for path in paths: + json_files = path.glob("*.json") + files.extend(json_files) + return files -def _x(): +def enabled(name, server_config): + """Given a server config object, return True if the extension + is explicitly enabled in the config. + """ + enabled = ( + server_config + .get("ServerApp", {}) + .get("jpserver_extensions", {}) + .get(name, False) + ) + return enabled - for f in py_files: - pyc = PyFileConfigLoader( - filename=str(f.name), - path=str(f.parent) - ) - c = pyc.load_config() - items = c.get("ServerApp", {}).get("jpserver_extensions", {}) - extensions.update(items) - - for f in json_files: - jsc = JSONFileConfigLoader( - filename=str(f.name), - path=str(f.parent) - ) - c = jsc.load_config() - items = c.get("ServerApp", {}).get("jpserver_extensions", {}) - extensions.update(items) + +def find_extension_in_configd( + name, + config_dir=None, + configd_prefix="jupyter_server", +): + """Search through all config.d files and return the + JSON Path for this named extension. If the extension + is not found, return None + """ + files = configd_files( + config_dir=config_dir, + configd_prefix=configd_prefix, + ) + for f in files: + if name == f.stem: + return f +def configd_enabled( + name, + config_dir=None, + configd_prefix="jupyter_server", +): + """Check if the named extension is enabled somewhere in + a config.d folder. + """ + config_file = find_extension_in_configd( + name, + config_dir=config_dir, + configd_prefix=configd_prefix, + ) + if config_file: + c = JSONFileConfigLoader( + filename=str(config_file.name), + path=str(config_file.parent) + ) + config = c.load_config() + return enabled(name, config) + else: + return False + def list_extensions_in_configd( configd_prefix="jupyter_server", @@ -51,6 +106,7 @@ def list_extensions_in_configd( List of config directories to search for the `jupyter_server_config.d` directory. """ + # Build directory name for `config.d` directory. configd = "_".join([configd_prefix, "config.d"]) @@ -71,58 +127,88 @@ def list_extensions_in_configd( return extensions -def list_extensions_from_entrypoints(): +class ExtensionLoadingError(Exception): pass -def validate_extension(): - +def get_loader(obj): + """Looks for _load_jupyter_server_extension as an attribute + of the object or module. + """ + try: + func = getattr(obj, '_load_jupyter_server_extension') + except AttributeError: + func = getattr(obj, 'load_jupyter_server_extension') + except Exception: + raise ExtensionLoadingError("_load_jupyter_server_extension function was not found.") + return func +class ExtensionPath: + def __init__(self, metadata): + self.module_name = metadata.get("module") + self.module = importlib.import_module(self.module_name) + self.app = metadata.get("app", None) + if self.app: + self.name = self.app.name + self.load = staticmethod(get_loader(self.app)) + self.link = self.app._link_jupyter_server_extension + else: + self.name = metadata.get("name", self.module_name) + self.load = staticmethod(get_loader(self.module)) + self.link = lambda: None +def get_metadata(package_name): + module = importlib.import_module(package_name) + return module._jupyter_server_extension_paths() +class Extension: + def __init__(self, package_name): + self.package_name = package_name + self.metadata = get_metadata(self.package_name) + self.paths = {} + for path_metadata in self.metadata: + path = ExtensionPath(path_metadata) + self.paths[path.name] = path -def _get_server_extension_metadata(module): - """Load server extension metadata from a module. +class ExtensionManager: - Returns a tuple of ( - the package as loaded - a list of server extension specs: [ - { - "module": "import.path.to.extension" - } - ] - ) + def __init__(self, jpserver_extensions): + self.extensions = {} + for package_name, enabled in jpserver_extensions.items(): + if enabled: + self.extensions[package_name] = Extension(package_name) - Parameters - ---------- - module : str - Importable Python module exposing the - magic-named `_jupyter_server_extension_paths` function - """ - m = import_item(module) - if not hasattr(m, '_jupyter_server_extension_paths'): - raise JupyterServerExtensionPathsMissing( - 'The Python module {} does not include ' - 'any valid server extensions'.format(module) - ) - return m, m._jupyter_server_extension_paths() + @property + def paths(self): + _paths = {} + for ext in self.extensions.values(): + _paths.update(ext.paths) + return _paths +def validate_extension(name): + """Raises an exception is the extension is missing a needed + hook or metadata field. + An extension is valid if: + 1) name is an importable Python package. + 1) the package has a _jupyter_server_extension_paths function + 2) each extension path has a _load_jupyter_server_extension function -def _get_load_jupyter_server_extension(obj): - """Looks for load_jupyter_server_extension as an attribute - of the object or module. + If this works, nothing should happen. """ - try: - func = getattr(obj, '_load_jupyter_server_extension') - except AttributeError: - func = getattr(obj, 'load_jupyter_server_extension') - except: - raise ExtensionLoadingError("_load_jupyter_server_extension function was not found.") - return func \ No newline at end of file + # 1) Try importing + mod = importlib.import_module(name) + # 2) Try calling extension paths function. + paths = mod._jupyter_server_extension_paths() + for path in paths: + submod_path = path.get("module") + submod = importlib.import_module(submod_path) + # Check that extension has loading function. + get_loader(submod) + diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index 91c012f184..ec9cde9c3e 100755 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -1565,7 +1565,7 @@ def init_server_extensions(self): # ExtensionApp here to load any ServerApp configuration # that might live in the Extension's config file. app = extapp() - app.link_to_serverapp(self) + app._link_jupyter_server_extension(self) # Build a new list where we self._enabled_extensions[app.name] = app elif extloc: diff --git a/tests/extension/test_utils.py b/tests/extension/test_utils.py index 34e3e5f455..51e6c31b3b 100644 --- a/tests/extension/test_utils.py +++ b/tests/extension/test_utils.py @@ -1,16 +1,22 @@ import pytest -from jupyter_server.extension.utils import list_extensions_from_configd +from jupyter_server.extension.utils import ( + list_extensions_in_configd, + configd_enabled, + ExtensionPath, + Extension, + ExtensionManager +) +# Use ServerApps environment because it monkeypatches +# jupyter_core.paths and provides a config directory +# that's not cross contaminating the user config directory. +pytestmark = pytest.mark.usefixtures("environ") -@pytest.fixture -def config_path(tmp_path): - root = tmp_path.joinpath('config') - root.mkdir() - return root @pytest.fixture -def configd(config_path): - configd = config_path.joinpath('jupyter_server_config.d') +def configd(env_config_path): + """A pathlib.Path object that acts like a jupyter_server_config.d folder.""" + configd = env_config_path.joinpath('jupyter_server_config.d') configd.mkdir() return configd @@ -25,31 +31,89 @@ def configd(config_path): } """ - @pytest.fixture def ext1_config(configd): config = configd.joinpath("ext1_config.json") config.write_text(ext1_json_config) -ext2_py_config = """\ -c.ServerApp.jpserver_extensions = { - "ext2_config": True +ext2_json_config = """\ +{ + "ServerApp": { + "jpserver_extensions": { + "ext2_config": false + } + } } """ @pytest.fixture def ext2_config(configd): - config = configd.joinpath("ext2_config.py") - config.write_text(ext2_py_config) + config = configd.joinpath("ext2_config.json") + config.write_text(ext2_json_config) -def test_list_extension_from_configd(config_path, ext1_config, ext2_config): - extensions = list_extensions_from_configd( - config_paths=[config_path] - ) +def test_list_extension_from_configd(ext1_config, ext2_config): + extensions = list_extensions_in_configd() assert "ext2_config" in extensions assert "ext1_config" in extensions - assert extensions["ext2_config"] - assert extensions["ext1_config"] + + +def test_config_enabled(ext1_config): + assert configd_enabled("ext1_config") + + +def test_extension_path_api(): + # Import mock extension metadata + from .mockextensions import _jupyter_server_extension_paths + + # Testing the first path (which is an extension app). + metadata_list = _jupyter_server_extension_paths() + path = metadata_list[0] + + module = path["module"] + app = path["app"] + + e = ExtensionPath(path) + assert e.module_name == module + assert e.name == app.name + assert app is not None + assert callable(e.load) + assert callable(e.link) + + +def test_extension_api(): + # Import mock extension metadata + from .mockextensions import _jupyter_server_extension_paths + + # Testing the first path (which is an extension app). + metadata_list = _jupyter_server_extension_paths() + path1 = metadata_list[0] + module = path1["module"] + app = path1["app"] + + e = Extension('tests.extension.mockextensions') + assert hasattr(e, "paths") + assert len(e.paths) == len(metadata_list) + assert app.name in e.paths + + +def test_extension_manager_api(): + # Import mock extension metadata + from .mockextensions import _jupyter_server_extension_paths + + # Testing the first path (which is an extension app). + metadata_list = _jupyter_server_extension_paths() + + jpserver_extensions = { + "tests.extension.mockextensions": True + } + manager = ExtensionManager(jpserver_extensions) + assert len(manager.extensions) == 1 + assert len(manager.paths) == len(metadata_list) + assert "mockextension" in manager.paths + assert "tests.extension.mockextensions.mock1" in manager.paths + + print(manager.paths) + assert False \ No newline at end of file From 34f99692edd0961710b19b8e35746a859e23c7ad Mon Sep 17 00:00:00 2001 From: Zsailer Date: Fri, 12 Jun 2020 11:51:30 -0700 Subject: [PATCH 05/30] remove debugging lines --- tests/extension/test_utils.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/extension/test_utils.py b/tests/extension/test_utils.py index 51e6c31b3b..262dc4dcf9 100644 --- a/tests/extension/test_utils.py +++ b/tests/extension/test_utils.py @@ -113,7 +113,4 @@ def test_extension_manager_api(): assert len(manager.extensions) == 1 assert len(manager.paths) == len(metadata_list) assert "mockextension" in manager.paths - assert "tests.extension.mockextensions.mock1" in manager.paths - - print(manager.paths) - assert False \ No newline at end of file + assert "tests.extension.mockextensions.mock1" in manager.paths \ No newline at end of file From fb90ad440034e8bcb0ddb18fb2a99e8c234ef3fe Mon Sep 17 00:00:00 2001 From: Zsailer Date: Fri, 12 Jun 2020 12:37:29 -0700 Subject: [PATCH 06/30] enable linking as a way to add server config --- jupyter_server/extension/application.py | 2 +- jupyter_server/extension/utils.py | 36 ++++++-- jupyter_server/serverapp.py | 118 +++--------------------- tests/extension/conftest.py | 4 +- tests/extension/test_app.py | 12 +-- tests/extension/test_utils.py | 1 - 6 files changed, 51 insertions(+), 122 deletions(-) diff --git a/jupyter_server/extension/application.py b/jupyter_server/extension/application.py index 6b86123d4f..02ab43c0d1 100644 --- a/jupyter_server/extension/application.py +++ b/jupyter_server/extension/application.py @@ -385,7 +385,7 @@ def _load_jupyter_server_extension(cls, serverapp): """ try: # Get loaded extension from serverapp. - extension = serverapp._enabled_extensions[cls.name] + extension = serverapp.extension_manager.paths[cls.name].app except KeyError: extension = cls() extension.link_to_serverapp(serverapp) diff --git a/jupyter_server/extension/utils.py b/jupyter_server/extension/utils.py index af1d156830..8552916626 100644 --- a/jupyter_server/extension/utils.py +++ b/jupyter_server/extension/utils.py @@ -134,6 +134,9 @@ class ExtensionLoadingError(Exception): def get_loader(obj): """Looks for _load_jupyter_server_extension as an attribute of the object or module. + + Adds backwards compatibility for old function name missing the + underscore prefix. """ try: func = getattr(obj, '_load_jupyter_server_extension') @@ -151,18 +154,37 @@ def __init__(self, metadata): self.module = importlib.import_module(self.module_name) self.app = metadata.get("app", None) if self.app: + self.app = self.app() self.name = self.app.name - self.load = staticmethod(get_loader(self.app)) self.link = self.app._link_jupyter_server_extension + self.loader = get_loader(self.app) else: self.name = metadata.get("name", self.module_name) - self.load = staticmethod(get_loader(self.module)) - self.link = lambda: None + self.link = getattr( + self.module, + '_link_jupyter_server_extension', + lambda serverapp: None + ) + self.loader = get_loader(self.module) + + def load(self, serverapp): + return self.loader(serverapp) def get_metadata(package_name): + """Find the extension metadata from an extension package. + + If it doesn't exist, return a basic metadata package given + the module name. + """ module = importlib.import_module(package_name) - return module._jupyter_server_extension_paths() + try: + return module._jupyter_server_extension_paths() + except AttributeError: + return [{ + "module": package_name, + "name": package_name + }] class Extension: @@ -177,7 +199,8 @@ def __init__(self, package_name): class ExtensionManager: - + """ + """ def __init__(self, jpserver_extensions): self.extensions = {} for package_name, enabled in jpserver_extensions.items(): @@ -210,5 +233,4 @@ def validate_extension(name): submod_path = path.get("module") submod = importlib.import_module(submod_path) # Check that extension has loading function. - get_loader(submod) - + get_loader(submod) \ No newline at end of file diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index ec9cde9c3e..8a6577fb24 100755 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -557,10 +557,6 @@ class ServerApp(JupyterApp): This launches a Tornado-based Jupyter Server.""") examples = _examples - # This trait is used to track _enabled_extensions. It should remain hidden - # and not configurable. - _enabled_extensions = {} - flags = Dict(flags) aliases = Dict(aliases) @@ -1519,67 +1515,16 @@ def init_server_extensions(self): this instance will inherit the ServerApp's config object and load its own config. """ - # Load extension metadata for enabled extensions, load config for - # enabled ExtensionApps, and store enabled extension metadata in the - # _enabled_extensions attribute. - # - # The _enabled_extensions trait will be used by `load_server_extensions` - # to call each extensions `_load_jupyter_server_extension` method - # after the ServerApp's Web application object is created. - for module_name, enabled in sorted(self.jpserver_extensions.items()): - if enabled: - metadata_list = [] - try: - # Load the metadata for this enabled extension. This will - # be a list of extension points, each having their own - # path to a `_load_jupyter_server_extensions()`function. - # Important note: a single extension can have *multiple* - # `_load_jupyter_server_extension` functions defined, hence - # _get_server_extension_metadata returns a list of metadata. - mod, metadata_list = _get_server_extension_metadata(module_name) - except KeyError: - # A KeyError suggests that the module does not have a - # _jupyter_server_extension-path. - log_msg = _( - "Error loading server extensions in " - "{module_name} module. There is no `_jupyter_server_extension_paths` " - "defined at the root of the extension module. Check " - "with the author of the extension to ensure this function " - "is added.".format(module_name=module_name) - ) - self.log.warning(log_msg) - - for metadata in metadata_list: - # Is this extension point an ExtensionApp? - # If "app" is not None, then the extension should be an ExtensionApp - # Otherwise use the 'module' key to locate the - #`_load_jupyter_server_extension` function. - extapp = metadata.get('app', None) - extloc = metadata.get('module', None) - if extapp and extloc: - # Verify that object found is a subclass of ExtensionApp. - from .extension.application import ExtensionApp - if not issubclass(extapp, ExtensionApp): - raise TypeError(extapp.__name__ + " must be a subclass of ExtensionApp.") - # ServerApp creates an instance of the - # ExtensionApp here to load any ServerApp configuration - # that might live in the Extension's config file. - app = extapp() - app._link_jupyter_server_extension(self) - # Build a new list where we - self._enabled_extensions[app.name] = app - elif extloc: - extmod = importlib.import_module(extloc) - func = _get_load_jupyter_server_extension(extmod) - self._enabled_extensions[extloc] = extmod - else: - log_msg = _( - "{module_name} is missing critical metadata. Check " - "that _jupyter_server_extension_paths returns `app` " - "and/or `module` as keys".format(module_name=module_name) - ) - self.log.warn(log_msg) + from jupyter_server.extension.utils import ExtensionManager + # Initialize each extension + self.extension_manager = ExtensionManager(self.jpserver_extensions) + + for path in self.extension_manager.paths.values(): + try: + path.link(self) + except Exception as e: + self.log.warning(e) def load_server_extensions(self): """Load any extensions specified by config. @@ -1589,48 +1534,11 @@ def load_server_extensions(self): The extension API is experimental, and may change in future releases. """ - - # Load all enabled extensions. - for extkey, extension in sorted(self._enabled_extensions.items()): - if isinstance(extension, ModuleType): - log_msg = ( - "Extension from {extloc} module enabled and " - "loaded".format(extloc=extkey) - ) - else: - log_msg = ( - "Extension {name} enabled and " - "loaded".format(name=extension.name) - ) - # Find the extension loading function. - func = None + for path in self.extension_manager.paths.values(): try: - # This function was prefixed with an underscore in in v1.0 - # because this shouldn't be a public API for most extensions. - func = getattr(extension, '_load_jupyter_server_extension') - except AttributeError: - try: - # For backwards compatibility, we will still look for non - # underscored loading functions. - func = getattr(extension, 'load_jupyter_server_extension') - warn_msg = _( - "{extkey} is enabled. " - "`load_jupyter_server_extension` function " - "was found but `_load_jupyter_server_extension`" - "is preferred.".format(extkey=extkey) - ) - self.log.warning(warn_msg) - except AttributeError: - warn_msg = _( - "{extkey} is enabled but no " - "`_load_jupyter_server_extension` function " - "was found.".format(extkey=extkey) - ) - self.log.warning(warn_msg) - if func: - func(self) - self.log.debug(log_msg) - + path.load(self) + except Exception as e: + self.log.warning(e) def init_mime_overrides(self): # On some Windows machines, an application has registered incorrect diff --git a/tests/extension/conftest.py b/tests/extension/conftest.py index 677f1e55de..ef5d04488e 100644 --- a/tests/extension/conftest.py +++ b/tests/extension/conftest.py @@ -30,8 +30,8 @@ def mock_template(template_dir): @pytest.fixture -def enabled_extensions(serverapp): - return serverapp._enabled_extensions +def extension_manager(serverapp): + return serverapp.extension_manager @pytest.fixture diff --git a/tests/extension/test_app.py b/tests/extension/test_app.py index feaf299bc5..9e8990210b 100644 --- a/tests/extension/test_app.py +++ b/tests/extension/test_app.py @@ -21,8 +21,8 @@ def server_config(request, template_dir): @pytest.fixture -def mock_extension(enabled_extensions): - return enabled_extensions["mockextension"] +def mock_extension(extension_manager): + return extension_manager.paths["mockextension"].app def test_initialize(mock_extension, template_dir): @@ -46,19 +46,19 @@ def test_instance_creation_with_argv( serverapp, trait_name, trait_value, - enabled_extensions + extension_manager ): - extension = enabled_extensions['mockextension'] + extension = extension_manager.paths['mockextension'].app assert getattr(extension, trait_name) == trait_value def test_extensionapp_load_config_file( extension_environ, config_file, - enabled_extensions, + extension_manager, serverapp, ): - extension = enabled_extensions["mockextension"] + extension = extension_manager.paths['mockextension'].app # Assert default config_file_paths is the same in the app and extension. assert extension.config_file_paths == serverapp.config_file_paths assert extension.config_dir == serverapp.config_dir diff --git a/tests/extension/test_utils.py b/tests/extension/test_utils.py index 262dc4dcf9..e97d09538d 100644 --- a/tests/extension/test_utils.py +++ b/tests/extension/test_utils.py @@ -90,7 +90,6 @@ def test_extension_api(): # Testing the first path (which is an extension app). metadata_list = _jupyter_server_extension_paths() path1 = metadata_list[0] - module = path1["module"] app = path1["app"] e = Extension('tests.extension.mockextensions') From 3f315ea143523ff55cad0c04fcfb1686ababb2f1 Mon Sep 17 00:00:00 2001 From: Zsailer Date: Thu, 18 Jun 2020 07:59:31 -0700 Subject: [PATCH 07/30] switch to traitlets object --- jupyter_server/extension/utils.py | 172 +++++++++++++++++++++++------- 1 file changed, 135 insertions(+), 37 deletions(-) diff --git a/jupyter_server/extension/utils.py b/jupyter_server/extension/utils.py index 8552916626..3c6ef930bc 100644 --- a/jupyter_server/extension/utils.py +++ b/jupyter_server/extension/utils.py @@ -2,10 +2,14 @@ import importlib from jupyter_core.paths import jupyter_config_path +from traitlets import ( + HasTraits, + Dict, + Unicode, + validate +) from traitlets.config.loader import ( - JSONFileConfigLoader, - PyFileConfigLoader, - Config + JSONFileConfigLoader ) @@ -147,34 +151,113 @@ def get_loader(obj): return func -class ExtensionPath: +class ExtensionMetadataError(Exception): + pass + + +class ExtensionModuleNotFound(Exception): + pass + + +class ExtensionPoint(HasTraits): + """A simple API for connecting to a Jupyter Server extension + point defined by metadata and importable from a Python package. + + Usage: + + metadata = { + "module": "extension_module", + "": + } - def __init__(self, metadata): - self.module_name = metadata.get("module") - self.module = importlib.import_module(self.module_name) - self.app = metadata.get("app", None) + point = ExtensionPoint(metadata) + """ + metadata = Dict() + + @validate('metadata') + def _valid_metadata(self, metadata): + # Verify that the metadata has a "name" key. + try: + self._module_name = metadata['module'] + except KeyError: + raise ExtensionMetadataError( + "There is no 'name' key in the extension's " + "metadata packet." + ) + + try: + self._module = importlib.import_module(self._module_name) + except ModuleNotFoundError: + raise ExtensionModuleNotFound( + f"The module '{self._module_name}' could not be found. Are you " + "sure the extension is installed?" + ) + return metadata + + @property + def app(self): + """If the metadata includes an `app` field""" + return self.metadata.get("app") + + @property + def module_name(self): + """Name of the Python package module where the extension's + _load_jupyter_server_extension can be found. + """ + return self._module_name + + @property + def name(self): + """Name of the extension. + + If it's not provided in the metadata, `name` is set + to the extensions' module name. + """ + return self.metadata.get("name", self.module_name) + + @property + def module(self): + """The imported module (using importlib.import_module) + """ + return self._module + + def link(self, serverapp): + """Link the extension to a Jupyter ServerApp object. + + This looks for a `_link_jupyter_server_extension` function + in the extension's module or ExtensionApp class. + """ if self.app: - self.app = self.app() - self.name = self.app.name - self.link = self.app._link_jupyter_server_extension - self.loader = get_loader(self.app) + linker = self.app._link_jupyter_server_extension else: - self.name = metadata.get("name", self.module_name) - self.link = getattr( + linker = getattr( self.module, + # Search for a _link_jupyter_extension '_link_jupyter_server_extension', + # Otherwise return a dummy function. lambda serverapp: None ) - self.loader = get_loader(self.module) + return linker(serverapp) def load(self, serverapp): - return self.loader(serverapp) + """Load the extension in a Jupyter ServerApp object. + + This looks for a `_load_jupyter_server_extension` function + in the extension's module or ExtensionApp class. + """ + # Use the ExtensionApp object to find a loading function + # if it exists. Otherwise, use the extension module given. + loc = self.app + if not loc: + loc = self.module + loader = get_loader(loc) + return loader(serverapp) def get_metadata(package_name): """Find the extension metadata from an extension package. - If it doesn't exist, return a basic metadata package given + If it doesn't exist, return a basic metadata packet given the module name. """ module = importlib.import_module(package_name) @@ -187,28 +270,51 @@ def get_metadata(package_name): }] -class Extension: +class ExtensionPackage(HasTraits): + """API for handling + """ + name = Unicode(help="Name of the extension's Python package.") + + @validate("name") + def _validate_name(self, name): + try: + self._metadata = get_metadata(name) + except ModuleNotFoundError: + raise ExtensionModuleNotFound( + f"The module '{self._module_name}' could not be found. Are you " + "sure the extension is installed?" + ) + # Create extension point interfaces for each extension path. + for m in self._metadata: + point = ExtensionPoint(m) + self._extension_points[point.name] = point + return name + + @property + def metadata(self): + """Extension metadata loaded from the extension package.""" + return self._metadata - def __init__(self, package_name): - self.package_name = package_name - self.metadata = get_metadata(self.package_name) - self.paths = {} - for path_metadata in self.metadata: - path = ExtensionPath(path_metadata) - self.paths[path.name] = path + @property + def extension_points(self): + """A dictionary of extension points.""" + return self._extension_points class ExtensionManager: + """High level interface for linking, loading, and managing + Jupyter Server extensions. """ - """ + jpserver_extensions = Dict() + def __init__(self, jpserver_extensions): self.extensions = {} for package_name, enabled in jpserver_extensions.items(): if enabled: - self.extensions[package_name] = Extension(package_name) + self.extensions[package_name] = ExtensionPackage(package_name) @property - def paths(self): + def extension_points(self): _paths = {} for ext in self.extensions.values(): _paths.update(ext.paths) @@ -225,12 +331,4 @@ def validate_extension(name): If this works, nothing should happen. """ - # 1) Try importing - mod = importlib.import_module(name) - # 2) Try calling extension paths function. - paths = mod._jupyter_server_extension_paths() - for path in paths: - submod_path = path.get("module") - submod = importlib.import_module(submod_path) - # Check that extension has loading function. - get_loader(submod) \ No newline at end of file + ExtensionPackage(name) \ No newline at end of file From bd47f7ee43bb924c89065fdc67687e00d7b560ca Mon Sep 17 00:00:00 2001 From: Zsailer Date: Mon, 22 Jun 2020 07:49:45 -0700 Subject: [PATCH 08/30] fix some misused validation methods in traits --- jupyter_server/extension/utils.py | 63 +++++++++++++++++++++++-------- tests/extension/test_utils.py | 34 +++++++++-------- 2 files changed, 65 insertions(+), 32 deletions(-) diff --git a/jupyter_server/extension/utils.py b/jupyter_server/extension/utils.py index 3c6ef930bc..bd6a3a38df 100644 --- a/jupyter_server/extension/utils.py +++ b/jupyter_server/extension/utils.py @@ -8,6 +8,7 @@ Unicode, validate ) +from traitlets.config import LoggingConfigurable from traitlets.config.loader import ( JSONFileConfigLoader ) @@ -175,13 +176,14 @@ class ExtensionPoint(HasTraits): metadata = Dict() @validate('metadata') - def _valid_metadata(self, metadata): + def _valid_metadata(self, proposed): + metadata = proposed['value'] # Verify that the metadata has a "name" key. try: self._module_name = metadata['module'] except KeyError: raise ExtensionMetadataError( - "There is no 'name' key in the extension's " + "There is no 'module' key in the extension's " "metadata packet." ) @@ -213,6 +215,8 @@ def name(self): If it's not provided in the metadata, `name` is set to the extensions' module name. """ + if self.app: + return self.app.name return self.metadata.get("name", self.module_name) @property @@ -273,20 +277,22 @@ def get_metadata(package_name): class ExtensionPackage(HasTraits): """API for handling """ - name = Unicode(help="Name of the extension's Python package.") + name = Unicode(help="Name of the an importable Python package.") @validate("name") - def _validate_name(self, name): + def _validate_name(self, proposed): + name = proposed['value'] + self._extension_points = {} try: self._metadata = get_metadata(name) except ModuleNotFoundError: raise ExtensionModuleNotFound( - f"The module '{self._module_name}' could not be found. Are you " + f"The module '{name}' could not be found. Are you " "sure the extension is installed?" ) # Create extension point interfaces for each extension path. for m in self._metadata: - point = ExtensionPoint(m) + point = ExtensionPoint(metadata=m) self._extension_points[point.name] = point return name @@ -301,24 +307,49 @@ def extension_points(self): return self._extension_points -class ExtensionManager: - """High level interface for linking, loading, and managing - Jupyter Server extensions. +class ExtensionManager(LoggingConfigurable): + """High level interface for findind, validating, + linking, loading, and managing Jupyter Server extensions. + + Usage: + + m = ExtensionManager( + jpserver_extensions=extensions, + ) + + m. """ jpserver_extensions = Dict() - def __init__(self, jpserver_extensions): - self.extensions = {} + @validate('jpserver_extensions') + def _validate_jpserver_extensions(self, proposed): + jpserver_extensions = proposed['value'] + self._extensions = {} for package_name, enabled in jpserver_extensions.items(): if enabled: - self.extensions[package_name] = ExtensionPackage(package_name) + try: + self._extensions[package_name] = ExtensionPackage(name=package_name) + # Raise a warning if the extension cannot be loaded. + except Exception as e: + self.log.warning(e) + return jpserver_extensions + + @property + def extensions(self): + """Dictionary with extension package names as keys + and an ExtensionPackage objects as values. + """ + return self._extensions @property def extension_points(self): - _paths = {} + points = {} for ext in self.extensions.values(): - _paths.update(ext.paths) - return _paths + points.update(ext.extension_points) + return points + + def link_extensions(self): + def validate_extension(name): @@ -331,4 +362,4 @@ def validate_extension(name): If this works, nothing should happen. """ - ExtensionPackage(name) \ No newline at end of file + return ExtensionPackage(name) \ No newline at end of file diff --git a/tests/extension/test_utils.py b/tests/extension/test_utils.py index e97d09538d..48fe4e4e22 100644 --- a/tests/extension/test_utils.py +++ b/tests/extension/test_utils.py @@ -2,8 +2,8 @@ from jupyter_server.extension.utils import ( list_extensions_in_configd, configd_enabled, - ExtensionPath, - Extension, + ExtensionPoint, + ExtensionPackage, ExtensionManager ) @@ -64,18 +64,19 @@ def test_config_enabled(ext1_config): assert configd_enabled("ext1_config") -def test_extension_path_api(): +def test_extension_point_api(): # Import mock extension metadata from .mockextensions import _jupyter_server_extension_paths # Testing the first path (which is an extension app). metadata_list = _jupyter_server_extension_paths() - path = metadata_list[0] + point = metadata_list[0] - module = path["module"] - app = path["app"] + module = point["module"] + app = point["app"] - e = ExtensionPath(path) + print(point) + e = ExtensionPoint(metadata=point) assert e.module_name == module assert e.name == app.name assert app is not None @@ -83,7 +84,7 @@ def test_extension_path_api(): assert callable(e.link) -def test_extension_api(): +def test_extension_package_api(): # Import mock extension metadata from .mockextensions import _jupyter_server_extension_paths @@ -92,10 +93,11 @@ def test_extension_api(): path1 = metadata_list[0] app = path1["app"] - e = Extension('tests.extension.mockextensions') - assert hasattr(e, "paths") - assert len(e.paths) == len(metadata_list) - assert app.name in e.paths + e = ExtensionPackage(name='tests.extension.mockextensions') + e.extension_points + assert hasattr(e, "extension_points") + assert len(e.extension_points) == len(metadata_list) + assert app.name in e.extension_points def test_extension_manager_api(): @@ -108,8 +110,8 @@ def test_extension_manager_api(): jpserver_extensions = { "tests.extension.mockextensions": True } - manager = ExtensionManager(jpserver_extensions) + manager = ExtensionManager(jpserver_extensions=jpserver_extensions) assert len(manager.extensions) == 1 - assert len(manager.paths) == len(metadata_list) - assert "mockextension" in manager.paths - assert "tests.extension.mockextensions.mock1" in manager.paths \ No newline at end of file + assert len(manager.extension_points) == len(metadata_list) + assert "mockextension" in manager.extension_points + assert "tests.extension.mockextensions.mock1" in manager.extension_points \ No newline at end of file From fd3b27f3c63aca2598df1a389f21afe29d593b5e Mon Sep 17 00:00:00 2001 From: Zsailer Date: Wed, 24 Jun 2020 20:37:27 -0700 Subject: [PATCH 09/30] add extension manager tests --- jupyter_server/extension/application.py | 3 +- jupyter_server/extension/manager.py | 241 ++++++++++++++++++++ jupyter_server/extension/serverextension.py | 117 ++-------- jupyter_server/extension/utils.py | 202 +--------------- jupyter_server/serverapp.py | 36 +-- tests/extension/test_app.py | 6 +- tests/extension/test_entrypoint.py | 35 +-- tests/extension/test_manager.py | 79 +++++++ tests/extension/test_serverextension.py | 16 +- tests/extension/test_utils.py | 64 +----- 10 files changed, 376 insertions(+), 423 deletions(-) create mode 100644 jupyter_server/extension/manager.py create mode 100644 tests/extension/test_manager.py diff --git a/jupyter_server/extension/application.py b/jupyter_server/extension/application.py index 02ab43c0d1..ad7548b5eb 100644 --- a/jupyter_server/extension/application.py +++ b/jupyter_server/extension/application.py @@ -383,9 +383,10 @@ def _load_jupyter_server_extension(cls, serverapp): """Initialize and configure this extension, then add the extension's settings and handlers to the server's web application. """ + extension_manager = serverapp.extension_manager try: # Get loaded extension from serverapp. - extension = serverapp.extension_manager.paths[cls.name].app + extension = extension_manager.extension_points[cls.name].app except KeyError: extension = cls() extension.link_to_serverapp(serverapp) diff --git a/jupyter_server/extension/manager.py b/jupyter_server/extension/manager.py new file mode 100644 index 0000000000..e572d2cd35 --- /dev/null +++ b/jupyter_server/extension/manager.py @@ -0,0 +1,241 @@ +import importlib + +from traitlets.config import LoggingConfigurable +from traitlets import ( + HasTraits, + Dict, + Unicode, + Instance, + default, + validate +) + +from .utils import ( + ExtensionMetadataError, + ExtensionModuleNotFound, + get_loader, + get_metadata, +) + + +class ExtensionPoint(HasTraits): + """A simple API for connecting to a Jupyter Server extension + point defined by metadata and importable from a Python package. + + Usage: + + metadata = { + "module": "extension_module", + "": + } + + point = ExtensionPoint(metadata) + """ + metadata = Dict() + + @validate('metadata') + def _valid_metadata(self, proposed): + metadata = proposed['value'] + # Verify that the metadata has a "name" key. + try: + self._module_name = metadata['module'] + except KeyError: + raise ExtensionMetadataError( + "There is no 'module' key in the extension's " + "metadata packet." + ) + + try: + self._module = importlib.import_module(self._module_name) + except ModuleNotFoundError: + raise ExtensionModuleNotFound( + f"The module '{self._module_name}' could not be found. Are " + "you sure the extension is installed?" + ) + # Initialize the app object if it exists. + app = self.metadata.get("app") + if app: + metadata["app"] = app() + return metadata + + @property + def app(self): + """If the metadata includes an `app` field""" + return self.metadata.get("app") + + @property + def module_name(self): + """Name of the Python package module where the extension's + _load_jupyter_server_extension can be found. + """ + return self._module_name + + @property + def name(self): + """Name of the extension. + + If it's not provided in the metadata, `name` is set + to the extensions' module name. + """ + if self.app: + return self.app.name + return self.metadata.get("name", self.module_name) + + @property + def module(self): + """The imported module (using importlib.import_module) + """ + return self._module + + def link(self, serverapp): + """Link the extension to a Jupyter ServerApp object. + + This looks for a `_link_jupyter_server_extension` function + in the extension's module or ExtensionApp class. + """ + if self.app: + linker = self.app._link_jupyter_server_extension + else: + linker = getattr( + self.module, + # Search for a _link_jupyter_extension + '_link_jupyter_server_extension', + # Otherwise return a dummy function. + lambda serverapp: None + ) + return linker(serverapp) + + def load(self, serverapp): + """Load the extension in a Jupyter ServerApp object. + + This looks for a `_load_jupyter_server_extension` function + in the extension's module or ExtensionApp class. + """ + # Use the ExtensionApp object to find a loading function + # if it exists. Otherwise, use the extension module given. + loc = self.app + if not loc: + loc = self.module + loader = get_loader(loc) + return loader(serverapp) + + +class ExtensionPackage(HasTraits): + """An API for interfacing with a Jupyter Server extension package. + + Usage: + + ext_name = "my_extensions" + extpkg = ExtensionPackage(name=ext_name) + """ + name = Unicode(help="Name of the an importable Python package.") + + @validate("name") + def _validate_name(self, proposed): + name = proposed['value'] + self._extension_points = {} + try: + self._metadata = get_metadata(name) + except ModuleNotFoundError: + raise ExtensionModuleNotFound( + f"The module '{name}' could not be found. Are you " + "sure the extension is installed?" + ) + # Create extension point interfaces for each extension path. + for m in self._metadata: + point = ExtensionPoint(metadata=m) + self._extension_points[point.name] = point + return name + + @property + def metadata(self): + """Extension metadata loaded from the extension package.""" + return self._metadata + + @property + def extension_points(self): + """A dictionary of extension points.""" + return self._extension_points + + +class ExtensionManager(LoggingConfigurable): + """High level interface for findind, validating, + linking, loading, and managing Jupyter Server extensions. + + Usage: + + m = ExtensionManager(jpserver_extensions=extensions) + """ + parent = Instance( + klass="jupyter_server.serverapp.ServerApp", + allow_none=True + ) + + jpserver_extensions = Dict( + help=( + "A dictionary with extension package names " + "as keys and booleans to enable as values." + ) + ) + + @default('jpserver_extensions') + def _default_jpserver_extensions(self): + return self.parent.jpserver_extensions + + @validate('jpserver_extensions') + def _validate_jpserver_extensions(self, proposed): + jpserver_extensions = proposed['value'] + self._extensions = {} + # Iterate over dictionary items and validate that + # we can interface with each extension. If the extension + # fails to interface, throw a warning through the logger + # interface. + for package_name, enabled in jpserver_extensions.items(): + if enabled: + try: + self._extensions[package_name] = ExtensionPackage( + name=package_name + ) + # Raise a warning if the extension cannot be loaded. + except Exception as e: + self.log.warning(e) + return jpserver_extensions + + @property + def extensions(self): + """Dictionary with extension package names as keys + and an ExtensionPackage objects as values. + """ + return self._extensions + + @property + def extension_points(self): + points = {} + for ext in self.extensions.values(): + points.update(ext.extension_points) + return points + + def link_extensions(self): + """Link all enabled extensions + to an instance of ServerApp + """ + # Sort the extension names to enforce deterministic linking + # order. + for name, ext in sorted(self.extension_points.items()): + try: + ext.link(self.parent) + except Exception as e: + self.log.warning(e) + + def load_extensions(self): + """Load all enabled extensions and append them to + the parent ServerApp. + """ + # Sort the extension names to enforce deterministic loading + # order. + for name, ext in sorted(self.extension_points.items()): + try: + ext.load(self.parent) + except Exception as e: + self.log.warning(e) + diff --git a/jupyter_server/extension/serverextension.py b/jupyter_server/extension/serverextension.py index 444e64d5fb..7bd77bfe0b 100644 --- a/jupyter_server/extension/serverextension.py +++ b/jupyter_server/extension/serverextension.py @@ -6,10 +6,8 @@ import os import sys -import importlib from tornado.log import LogFormatter -from traitlets import Bool, Any -from traitlets.utils.importstring import import_item +from traitlets import Bool from jupyter_core.application import JupyterApp from jupyter_core.paths import ( @@ -20,30 +18,7 @@ ) from jupyter_server._version import __version__ from jupyter_server.config_manager import BaseJSONConfigManager - - -def _get_server_extension_metadata(module): - """Load server extension metadata from a module. - - Returns a tuple of ( - the package as loaded - a list of server extension specs: [ - { - "module": "import.path.to.extension" - } - ] - ) - - Parameters - ---------- - module : str - Importable Python module exposing the - magic-named `_jupyter_server_extension_paths` function - """ - m = import_item(module) - if not hasattr(m, '_jupyter_server_extension_paths'): - raise KeyError(u'The Python module {} does not include any valid server extensions'.format(module)) - return m, m._jupyter_server_extension_paths() +from .utils import validate_extension class ArgumentConflict(ValueError): @@ -134,75 +109,14 @@ def _get_config_dir(user=False, sys_prefix=False): # Public API # ------------------------------------------------------------------------------ -class ExtensionLoadingError(Exception): pass - - -class ExtensionValidationError(Exception): pass - - - -def _get_load_jupyter_server_extension(obj): - """Looks for load_jupyter_server_extension as an attribute - of the object or module. - """ - try: - func = getattr(obj, '_load_jupyter_server_extension') - except AttributeError: - func = getattr(obj, 'load_jupyter_server_extension') - except BaseException: - raise ExtensionLoadingError( - "_load_jupyter_server_extension function was not found." - ) from e - return func - - -def validate_server_extension(name): - """Validates that you can import the extension module, - gather all extension metadata, and find `load_jupyter_server_extension` - functions for each extension. - - Raises a validation error if extensions cannot be found. - - Parameter - --------- - extension_module: module - The extension module (first value) returned by _get_server_extension_metadata - - extension_metadata : list - The list (second value) returned by _get_server_extension_metadata - - Returns - ------- - version : str - Extension version. - """ - # If the extension does not exist, raise an exception - try: - mod, metadata = _get_server_extension_metadata(name) - version = getattr(mod, '__version__', '') - except ImportError as e: - raise ExtensionValidationError('{} is not importable.'.format(name)) from e - - try: - for item in metadata: - extapp = item.get('app', None) - extloc = item.get('module', None) - if extapp and extloc: - func = _get_load_jupyter_server_extension(extapp) - elif extloc: - extmod = importlib.import_module(extloc) - func = _get_load_jupyter_server_extension(extmod) - else: - raise AttributeError - # If the extension does not have a `load_jupyter_server_extension` function, raise exception. - except AttributeError as e: - raise ExtensionValidationError( - 'Found "{}" module but cannot load it.'.format(name) - ) from e - return version - -def toggle_server_extension_python(import_name, enabled=None, parent=None, user=False, sys_prefix=True): +def toggle_server_extension_python( + import_name, + enabled=None, + parent=None, + user=False, + sys_prefix=True +): """Toggle the boolean setting for a given server extension in a Jupyter config file. """ @@ -283,7 +197,7 @@ def toggle_server_extension(self, import_name): self.log.info("{}: {}".format(self._toggle_pre_message.capitalize(), import_name)) # Validate the server extension. self.log.info(" - Validating {}...".format(import_name)) - version = validate_server_extension(import_name) + version = validate_extension(import_name) # Toggle the server extension to active. toggle_server_extension_python( @@ -297,7 +211,7 @@ def toggle_server_extension(self, import_name): # If successful, let's log. self.log.info(" - Extension successfully {}.".format(self._toggle_post_message)) - except ExtensionValidationError as err: + except Exception as err: self.log.info(" {} Validation failed: {}".format(RED_X, err)) def toggle_server_extension_python(self, package): @@ -383,16 +297,17 @@ def list_server_extensions(self): # Iterate over packages listed in jpserver_extensions. for pkg_name, enabled in server_extensions.items(): # Attempt to get extension metadata - _, __ = _get_server_extension_metadata(pkg_name) self.log.info(u' {} {}'.format( pkg_name, GREEN_ENABLED if enabled else RED_DISABLED)) try: self.log.info(" - Validating {}...".format(pkg_name)) - version = validate_server_extension(pkg_name) - self.log.info(" {} {} {}".format(pkg_name, version, GREEN_OK)) + version = validate_extension(pkg_name) + self.log.info( + " {} {} {}".format(pkg_name, version, GREEN_OK) + ) - except ExtensionValidationError as err: + except Exception as err: self.log.warn(" {} {}".format(RED_X, err)) def start(self): diff --git a/jupyter_server/extension/utils.py b/jupyter_server/extension/utils.py index bd6a3a38df..32898b06bb 100644 --- a/jupyter_server/extension/utils.py +++ b/jupyter_server/extension/utils.py @@ -2,13 +2,6 @@ import importlib from jupyter_core.paths import jupyter_config_path -from traitlets import ( - HasTraits, - Dict, - Unicode, - validate -) -from traitlets.config import LoggingConfigurable from traitlets.config.loader import ( JSONFileConfigLoader ) @@ -136,6 +129,14 @@ class ExtensionLoadingError(Exception): pass +class ExtensionMetadataError(Exception): + pass + + +class ExtensionModuleNotFound(Exception): + pass + + def get_loader(obj): """Looks for _load_jupyter_server_extension as an attribute of the object or module. @@ -152,112 +153,6 @@ def get_loader(obj): return func -class ExtensionMetadataError(Exception): - pass - - -class ExtensionModuleNotFound(Exception): - pass - - -class ExtensionPoint(HasTraits): - """A simple API for connecting to a Jupyter Server extension - point defined by metadata and importable from a Python package. - - Usage: - - metadata = { - "module": "extension_module", - "": - } - - point = ExtensionPoint(metadata) - """ - metadata = Dict() - - @validate('metadata') - def _valid_metadata(self, proposed): - metadata = proposed['value'] - # Verify that the metadata has a "name" key. - try: - self._module_name = metadata['module'] - except KeyError: - raise ExtensionMetadataError( - "There is no 'module' key in the extension's " - "metadata packet." - ) - - try: - self._module = importlib.import_module(self._module_name) - except ModuleNotFoundError: - raise ExtensionModuleNotFound( - f"The module '{self._module_name}' could not be found. Are you " - "sure the extension is installed?" - ) - return metadata - - @property - def app(self): - """If the metadata includes an `app` field""" - return self.metadata.get("app") - - @property - def module_name(self): - """Name of the Python package module where the extension's - _load_jupyter_server_extension can be found. - """ - return self._module_name - - @property - def name(self): - """Name of the extension. - - If it's not provided in the metadata, `name` is set - to the extensions' module name. - """ - if self.app: - return self.app.name - return self.metadata.get("name", self.module_name) - - @property - def module(self): - """The imported module (using importlib.import_module) - """ - return self._module - - def link(self, serverapp): - """Link the extension to a Jupyter ServerApp object. - - This looks for a `_link_jupyter_server_extension` function - in the extension's module or ExtensionApp class. - """ - if self.app: - linker = self.app._link_jupyter_server_extension - else: - linker = getattr( - self.module, - # Search for a _link_jupyter_extension - '_link_jupyter_server_extension', - # Otherwise return a dummy function. - lambda serverapp: None - ) - return linker(serverapp) - - def load(self, serverapp): - """Load the extension in a Jupyter ServerApp object. - - This looks for a `_load_jupyter_server_extension` function - in the extension's module or ExtensionApp class. - """ - # Use the ExtensionApp object to find a loading function - # if it exists. Otherwise, use the extension module given. - loc = self.app - if not loc: - loc = self.module - loader = get_loader(loc) - return loader(serverapp) - - def get_metadata(package_name): """Find the extension metadata from an extension package. @@ -274,84 +169,6 @@ def get_metadata(package_name): }] -class ExtensionPackage(HasTraits): - """API for handling - """ - name = Unicode(help="Name of the an importable Python package.") - - @validate("name") - def _validate_name(self, proposed): - name = proposed['value'] - self._extension_points = {} - try: - self._metadata = get_metadata(name) - except ModuleNotFoundError: - raise ExtensionModuleNotFound( - f"The module '{name}' could not be found. Are you " - "sure the extension is installed?" - ) - # Create extension point interfaces for each extension path. - for m in self._metadata: - point = ExtensionPoint(metadata=m) - self._extension_points[point.name] = point - return name - - @property - def metadata(self): - """Extension metadata loaded from the extension package.""" - return self._metadata - - @property - def extension_points(self): - """A dictionary of extension points.""" - return self._extension_points - - -class ExtensionManager(LoggingConfigurable): - """High level interface for findind, validating, - linking, loading, and managing Jupyter Server extensions. - - Usage: - - m = ExtensionManager( - jpserver_extensions=extensions, - ) - - m. - """ - jpserver_extensions = Dict() - - @validate('jpserver_extensions') - def _validate_jpserver_extensions(self, proposed): - jpserver_extensions = proposed['value'] - self._extensions = {} - for package_name, enabled in jpserver_extensions.items(): - if enabled: - try: - self._extensions[package_name] = ExtensionPackage(name=package_name) - # Raise a warning if the extension cannot be loaded. - except Exception as e: - self.log.warning(e) - return jpserver_extensions - - @property - def extensions(self): - """Dictionary with extension package names as keys - and an ExtensionPackage objects as values. - """ - return self._extensions - - @property - def extension_points(self): - points = {} - for ext in self.extensions.values(): - points.update(ext.extension_points) - return points - - def link_extensions(self): - - - def validate_extension(name): """Raises an exception is the extension is missing a needed hook or metadata field. @@ -362,4 +179,5 @@ def validate_extension(name): If this works, nothing should happen. """ - return ExtensionPackage(name) \ No newline at end of file + from .manager import ExtensionPackage + return ExtensionPackage(name=name) \ No newline at end of file diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index 8a6577fb24..ffad524760 100755 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -100,11 +100,8 @@ from ._tz import utcnow, utcfromtimestamp from .utils import url_path_join, check_pid, url_escape, urljoin, pathname2url -from jupyter_server.extension.serverextension import ( - ServerExtensionApp, - _get_server_extension_metadata, - _get_load_jupyter_server_extension -) +from jupyter_server.extension.serverextension import ServerExtensionApp +from jupyter_server.extension.manager import ExtensionManager #----------------------------------------------------------------------------- # Module globals @@ -393,6 +390,7 @@ def start(self): set_password(config_file=self.config_file) self.log.info("Wrote hashed password to %s" % self.config_file) + def shutdown_server(server_info, timeout=5, log=None): """Shutdown a notebook server in a separate process. @@ -1488,17 +1486,12 @@ def find_server_extensions(self): # Load server extensions with ConfigManager. # This enables merging on keys, which we want for extension enabling. # Regular config loading only merges at the class level, - # so each level (system > env > user ... opposite of jupyter/notebook) - # clobbers the previous. + # so each level clobbers the previous. config_path = jupyter_config_path() if self.config_dir not in config_path: # add self.config_dir to the front, if set manually config_path.insert(0, self.config_dir) - # Flip the order of ordered_config_path to system > env > user. - # This is different that jupyter/notebook. See the Jupyter - # Enhancement Proposal 29 (Jupyter Server) for more information. - reversed_config_path = config_path[::-1] - manager = ConfigManager(read_config_path=reversed_config_path) + manager = ConfigManager(read_config_path=config_path) section = manager.get(self.config_file_name) extensions = section.get('ServerApp', {}).get('jpserver_extensions', {}) @@ -1515,16 +1508,13 @@ def init_server_extensions(self): this instance will inherit the ServerApp's config object and load its own config. """ - from jupyter_server.extension.utils import ExtensionManager # Initialize each extension - self.extension_manager = ExtensionManager(self.jpserver_extensions) - - for path in self.extension_manager.paths.values(): - try: - path.link(self) - except Exception as e: - self.log.warning(e) + self.extension_manager = ExtensionManager( + parent=self, + jpserver_extensions=self.jpserver_extensions + ) + self.extension_manager.link_extensions() def load_server_extensions(self): """Load any extensions specified by config. @@ -1534,11 +1524,7 @@ def load_server_extensions(self): The extension API is experimental, and may change in future releases. """ - for path in self.extension_manager.paths.values(): - try: - path.load(self) - except Exception as e: - self.log.warning(e) + self.extension_manager.load_extensions() def init_mime_overrides(self): # On some Windows machines, an application has registered incorrect diff --git a/tests/extension/test_app.py b/tests/extension/test_app.py index 9e8990210b..9e7e068882 100644 --- a/tests/extension/test_app.py +++ b/tests/extension/test_app.py @@ -22,7 +22,7 @@ def server_config(request, template_dir): @pytest.fixture def mock_extension(extension_manager): - return extension_manager.paths["mockextension"].app + return extension_manager.extension_points["mockextension"].app def test_initialize(mock_extension, template_dir): @@ -48,7 +48,7 @@ def test_instance_creation_with_argv( trait_value, extension_manager ): - extension = extension_manager.paths['mockextension'].app + extension = extension_manager.extension_points['mockextension'].app assert getattr(extension, trait_name) == trait_value @@ -58,7 +58,7 @@ def test_extensionapp_load_config_file( extension_manager, serverapp, ): - extension = extension_manager.paths['mockextension'].app + extension = extension_manager.extension_points['mockextension'].app # Assert default config_file_paths is the same in the app and extension. assert extension.config_file_paths == serverapp.config_file_paths assert extension.config_dir == serverapp.config_dir diff --git a/tests/extension/test_entrypoint.py b/tests/extension/test_entrypoint.py index 49c436aa21..483f980eb6 100644 --- a/tests/extension/test_entrypoint.py +++ b/tests/extension/test_entrypoint.py @@ -12,38 +12,5 @@ def test_server_extension_list(environ, script_runner): 'server', 'extension', 'list', - env=os.environ ) - assert ret.success - - -def test_server_extension_enable(environ, script_runner): - # 'mock' is not a valid extension The entry point should complete - # but print to sterr. - extension_name = "mockextension" - ret = script_runner.run( - "jupyter", - "server", - "extension", - "enable", - extension_name, - env=os.environ - ) - assert ret.success - assert 'Enabling: {}'.format(extension_name) in ret.stderr - - -def test_server_extension_disable(environ, script_runner): - # 'mock' is not a valid extension The entry point should complete - # but print to sterr. - extension_name = 'mockextension' - ret = script_runner.run( - 'jupyter', - 'server', - 'extension', - 'disable', - extension_name, - env=os.environ - ) - assert ret.success - assert 'Disabling: {}'.format(extension_name) in ret.stderr + assert ret.success \ No newline at end of file diff --git a/tests/extension/test_manager.py b/tests/extension/test_manager.py new file mode 100644 index 0000000000..9866c2b15b --- /dev/null +++ b/tests/extension/test_manager.py @@ -0,0 +1,79 @@ +import pytest +from jupyter_server.extension.manager import ( + ExtensionPoint, + ExtensionPackage, + ExtensionManager, + ExtensionMetadataError, + ExtensionModuleNotFound +) + + +def test_extension_point_api(): + # Import mock extension metadata + from .mockextensions import _jupyter_server_extension_paths + + # Testing the first path (which is an extension app). + metadata_list = _jupyter_server_extension_paths() + point = metadata_list[0] + + module = point["module"] + app = point["app"] + + e = ExtensionPoint(metadata=point) + assert e.module_name == module + assert e.name == app.name + assert app is not None + assert callable(e.load) + assert callable(e.link) + + +def test_extension_point_metadata_error(): + # Missing the "module" key. + bad_metadata = {"name": "nonexistent"} + with pytest.raises(ExtensionMetadataError): + ExtensionPoint(metadata=bad_metadata) + + +def test_extension_point_notfound_error(): + bad_metadata = {"module": "nonexistent"} + with pytest.raises(ExtensionModuleNotFound): + ExtensionPoint(metadata=bad_metadata) + + +def test_extension_package_api(): + # Import mock extension metadata + from .mockextensions import _jupyter_server_extension_paths + + # Testing the first path (which is an extension app). + metadata_list = _jupyter_server_extension_paths() + path1 = metadata_list[0] + app = path1["app"] + + e = ExtensionPackage(name='tests.extension.mockextensions') + e.extension_points + assert hasattr(e, "extension_points") + assert len(e.extension_points) == len(metadata_list) + assert app.name in e.extension_points + + +def test_extension_package_notfound_error(): + with pytest.raises(ExtensionModuleNotFound): + ExtensionPackage(name="nonexistent") + + +def test_extension_manager_api(): + # Import mock extension metadata + from .mockextensions import _jupyter_server_extension_paths + + # Testing the first path (which is an extension app). + metadata_list = _jupyter_server_extension_paths() + + jpserver_extensions = { + "tests.extension.mockextensions": True + } + manager = ExtensionManager(jpserver_extensions=jpserver_extensions) + assert len(manager.extensions) == 1 + assert len(manager.extension_points) == len(metadata_list) + assert "mockextension" in manager.extension_points + assert "tests.extension.mockextensions.mock1" in manager.extension_points + diff --git a/tests/extension/test_serverextension.py b/tests/extension/test_serverextension.py index 69cabf6542..009d5b059b 100644 --- a/tests/extension/test_serverextension.py +++ b/tests/extension/test_serverextension.py @@ -3,7 +3,6 @@ from traitlets.tests.utils import check_help_all_output from jupyter_server.extension.serverextension import ( - validate_server_extension, toggle_server_extension_python, _get_config_dir ) @@ -43,15 +42,6 @@ def test_merge_config( configurable_serverapp, extension_environ ): - # enabled at sys level - validate_server_extension('tests.extension.mockextensions.mockext_sys') - # enabled at sys, disabled at user - validate_server_extension('tests.extension.mockextensions.mockext_both') - # enabled at user - validate_server_extension('tests.extension.mockextensions.mockext_user') - # enabled at Python - validate_server_extension('tests.extension.mockextensions.mockext_py') - # Toggle each extension module with a JSON config file # at the sys-prefix config dir. toggle_server_extension_python( @@ -71,12 +61,12 @@ def test_merge_config( toggle_server_extension_python( 'tests.extension.mockextensions.mockext_both', enabled=True, - user=True + sys_prefix=True ) toggle_server_extension_python( 'tests.extension.mockextensions.mockext_both', enabled=False, - sys_prefix=True + user=True ) arg = "--ServerApp.jpserver_extensions={{'{mockext_py}': True}}".format( @@ -88,7 +78,7 @@ def test_merge_config( config_dir=str(env_config_path), argv=[arg] ) - # Verify that extensions are enabled and merged properly. + # Verify that extensions are enabled and merged in proper order. extensions = app.jpserver_extensions assert extensions['tests.extension.mockextensions.mockext_user'] assert extensions['tests.extension.mockextensions.mockext_sys'] diff --git a/tests/extension/test_utils.py b/tests/extension/test_utils.py index 48fe4e4e22..5e24d88b99 100644 --- a/tests/extension/test_utils.py +++ b/tests/extension/test_utils.py @@ -2,9 +2,7 @@ from jupyter_server.extension.utils import ( list_extensions_in_configd, configd_enabled, - ExtensionPoint, - ExtensionPackage, - ExtensionManager + validate_extension ) # Use ServerApps environment because it monkeypatches @@ -64,54 +62,12 @@ def test_config_enabled(ext1_config): assert configd_enabled("ext1_config") -def test_extension_point_api(): - # Import mock extension metadata - from .mockextensions import _jupyter_server_extension_paths - - # Testing the first path (which is an extension app). - metadata_list = _jupyter_server_extension_paths() - point = metadata_list[0] - - module = point["module"] - app = point["app"] - - print(point) - e = ExtensionPoint(metadata=point) - assert e.module_name == module - assert e.name == app.name - assert app is not None - assert callable(e.load) - assert callable(e.link) - - -def test_extension_package_api(): - # Import mock extension metadata - from .mockextensions import _jupyter_server_extension_paths - - # Testing the first path (which is an extension app). - metadata_list = _jupyter_server_extension_paths() - path1 = metadata_list[0] - app = path1["app"] - - e = ExtensionPackage(name='tests.extension.mockextensions') - e.extension_points - assert hasattr(e, "extension_points") - assert len(e.extension_points) == len(metadata_list) - assert app.name in e.extension_points - - -def test_extension_manager_api(): - # Import mock extension metadata - from .mockextensions import _jupyter_server_extension_paths - - # Testing the first path (which is an extension app). - metadata_list = _jupyter_server_extension_paths() - - jpserver_extensions = { - "tests.extension.mockextensions": True - } - manager = ExtensionManager(jpserver_extensions=jpserver_extensions) - assert len(manager.extensions) == 1 - assert len(manager.extension_points) == len(metadata_list) - assert "mockextension" in manager.extension_points - assert "tests.extension.mockextensions.mock1" in manager.extension_points \ No newline at end of file +def test_validate_extension(): + # enabled at sys level + assert validate_extension('tests.extension.mockextensions.mockext_sys') + # enabled at sys, disabled at user + assert validate_extension('tests.extension.mockextensions.mockext_both') + # enabled at user + assert validate_extension('tests.extension.mockextensions.mockext_user') + # enabled at Python + assert validate_extension('tests.extension.mockextensions.mockext_py') \ No newline at end of file From ed3317cee049f779a40653e112b9e7a8da064dec Mon Sep 17 00:00:00 2001 From: Zsailer Date: Wed, 24 Jun 2020 21:16:42 -0700 Subject: [PATCH 10/30] drop string literal for python 3.5 --- examples/simple/pyproject.toml | 3 +++ examples/simple/setup.py | 31 ++++++++++++++++++++--------- jupyter_server/extension/manager.py | 8 ++++---- pyproject.toml | 2 +- 4 files changed, 30 insertions(+), 14 deletions(-) create mode 100644 examples/simple/pyproject.toml diff --git a/examples/simple/pyproject.toml b/examples/simple/pyproject.toml new file mode 100644 index 0000000000..e2d7e08323 --- /dev/null +++ b/examples/simple/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["jupyter_packaging~=0.5.0", "setuptools>=40.8.0", "wheel"] +build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/examples/simple/setup.py b/examples/simple/setup.py index b1dbc9379e..92be6e984d 100755 --- a/examples/simple/setup.py +++ b/examples/simple/setup.py @@ -1,20 +1,25 @@ -import os, setuptools -from setuptools import find_packages +import os +from setuptools import setup +from jupyter_packaging import create_cmdclass + VERSION = '0.0.1' + def get_data_files(): """Get the data files for the package. """ data_files = [ - ('etc/jupyter/jupyter_server_config.d', ['etc/jupyter/jupyter_server_config.d/simple_ext1.json']), - ('etc/jupyter/jupyter_server_config.d', ['etc/jupyter/jupyter_server_config.d/simple_ext2.json']), - ('etc/jupyter/jupyter_server_config.d', ['etc/jupyter/jupyter_server_config.d/simple_ext11.json']), + ('etc/jupyter/jupyter_server_config.d', 'etc/jupyter/jupyter_server_config.d/', '*.json'), + # ('etc/jupyter/jupyter_server_config.d', ['etc/jupyter/jupyter_server_config.d/simple_ext1.json']), + # ('etc/jupyter/jupyter_server_config.d', ['etc/jupyter/jupyter_server_config.d/simple_ext2.json']), + # ('etc/jupyter/jupyter_server_config.d', ['etc/jupyter/jupyter_server_config.d/simple_ext11.json']), ] def add_data_files(path): for (dirpath, dirnames, filenames) in os.walk(path): if filenames: - data_files.append((dirpath, [os.path.join(dirpath, filename) for filename in filenames])) + paths = [(dirpath, dirpath, filename) for filename in filenames] + data_files.extend(paths) # Add all static and templates folders. add_data_files('simple_ext1/static') add_data_files('simple_ext1/templates') @@ -22,19 +27,23 @@ def add_data_files(path): add_data_files('simple_ext2/templates') return data_files -setuptools.setup( + +cmdclass = create_cmdclass( + data_files_spec=get_data_files() +) + +setup_args = dict( name = 'jupyter_server_example', version = VERSION, description = 'Jupyter Server Example', long_description = open('README.md').read(), - packages = find_packages(), python_requires = '>=3.5', install_requires = [ 'jupyter_server', 'jinja2', ], include_package_data=True, - data_files = get_data_files(), + cmdclass = cmdclass, entry_points = { 'console_scripts': [ 'jupyter-simple-ext1 = simple_ext1.application:main', @@ -43,3 +52,7 @@ def add_data_files(path): ] }, ) + + +if __name__ == '__main__': + setup(**setup_args) \ No newline at end of file diff --git a/jupyter_server/extension/manager.py b/jupyter_server/extension/manager.py index e572d2cd35..5676f0fe4b 100644 --- a/jupyter_server/extension/manager.py +++ b/jupyter_server/extension/manager.py @@ -49,8 +49,8 @@ def _valid_metadata(self, proposed): self._module = importlib.import_module(self._module_name) except ModuleNotFoundError: raise ExtensionModuleNotFound( - f"The module '{self._module_name}' could not be found. Are " - "you sure the extension is installed?" + "The module '{}' could not be found. Are you " + "sure the extension is installed?".format(self._module_name) ) # Initialize the app object if it exists. app = self.metadata.get("app") @@ -138,8 +138,8 @@ def _validate_name(self, proposed): self._metadata = get_metadata(name) except ModuleNotFoundError: raise ExtensionModuleNotFound( - f"The module '{name}' could not be found. Are you " - "sure the extension is installed?" + "The module '{name}' could not be found. Are you " + "sure the extension is installed?".format(name=name) ) # Create extension point interfaces for each extension path. for m in self._metadata: diff --git a/pyproject.toml b/pyproject.toml index 4ee2d425dc..e2d7e08323 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,3 @@ [build-system] -requires = ["jupyter_packaging~=0.5.0", "jupyterlab~=2.0", "setuptools>=40.8.0", "wheel"] +requires = ["jupyter_packaging~=0.5.0", "setuptools>=40.8.0", "wheel"] build-backend = "setuptools.build_meta" \ No newline at end of file From 416c4285b751f3501d3b73061f76ebf633d580fd Mon Sep 17 00:00:00 2001 From: Zsailer Date: Wed, 24 Jun 2020 21:23:27 -0700 Subject: [PATCH 11/30] remove commented lines --- examples/simple/setup.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/examples/simple/setup.py b/examples/simple/setup.py index 92be6e984d..97b04bd2fa 100755 --- a/examples/simple/setup.py +++ b/examples/simple/setup.py @@ -11,9 +11,6 @@ def get_data_files(): """ data_files = [ ('etc/jupyter/jupyter_server_config.d', 'etc/jupyter/jupyter_server_config.d/', '*.json'), - # ('etc/jupyter/jupyter_server_config.d', ['etc/jupyter/jupyter_server_config.d/simple_ext1.json']), - # ('etc/jupyter/jupyter_server_config.d', ['etc/jupyter/jupyter_server_config.d/simple_ext2.json']), - # ('etc/jupyter/jupyter_server_config.d', ['etc/jupyter/jupyter_server_config.d/simple_ext11.json']), ] def add_data_files(path): for (dirpath, dirnames, filenames) in os.walk(path): From b3450f5a65d9aec64121beaa7f1ed573f8ecde05 Mon Sep 17 00:00:00 2001 From: Zsailer Date: Wed, 24 Jun 2020 21:36:43 -0700 Subject: [PATCH 12/30] replace ModuleNotFoundError with ImportError for py35 --- jupyter_server/extension/manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jupyter_server/extension/manager.py b/jupyter_server/extension/manager.py index 5676f0fe4b..19cdb50165 100644 --- a/jupyter_server/extension/manager.py +++ b/jupyter_server/extension/manager.py @@ -47,7 +47,7 @@ def _valid_metadata(self, proposed): try: self._module = importlib.import_module(self._module_name) - except ModuleNotFoundError: + except ImportError: raise ExtensionModuleNotFound( "The module '{}' could not be found. Are you " "sure the extension is installed?".format(self._module_name) @@ -136,7 +136,7 @@ def _validate_name(self, proposed): self._extension_points = {} try: self._metadata = get_metadata(name) - except ModuleNotFoundError: + except ImportError: raise ExtensionModuleNotFound( "The module '{name}' could not be found. Are you " "sure the extension is installed?".format(name=name) From e8e0f946a7d86b7a09f1996398dd148fd3627681 Mon Sep 17 00:00:00 2001 From: Zsailer Date: Fri, 26 Jun 2020 14:08:56 -0700 Subject: [PATCH 13/30] attempt to address naming issues in extension loader --- examples/simple/simple_ext1/application.py | 4 ++-- jupyter_server/extension/application.py | 12 ++++++++++-- jupyter_server/extension/manager.py | 2 +- jupyter_server/extension/utils.py | 11 +++++++++++ jupyter_server/serverapp.py | 2 +- 5 files changed, 25 insertions(+), 6 deletions(-) diff --git a/examples/simple/simple_ext1/application.py b/examples/simple/simple_ext1/application.py index 4ddf889096..d183f8b527 100644 --- a/examples/simple/simple_ext1/application.py +++ b/examples/simple/simple_ext1/application.py @@ -10,10 +10,10 @@ class SimpleApp1(ExtensionAppJinjaMixin, ExtensionApp): # The name of the extension. - name = "simple_ext1" + name = "foo" # The url that your extension will serve its homepage. - extension_url = '/simple_ext1/default' + extension_url = '/foo/default' # Should your extension expose other server extensions when launched directly? load_other_extensions = True diff --git a/jupyter_server/extension/application.py b/jupyter_server/extension/application.py index ad7548b5eb..e6887e3c56 100644 --- a/jupyter_server/extension/application.py +++ b/jupyter_server/extension/application.py @@ -153,6 +153,14 @@ class method. This method can be set as a entry_point in # this extension from the CLI, e.g. `jupyter {name}`. name = None + @classmethod + def get_extension_package(cls): + return cls.__module__.split('.')[0] + + @classmethod + def get_extension_point(cls): + return cls.__module__ + # Extension URL sets the default landing page for this extension. extension_url = "/" @@ -370,7 +378,7 @@ def stop(self): def _jupyter_server_config(cls): base_config = { "ServerApp": { - "jpserver_extensions": {cls.name: True}, + "jpserver_extensions": {cls.get_extension_package(): True}, "open_browser": True, "default_url": cls.extension_url } @@ -389,7 +397,7 @@ def _load_jupyter_server_extension(cls, serverapp): extension = extension_manager.extension_points[cls.name].app except KeyError: extension = cls() - extension.link_to_serverapp(serverapp) + extension._link_jupyter_server_extension(serverapp) extension.initialize() return extension diff --git a/jupyter_server/extension/manager.py b/jupyter_server/extension/manager.py index 19cdb50165..89b3df930c 100644 --- a/jupyter_server/extension/manager.py +++ b/jupyter_server/extension/manager.py @@ -49,7 +49,7 @@ def _valid_metadata(self, proposed): self._module = importlib.import_module(self._module_name) except ImportError: raise ExtensionModuleNotFound( - "The module '{}' could not be found. Are you " + "The submodule '{}' could not be found. Are you " "sure the extension is installed?".format(self._module_name) ) # Initialize the app object if it exists. diff --git a/jupyter_server/extension/utils.py b/jupyter_server/extension/utils.py index 32898b06bb..62bf609552 100644 --- a/jupyter_server/extension/utils.py +++ b/jupyter_server/extension/utils.py @@ -137,6 +137,17 @@ class ExtensionModuleNotFound(Exception): pass +class NotAnExtensionApp(Exception): + pass + + +def get_extension_app_pkg(app_cls): + """Get the Python package name + """ + if not isinstance(app_cls, "ExtensionApp"): + raise NotAnExtensionApp("The ") + + def get_loader(obj): """Looks for _load_jupyter_server_extension as an attribute of the object or module. diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index ffad524760..97acbdeb83 100755 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -1511,7 +1511,7 @@ def init_server_extensions(self): # Initialize each extension self.extension_manager = ExtensionManager( - parent=self, + logger=self.log, jpserver_extensions=self.jpserver_extensions ) self.extension_manager.link_extensions() From a86e123368cf2040e29f384aa2d3257d44d2ebf2 Mon Sep 17 00:00:00 2001 From: Zsailer Date: Fri, 26 Jun 2020 14:16:48 -0700 Subject: [PATCH 14/30] pass serverapp to extension manager methods --- jupyter_server/extension/application.py | 52 ++++++++++++------------- jupyter_server/extension/manager.py | 8 ++-- jupyter_server/serverapp.py | 4 +- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/jupyter_server/extension/application.py b/jupyter_server/extension/application.py index e6887e3c56..2e3f87f5fe 100644 --- a/jupyter_server/extension/application.py +++ b/jupyter_server/extension/application.py @@ -292,20 +292,16 @@ def _prepare_templates(self): self.initialize_templates() @classmethod - def initialize_server(cls, argv=[], load_other_extensions=True, **kwargs): - """Creates an instance of ServerApp where this extension is enabled - (superceding disabling found in other config from files). - - This is necessary when launching the ExtensionApp directly from - the `launch_instance` classmethod. - """ - # The ExtensionApp needs to add itself as enabled extension - # to the jpserver_extensions trait, so that the ServerApp - # initializes it. - config = Config(cls._jupyter_server_config()) - serverapp = ServerApp.instance(**kwargs, argv=[], config=config) - serverapp.initialize(argv=argv, find_extensions=load_other_extensions) - return serverapp + def _jupyter_server_config(cls): + base_config = { + "ServerApp": { + "jpserver_extensions": {cls.get_extension_package(): True}, + "open_browser": True, + "default_url": cls.extension_url + } + } + base_config["ServerApp"].update(cls.server_config) + return base_config def _link_jupyter_server_extension(self, serverapp): """Link the ExtensionApp to an initialized ServerApp. @@ -335,6 +331,22 @@ def _link_jupyter_server_extension(self, serverapp): # i.e. ServerApp traits <--- ExtensionApp config self.serverapp.update_config(self.config) + @classmethod + def initialize_server(cls, argv=[], load_other_extensions=True, **kwargs): + """Creates an instance of ServerApp where this extension is enabled + (superceding disabling found in other config from files). + + This is necessary when launching the ExtensionApp directly from + the `launch_instance` classmethod. + """ + # The ExtensionApp needs to add itself as enabled extension + # to the jpserver_extensions trait, so that the ServerApp + # initializes it. + config = Config(cls._jupyter_server_config()) + serverapp = ServerApp.instance(**kwargs, argv=[], config=config) + serverapp.initialize(argv=argv, find_extensions=load_other_extensions) + return serverapp + def initialize(self): """Initialize the extension app. The corresponding server app and webapp should already @@ -374,18 +386,6 @@ def stop(self): self.serverapp.stop() self.serverapp.clear_instance() - @classmethod - def _jupyter_server_config(cls): - base_config = { - "ServerApp": { - "jpserver_extensions": {cls.get_extension_package(): True}, - "open_browser": True, - "default_url": cls.extension_url - } - } - base_config["ServerApp"].update(cls.server_config) - return base_config - @classmethod def _load_jupyter_server_extension(cls, serverapp): """Initialize and configure this extension, then add the extension's diff --git a/jupyter_server/extension/manager.py b/jupyter_server/extension/manager.py index 89b3df930c..245ec30752 100644 --- a/jupyter_server/extension/manager.py +++ b/jupyter_server/extension/manager.py @@ -215,7 +215,7 @@ def extension_points(self): points.update(ext.extension_points) return points - def link_extensions(self): + def link_extensions(self, serverapp): """Link all enabled extensions to an instance of ServerApp """ @@ -223,11 +223,11 @@ def link_extensions(self): # order. for name, ext in sorted(self.extension_points.items()): try: - ext.link(self.parent) + ext.link(serverapp) except Exception as e: self.log.warning(e) - def load_extensions(self): + def load_extensions(self, serverapp): """Load all enabled extensions and append them to the parent ServerApp. """ @@ -235,7 +235,7 @@ def load_extensions(self): # order. for name, ext in sorted(self.extension_points.items()): try: - ext.load(self.parent) + ext.load(serverapp) except Exception as e: self.log.warning(e) diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index 97acbdeb83..b0c76e133a 100755 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -1514,7 +1514,7 @@ def init_server_extensions(self): logger=self.log, jpserver_extensions=self.jpserver_extensions ) - self.extension_manager.link_extensions() + self.extension_manager.link_extensions(self) def load_server_extensions(self): """Load any extensions specified by config. @@ -1524,7 +1524,7 @@ def load_server_extensions(self): The extension API is experimental, and may change in future releases. """ - self.extension_manager.load_extensions() + self.extension_manager.load_extensions(self) def init_mime_overrides(self): # On some Windows machines, an application has registered incorrect From 9027bafb1c7b0ff3c9ec604fe6a64773b901452a Mon Sep 17 00:00:00 2001 From: Zsailer Date: Fri, 26 Jun 2020 14:21:22 -0700 Subject: [PATCH 15/30] change example back to orignal name --- examples/simple/simple_ext1/application.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/simple/simple_ext1/application.py b/examples/simple/simple_ext1/application.py index d183f8b527..4ddf889096 100644 --- a/examples/simple/simple_ext1/application.py +++ b/examples/simple/simple_ext1/application.py @@ -10,10 +10,10 @@ class SimpleApp1(ExtensionAppJinjaMixin, ExtensionApp): # The name of the extension. - name = "foo" + name = "simple_ext1" # The url that your extension will serve its homepage. - extension_url = '/foo/default' + extension_url = '/simple_ext1/default' # Should your extension expose other server extensions when launched directly? load_other_extensions = True From a790cf44548e3861e4fc2dacffd2ea0e23cb1cac Mon Sep 17 00:00:00 2001 From: Zsailer Date: Fri, 26 Jun 2020 14:28:36 -0700 Subject: [PATCH 16/30] enable extensionapps to configure the server when launched directly --- jupyter_server/extension/application.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/jupyter_server/extension/application.py b/jupyter_server/extension/application.py index 2e3f87f5fe..5e0199079b 100644 --- a/jupyter_server/extension/application.py +++ b/jupyter_server/extension/application.py @@ -144,8 +144,12 @@ class method. This method can be set as a entry_point in # side-by-side when launched directly. load_other_extensions = True - # Pass se - server_config = {} + # A useful class property that subclasses can override to + # configure the underlying Jupyter Server when this extension + # is launched directly (using its `launch_instance` method). + serverapp_config = { + "open_browser": True + } # The extension name used to name the jupyter config # file, jupyter_{name}_config. @@ -296,11 +300,10 @@ def _jupyter_server_config(cls): base_config = { "ServerApp": { "jpserver_extensions": {cls.get_extension_package(): True}, - "open_browser": True, "default_url": cls.extension_url } } - base_config["ServerApp"].update(cls.server_config) + base_config["ServerApp"].update(cls.serverapp_config) return base_config def _link_jupyter_server_extension(self, serverapp): From feb052bbc28cc2723a478b7b03841f008cc1e320 Mon Sep 17 00:00:00 2001 From: Zsailer Date: Thu, 9 Jul 2020 13:56:53 -0700 Subject: [PATCH 17/30] track linking of extension to prevent duplicate loads --- jupyter_server/extension/application.py | 9 +- jupyter_server/extension/manager.py | 144 +++++++++++++----------- jupyter_server/serverapp.py | 17 ++- 3 files changed, 92 insertions(+), 78 deletions(-) diff --git a/jupyter_server/extension/application.py b/jupyter_server/extension/application.py index 5e0199079b..f8ead2001f 100644 --- a/jupyter_server/extension/application.py +++ b/jupyter_server/extension/application.py @@ -173,6 +173,9 @@ def get_extension_point(cls): ServerApp, ] + # A ServerApp is not defined yet, but will be initialized below. + serverapp = None + @property def static_url_prefix(self): return "/static/{name}/".format( @@ -361,7 +364,7 @@ def initialize(self): 3) Points Tornado Webapp to templates and static assets. """ - if not hasattr(self, 'serverapp'): + if not self.serverapp: msg = ( "This extension has no attribute `serverapp`. " "Try calling `.link_to_serverapp()` before calling " @@ -397,7 +400,9 @@ def _load_jupyter_server_extension(cls, serverapp): extension_manager = serverapp.extension_manager try: # Get loaded extension from serverapp. - extension = extension_manager.extension_points[cls.name].app + pkg = extension_manager.enabled_extensions[cls.name] + point = pkg.extension_points[cls.name] + extension = point.app except KeyError: extension = cls() extension._link_jupyter_server_extension(serverapp) diff --git a/jupyter_server/extension/manager.py b/jupyter_server/extension/manager.py index 245ec30752..9d452282d1 100644 --- a/jupyter_server/extension/manager.py +++ b/jupyter_server/extension/manager.py @@ -21,15 +21,6 @@ class ExtensionPoint(HasTraits): """A simple API for connecting to a Jupyter Server extension point defined by metadata and importable from a Python package. - - Usage: - - metadata = { - "module": "extension_module", - "": - } - - point = ExtensionPoint(metadata) """ metadata = Dict() @@ -58,6 +49,10 @@ def _valid_metadata(self, proposed): metadata["app"] = app() return metadata + @property + def linked(self): + return self._linked + @property def app(self): """If the metadata includes an `app` field""" @@ -103,7 +98,10 @@ def link(self, serverapp): # Otherwise return a dummy function. lambda serverapp: None ) - return linker(serverapp) + # Capture output to return + out = linker(serverapp) + # Store that this extension has been linked + return out def load(self, serverapp): """Load the extension in a Jupyter ServerApp object. @@ -130,6 +128,9 @@ class ExtensionPackage(HasTraits): """ name = Unicode(help="Name of the an importable Python package.") + # A dictionary that stores whether the extension point has been linked. + _linked_points = {} + @validate("name") def _validate_name(self, proposed): name = proposed['value'] @@ -157,85 +158,96 @@ def extension_points(self): """A dictionary of extension points.""" return self._extension_points + def link_point(self, point_name, serverapp): + linked = self._linked_points.get(point_name, False) + if not linked: + point = self.extension_points[point_name] + point.link(serverapp) + + def load_point(self, point_name, serverapp): + point = self.extension_points[point_name] + point.load(serverapp) + + def link_all_points(self, serverapp): + for point_name in self.extension_points: + self.link_point(point_name, serverapp) + + def load_all_points(self, serverapp): + for point_name in self.extension_points: + self.load_point(point_name, serverapp) + class ExtensionManager(LoggingConfigurable): """High level interface for findind, validating, linking, loading, and managing Jupyter Server extensions. Usage: - m = ExtensionManager(jpserver_extensions=extensions) """ - parent = Instance( - klass="jupyter_server.serverapp.ServerApp", - allow_none=True - ) - - jpserver_extensions = Dict( - help=( - "A dictionary with extension package names " - "as keys and booleans to enable as values." - ) - ) - - @default('jpserver_extensions') - def _default_jpserver_extensions(self): - return self.parent.jpserver_extensions - - @validate('jpserver_extensions') - def _validate_jpserver_extensions(self, proposed): - jpserver_extensions = proposed['value'] - self._extensions = {} - # Iterate over dictionary items and validate that - # we can interface with each extension. If the extension - # fails to interface, throw a warning through the logger - # interface. - for package_name, enabled in jpserver_extensions.items(): - if enabled: - try: - self._extensions[package_name] = ExtensionPackage( - name=package_name - ) - # Raise a warning if the extension cannot be loaded. - except Exception as e: - self.log.warning(e) - return jpserver_extensions + # The `enabled_extensions` attribute provides a dictionary + # with extension names mapped to their ExtensionPackage interface + # (see above). This manager simplifies the interaction between the + # ServerApp and the extensions being appended. + _enabled_extensions = {} + # The `_linked_extensions` attribute tracks when each extension + # has been successfully linked to a ServerApp. This helps prevent + # extensions from being re-linked recursively unintentionally if another + # extension attempts to link extensions again. + _linked_extensions = {} @property - def extensions(self): + def enabled_extensions(self): """Dictionary with extension package names as keys and an ExtensionPackage objects as values. """ - return self._extensions + return dict(sorted(self._enabled_extensions.items())) - @property - def extension_points(self): - points = {} - for ext in self.extensions.values(): - points.update(ext.extension_points) - return points + def from_jpserver_extensions(self, jpserver_extensions): + """Add extensions from 'jpserver_extensions'-like dictionary.""" + for name, enabled in jpserver_extensions.items(): + if enabled: + self.add_extension(name) + + def add_extension(self, extension_name): + try: + extpkg = ExtensionPackage(name=extension_name) + self._enabled_extensions[extension_name] = extpkg + # Raise a warning if the extension cannot be loaded. + except Exception as e: + self.log.warning(e) + + def link_extension(self, name, serverapp): + linked = self._linked_extensions.get(name, False) + extension = self.enabled_extensions[name] + if not linked: + try: + extension.link_all_points(serverapp) + self.log.debug("The '{}' extension was successfully linked.".format(name)) + except Exception as e: + self.log.warning(e) + + def load_extension(self, name, serverapp): + extension = self.enabled_extensions.get(name) + try: + extension.load_all_points(serverapp) + except Exception as e: + self.log.warning(e) - def link_extensions(self, serverapp): + def link_all_extensions(self, serverapp): """Link all enabled extensions - to an instance of ServerApp + to an instance of ServerApp """ # Sort the extension names to enforce deterministic linking # order. - for name, ext in sorted(self.extension_points.items()): - try: - ext.link(serverapp) - except Exception as e: - self.log.warning(e) + for name in self.enabled_extensions: + self.link_extension(name, serverapp) - def load_extensions(self, serverapp): + def load_all_extensions(self, serverapp): """Load all enabled extensions and append them to the parent ServerApp. """ # Sort the extension names to enforce deterministic loading # order. - for name, ext in sorted(self.extension_points.items()): - try: - ext.load(serverapp) - except Exception as e: - self.log.warning(e) + for name in self.enabled_extensions: + self.load_extension(name, serverapp) diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index b0c76e133a..34bd0214cc 100755 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -1474,9 +1474,9 @@ def init_components(self): def find_server_extensions(self): """ - Searches Jupyter paths for jpserver_extensions and captures - metadata for all enabled extensions. + Searches Jupyter paths for jpserver_extensions. """ + # Walk through all config files looking for jpserver_extensions. # # Each extension will likely have a JSON config file enabling itself in @@ -1508,13 +1508,10 @@ def init_server_extensions(self): this instance will inherit the ServerApp's config object and load its own config. """ - - # Initialize each extension - self.extension_manager = ExtensionManager( - logger=self.log, - jpserver_extensions=self.jpserver_extensions - ) - self.extension_manager.link_extensions(self) + # Create an instance of the ExtensionManager. + self.extension_manager = ExtensionManager(logger=self.log) + self.extension_manager.from_jpserver_extensions(self.jpserver_extensions) + self.extension_manager.link_all_extensions(self) def load_server_extensions(self): """Load any extensions specified by config. @@ -1524,7 +1521,7 @@ def load_server_extensions(self): The extension API is experimental, and may change in future releases. """ - self.extension_manager.load_extensions(self) + self.extension_manager.load_all_extensions(self) def init_mime_overrides(self): # On some Windows machines, an application has registered incorrect From 8e8f8a4c5685e42444ae4f608fa80dc5a0f357c1 Mon Sep 17 00:00:00 2001 From: Zsailer Date: Thu, 16 Jul 2020 20:33:29 -0700 Subject: [PATCH 18/30] minor changes to logging in extension manager --- jupyter_server/extension/manager.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jupyter_server/extension/manager.py b/jupyter_server/extension/manager.py index 9d452282d1..6f8947c4c9 100644 --- a/jupyter_server/extension/manager.py +++ b/jupyter_server/extension/manager.py @@ -98,6 +98,7 @@ def link(self, serverapp): # Otherwise return a dummy function. lambda serverapp: None ) + # Capture output to return out = linker(serverapp) # Store that this extension has been linked @@ -222,7 +223,7 @@ def link_extension(self, name, serverapp): if not linked: try: extension.link_all_points(serverapp) - self.log.debug("The '{}' extension was successfully linked.".format(name)) + self.log.info("{name} | extension was successfully linked.".format(name=name)) except Exception as e: self.log.warning(e) @@ -230,6 +231,7 @@ def load_extension(self, name, serverapp): extension = self.enabled_extensions.get(name) try: extension.load_all_points(serverapp) + self.log.info("{name} | extension was successfully loaded.".format(name=name)) except Exception as e: self.log.warning(e) From 058db714feca7d145e2d97ec2e09a41483c5cf6a Mon Sep 17 00:00:00 2001 From: Zsailer Date: Thu, 16 Jul 2020 20:33:44 -0700 Subject: [PATCH 19/30] remove custom handler from server --- jupyter_server/serverapp.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index 34bd0214cc..6b1bb09dfa 100755 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -320,12 +320,6 @@ def init_handlers(self, default_services, settings): handlers[j] = (gwh[0], gwh[1]) break - handlers.append( - (r"/custom/(.*)", FileFindHandler, { - 'path': settings['static_custom_path'], - 'no_cache_paths': ['/'], # don't cache anything in custom - }) - ) # register base handlers last handlers.extend(load_handlers('jupyter_server.base.handlers')) From 9ac399f2523036e69860ad559401f8a45f0d6891 Mon Sep 17 00:00:00 2001 From: Zsailer Date: Thu, 16 Jul 2020 20:36:58 -0700 Subject: [PATCH 20/30] remove unused jupyter paths logic --- jupyter_server/extension/utils.py | 124 ------------------------------ 1 file changed, 124 deletions(-) diff --git a/jupyter_server/extension/utils.py b/jupyter_server/extension/utils.py index 62bf609552..d06f5903f3 100644 --- a/jupyter_server/extension/utils.py +++ b/jupyter_server/extension/utils.py @@ -1,129 +1,5 @@ -import pathlib import importlib -from jupyter_core.paths import jupyter_config_path -from traitlets.config.loader import ( - JSONFileConfigLoader -) - - -def configd_path( - config_dir=None, - configd_prefix="jupyter_server", -): - # Build directory name for `config.d` directory. - configd = "_".join([configd_prefix, "config.d"]) - if not config_dir: - config_dir = jupyter_config_path() - # Leverage pathlib for path management. - return [pathlib.Path(path).joinpath(configd) for path in config_dir] - - -def configd_files( - config_dir=None, - configd_prefix="jupyter_server", -): - """Lists (only) JSON files found in a Jupyter config.d folder. - """ - paths = configd_path( - config_dir=config_dir, - configd_prefix=configd_prefix - ) - files = [] - for path in paths: - json_files = path.glob("*.json") - files.extend(json_files) - return files - - -def enabled(name, server_config): - """Given a server config object, return True if the extension - is explicitly enabled in the config. - """ - enabled = ( - server_config - .get("ServerApp", {}) - .get("jpserver_extensions", {}) - .get(name, False) - ) - return enabled - - -def find_extension_in_configd( - name, - config_dir=None, - configd_prefix="jupyter_server", -): - """Search through all config.d files and return the - JSON Path for this named extension. If the extension - is not found, return None - """ - files = configd_files( - config_dir=config_dir, - configd_prefix=configd_prefix, - ) - for f in files: - if name == f.stem: - return f - - -def configd_enabled( - name, - config_dir=None, - configd_prefix="jupyter_server", -): - """Check if the named extension is enabled somewhere in - a config.d folder. - """ - config_file = find_extension_in_configd( - name, - config_dir=config_dir, - configd_prefix=configd_prefix, - ) - if config_file: - c = JSONFileConfigLoader( - filename=str(config_file.name), - path=str(config_file.parent) - ) - config = c.load_config() - return enabled(name, config) - else: - return False - - -def list_extensions_in_configd( - configd_prefix="jupyter_server", - config_paths=None -): - """Get a dictionary of all jpserver_extensions found in the - config directories list. - - Parameters - ---------- - config_paths : list - List of config directories to search for the - `jupyter_server_config.d` directory. - """ - - # Build directory name for `config.d` directory. - configd = "_".join([configd_prefix, "config.d"]) - - if not config_paths: - config_paths = jupyter_config_path() - - # Leverage pathlib for path management. - config_paths = [pathlib.Path(p) for p in config_paths] - - extensions = [] - for path in config_paths: - json_files = path.joinpath(configd).glob("*.json") - for file in json_files: - # The extension name is the file name (minus file suffix) - extension_name = file.stem - extensions.append(extension_name) - - return extensions - class ExtensionLoadingError(Exception): pass From 9ac50db173b40c4c13aacb6ad300a3da15c5fbde Mon Sep 17 00:00:00 2001 From: Zsailer Date: Wed, 22 Jul 2020 21:38:47 -0700 Subject: [PATCH 21/30] update tests with changes in extension manager --- jupyter_server/extension/application.py | 3 +- jupyter_server/extension/manager.py | 28 +++++++---- tests/extension/mockextensions/app.py | 3 +- tests/extension/test_app.py | 32 ++++++++----- tests/extension/test_manager.py | 15 ++---- tests/extension/test_utils.py | 62 +------------------------ 6 files changed, 46 insertions(+), 97 deletions(-) diff --git a/jupyter_server/extension/application.py b/jupyter_server/extension/application.py index f8ead2001f..3cc916695a 100644 --- a/jupyter_server/extension/application.py +++ b/jupyter_server/extension/application.py @@ -400,8 +400,7 @@ def _load_jupyter_server_extension(cls, serverapp): extension_manager = serverapp.extension_manager try: # Get loaded extension from serverapp. - pkg = extension_manager.enabled_extensions[cls.name] - point = pkg.extension_points[cls.name] + point = extension_manager.extension_points[cls.name] extension = point.app except KeyError: extension = cls() diff --git a/jupyter_server/extension/manager.py b/jupyter_server/extension/manager.py index 6f8947c4c9..e536d7e13a 100644 --- a/jupyter_server/extension/manager.py +++ b/jupyter_server/extension/manager.py @@ -5,8 +5,6 @@ HasTraits, Dict, Unicode, - Instance, - default, validate ) @@ -23,6 +21,7 @@ class ExtensionPoint(HasTraits): point defined by metadata and importable from a Python package. """ metadata = Dict() + _app = None @validate('metadata') def _valid_metadata(self, proposed): @@ -43,10 +42,11 @@ def _valid_metadata(self, proposed): "The submodule '{}' could not be found. Are you " "sure the extension is installed?".format(self._module_name) ) - # Initialize the app object if it exists. - app = self.metadata.get("app") - if app: - metadata["app"] = app() + # If the metadata includes an ExtensionApp, create an instance. + try: + self._app = metadata.get("app")() + except TypeError: + pass return metadata @property @@ -56,7 +56,7 @@ def linked(self): @property def app(self): """If the metadata includes an `app` field""" - return self.metadata.get("app") + return self._app @property def module_name(self): @@ -129,7 +129,7 @@ class ExtensionPackage(HasTraits): """ name = Unicode(help="Name of the an importable Python package.") - # A dictionary that stores whether the extension point has been linked. + # Store extension points that have been linked. _linked_points = {} @validate("name") @@ -186,7 +186,7 @@ class ExtensionManager(LoggingConfigurable): m = ExtensionManager(jpserver_extensions=extensions) """ # The `enabled_extensions` attribute provides a dictionary - # with extension names mapped to their ExtensionPackage interface + # with extension (package) names mapped to their ExtensionPackage interface # (see above). This manager simplifies the interaction between the # ServerApp and the extensions being appended. _enabled_extensions = {} @@ -203,6 +203,16 @@ def enabled_extensions(self): """ return dict(sorted(self._enabled_extensions.items())) + @property + def extension_points(self): + extensions = self.enabled_extensions + return { + name: point + for value in extensions.values() + for name, point in value.extension_points.items() + } + + def from_jpserver_extensions(self, jpserver_extensions): """Add extensions from 'jpserver_extensions'-like dictionary.""" for name, enabled in jpserver_extensions.items(): diff --git a/tests/extension/mockextensions/app.py b/tests/extension/mockextensions/app.py index fac5a5f957..6978a7289c 100644 --- a/tests/extension/mockextensions/app.py +++ b/tests/extension/mockextensions/app.py @@ -37,5 +37,4 @@ class MockExtensionApp(ExtensionAppJinjaMixin, ExtensionApp): def initialize_handlers(self): self.handlers.append(('/mock', MockExtensionHandler)) self.handlers.append(('/mock_template', MockExtensionTemplateHandler)) - self.loaded = True - + self.loaded = True \ No newline at end of file diff --git a/tests/extension/test_app.py b/tests/extension/test_app.py index 9e7e068882..f268aa8939 100644 --- a/tests/extension/test_app.py +++ b/tests/extension/test_app.py @@ -1,6 +1,11 @@ import pytest from jupyter_server.serverapp import ServerApp +# Use ServerApps environment because it monkeypatches +# jupyter_core.paths and provides a config directory +# that's not cross contaminating the user config directory. +pytestmark = pytest.mark.usefixtures("environ") + @pytest.fixture def server_config(request, template_dir): @@ -21,14 +26,19 @@ def server_config(request, template_dir): @pytest.fixture -def mock_extension(extension_manager): - return extension_manager.extension_points["mockextension"].app +def mock_extension(serverapp, extension_manager): + name = "tests.extension.mockextensions" + pkg = extension_manager.enabled_extensions[name] + point = pkg.extension_points["mockextension"] + app = point.app + return app -def test_initialize(mock_extension, template_dir): +def test_initialize(serverapp, mock_extension, template_dir): # Check that settings and handlers were added to the mock extension. assert isinstance(mock_extension.serverapp, ServerApp) assert len(mock_extension.handlers) > 0 + assert mock_extension.loaded assert mock_extension.template_paths == [str(template_dir)] @@ -46,22 +56,20 @@ def test_instance_creation_with_argv( serverapp, trait_name, trait_value, - extension_manager + mock_extension, ): - extension = extension_manager.extension_points['mockextension'].app - assert getattr(extension, trait_name) == trait_value + assert getattr(mock_extension, trait_name) == trait_value def test_extensionapp_load_config_file( extension_environ, config_file, - extension_manager, + mock_extension, serverapp, ): - extension = extension_manager.extension_points['mockextension'].app # Assert default config_file_paths is the same in the app and extension. - assert extension.config_file_paths == serverapp.config_file_paths - assert extension.config_dir == serverapp.config_dir - assert extension.config_file_name == 'jupyter_mockextension_config' + assert mock_extension.config_file_paths == serverapp.config_file_paths + assert mock_extension.config_dir == serverapp.config_dir + assert mock_extension.config_file_name == 'jupyter_mockextension_config' # Assert that the trait is updated by config file - assert extension.mock_trait == 'config from file' + assert mock_extension.mock_trait == 'config from file' diff --git a/tests/extension/test_manager.py b/tests/extension/test_manager.py index 9866c2b15b..bfa349ec88 100644 --- a/tests/extension/test_manager.py +++ b/tests/extension/test_manager.py @@ -62,18 +62,11 @@ def test_extension_package_notfound_error(): def test_extension_manager_api(): - # Import mock extension metadata - from .mockextensions import _jupyter_server_extension_paths - - # Testing the first path (which is an extension app). - metadata_list = _jupyter_server_extension_paths() - jpserver_extensions = { "tests.extension.mockextensions": True } - manager = ExtensionManager(jpserver_extensions=jpserver_extensions) - assert len(manager.extensions) == 1 - assert len(manager.extension_points) == len(metadata_list) - assert "mockextension" in manager.extension_points - assert "tests.extension.mockextensions.mock1" in manager.extension_points + manager = ExtensionManager() + manager.from_jpserver_extensions(jpserver_extensions) + assert len(manager.enabled_extensions) == 1 + assert "tests.extension.mockextensions" in manager.enabled_extensions diff --git a/tests/extension/test_utils.py b/tests/extension/test_utils.py index 5e24d88b99..e12d665e99 100644 --- a/tests/extension/test_utils.py +++ b/tests/extension/test_utils.py @@ -1,65 +1,5 @@ import pytest -from jupyter_server.extension.utils import ( - list_extensions_in_configd, - configd_enabled, - validate_extension -) - -# Use ServerApps environment because it monkeypatches -# jupyter_core.paths and provides a config directory -# that's not cross contaminating the user config directory. -pytestmark = pytest.mark.usefixtures("environ") - - -@pytest.fixture -def configd(env_config_path): - """A pathlib.Path object that acts like a jupyter_server_config.d folder.""" - configd = env_config_path.joinpath('jupyter_server_config.d') - configd.mkdir() - return configd - - -ext1_json_config = """\ -{ - "ServerApp": { - "jpserver_extensions": { - "ext1_config": true - } - } -} -""" - -@pytest.fixture -def ext1_config(configd): - config = configd.joinpath("ext1_config.json") - config.write_text(ext1_json_config) - - -ext2_json_config = """\ -{ - "ServerApp": { - "jpserver_extensions": { - "ext2_config": false - } - } -} -""" - - -@pytest.fixture -def ext2_config(configd): - config = configd.joinpath("ext2_config.json") - config.write_text(ext2_json_config) - - -def test_list_extension_from_configd(ext1_config, ext2_config): - extensions = list_extensions_in_configd() - assert "ext2_config" in extensions - assert "ext1_config" in extensions - - -def test_config_enabled(ext1_config): - assert configd_enabled("ext1_config") +from jupyter_server.extension.utils import validate_extension def test_validate_extension(): From dadc124f03887576f897613c6413a69b96eea0d9 Mon Sep 17 00:00:00 2001 From: Zsailer Date: Sun, 26 Jul 2020 20:45:06 -0700 Subject: [PATCH 22/30] =?UTF-8?q?fixed=20subtle=20bug=20in=20extension=20m?= =?UTF-8?q?anager=20classes=E2=80=94accidently=20used=20class=20attributes?= =?UTF-8?q?=20instead=20of=20instance=20attributes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- jupyter_server/extension/application.py | 2 +- jupyter_server/extension/manager.py | 45 +++++++++++++++---------- jupyter_server/pytest_plugin.py | 18 +++++----- jupyter_server/serverapp.py | 3 +- tests/conftest.py | 2 +- tests/extension/conftest.py | 2 ++ tests/extension/test_app.py | 15 +++------ tests/extension/test_manager.py | 5 +++ tests/extension/test_serverextension.py | 6 ++++ tests/extension/test_utils.py | 7 ++++ tests/services/kernels/test_api.py | 24 ------------- tests/services/kernels/test_config.py | 25 ++++++++++++++ 12 files changed, 90 insertions(+), 64 deletions(-) diff --git a/jupyter_server/extension/application.py b/jupyter_server/extension/application.py index 3cc916695a..0307f53674 100644 --- a/jupyter_server/extension/application.py +++ b/jupyter_server/extension/application.py @@ -322,7 +322,7 @@ def _link_jupyter_server_extension(self, serverapp): # Load config from an ExtensionApp's config files. self.load_config_file() # ServerApp's config might have picked up - # CLI config for the ExtensionApp. We call + # config for the ExtensionApp. We call # update_config to update ExtensionApp's # traits with these values found in ServerApp's # config. diff --git a/jupyter_server/extension/manager.py b/jupyter_server/extension/manager.py index e536d7e13a..f49081dfa9 100644 --- a/jupyter_server/extension/manager.py +++ b/jupyter_server/extension/manager.py @@ -21,7 +21,11 @@ class ExtensionPoint(HasTraits): point defined by metadata and importable from a Python package. """ metadata = Dict() - _app = None + + def __init__(self, *args, **kwargs): + # Store extension points that have been linked. + self._app = None + super().__init__(*args, **kwargs) @validate('metadata') def _valid_metadata(self, proposed): @@ -43,10 +47,8 @@ def _valid_metadata(self, proposed): "sure the extension is installed?".format(self._module_name) ) # If the metadata includes an ExtensionApp, create an instance. - try: - self._app = metadata.get("app")() - except TypeError: - pass + if 'app' in metadata: + self._app = metadata["app"]() return metadata @property @@ -129,7 +131,11 @@ class ExtensionPackage(HasTraits): """ name = Unicode(help="Name of the an importable Python package.") - # Store extension points that have been linked. + def __init__(self, *args, **kwargs): + # Store extension points that have been linked. + self._linked_points = {} + super().__init__(*args, **kwargs) + _linked_points = {} @validate("name") @@ -185,23 +191,27 @@ class ExtensionManager(LoggingConfigurable): Usage: m = ExtensionManager(jpserver_extensions=extensions) """ - # The `enabled_extensions` attribute provides a dictionary - # with extension (package) names mapped to their ExtensionPackage interface - # (see above). This manager simplifies the interaction between the - # ServerApp and the extensions being appended. - _enabled_extensions = {} - # The `_linked_extensions` attribute tracks when each extension - # has been successfully linked to a ServerApp. This helps prevent - # extensions from being re-linked recursively unintentionally if another - # extension attempts to link extensions again. - _linked_extensions = {} + def __init__(self, *args, **kwargs): + # The `enabled_extensions` attribute provides a dictionary + # with extension (package) names mapped to their ExtensionPackage interface + # (see above). This manager simplifies the interaction between the + # ServerApp and the extensions being appended. + self._enabled_extensions = {} + # The `_linked_extensions` attribute tracks when each extension + # has been successfully linked to a ServerApp. This helps prevent + # extensions from being re-linked recursively unintentionally if another + # extension attempts to link extensions again. + self._linked_extensions = {} + super().__init__(*args, **kwargs) @property def enabled_extensions(self): """Dictionary with extension package names as keys and an ExtensionPackage objects as values. """ - return dict(sorted(self._enabled_extensions.items())) + # Sort enabled extensions before returning + out = sorted(self._enabled_extensions.items()) + return dict(out) @property def extension_points(self): @@ -212,7 +222,6 @@ def extension_points(self): for name, point in value.extension_points.items() } - def from_jpserver_extensions(self, jpserver_extensions): """Add extensions from 'jpserver_extensions'-like dictionary.""" for name, enabled in jpserver_extensions.items(): diff --git a/jupyter_server/pytest_plugin.py b/jupyter_server/pytest_plugin.py index 7bc6842b50..05c2000c2f 100644 --- a/jupyter_server/pytest_plugin.py +++ b/jupyter_server/pytest_plugin.py @@ -111,16 +111,18 @@ def extension_environ(env_config_path, monkeypatch): @pytest.fixture(scope='function') def configurable_serverapp( environ, + server_config, + argv, http_port, tmp_path, root_dir, io_loop, - server_config, - **kwargs ): - def serverapp( + ServerApp.clear_instance() + + def _configurable_serverapp( config=server_config, - argv=[], + argv=argv, environ=environ, http_port=http_port, tmp_path=tmp_path, @@ -144,6 +146,7 @@ def serverapp( token=token, **kwargs ) + app.init_signal = lambda: None app.log.propagate = True app.log.handlers = [] @@ -155,12 +158,11 @@ def serverapp( app.start_app() return app - yield serverapp - ServerApp.clear_instance() + return _configurable_serverapp -@pytest.fixture -def serverapp(configurable_serverapp, server_config, argv): +@pytest.fixture(scope="function") +def serverapp(server_config, argv, configurable_serverapp): app = configurable_serverapp(config=server_config, argv=argv) yield app app.remove_server_info_file() diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index 6b1bb09dfa..a4f05803bd 100755 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -1503,7 +1503,7 @@ def init_server_extensions(self): and load its own config. """ # Create an instance of the ExtensionManager. - self.extension_manager = ExtensionManager(logger=self.log) + self.extension_manager = ExtensionManager(log=self.log) self.extension_manager.from_jpserver_extensions(self.jpserver_extensions) self.extension_manager.link_all_extensions(self) @@ -1682,6 +1682,7 @@ def initialize(self, argv=None, find_extensions=True, new_httpserver=True): self.init_mime_overrides() self.init_shutdown_no_activity() + def cleanup_kernels(self): """Shutdown all kernels. diff --git a/tests/conftest.py b/tests/conftest.py index 6369f8eac3..9d694425b2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1 +1 @@ -pytest_plugins = ['pytest_jupyter_server'] +pytest_plugins = ['pytest_jupyter_server'] \ No newline at end of file diff --git a/tests/extension/conftest.py b/tests/extension/conftest.py index ef5d04488e..0be56e2729 100644 --- a/tests/extension/conftest.py +++ b/tests/extension/conftest.py @@ -1,5 +1,6 @@ import pytest + mock_html = """ @@ -23,6 +24,7 @@ """ + @pytest.fixture def mock_template(template_dir): index = template_dir.joinpath('index.html') diff --git a/tests/extension/test_app.py b/tests/extension/test_app.py index f268aa8939..7303609e6f 100644 --- a/tests/extension/test_app.py +++ b/tests/extension/test_app.py @@ -1,14 +1,9 @@ import pytest from jupyter_server.serverapp import ServerApp -# Use ServerApps environment because it monkeypatches -# jupyter_core.paths and provides a config directory -# that's not cross contaminating the user config directory. -pytestmark = pytest.mark.usefixtures("environ") - @pytest.fixture -def server_config(request, template_dir): +def server_config(template_dir): config = { "ServerApp": { "jpserver_extensions": { @@ -26,7 +21,7 @@ def server_config(request, template_dir): @pytest.fixture -def mock_extension(serverapp, extension_manager): +def mock_extension(extension_manager): name = "tests.extension.mockextensions" pkg = extension_manager.enabled_extensions[name] point = pkg.extension_points["mockextension"] @@ -34,7 +29,7 @@ def mock_extension(serverapp, extension_manager): return app -def test_initialize(serverapp, mock_extension, template_dir): +def test_initialize(serverapp, template_dir, mock_extension): # Check that settings and handlers were added to the mock extension. assert isinstance(mock_extension.serverapp, ServerApp) assert len(mock_extension.handlers) > 0 @@ -53,7 +48,6 @@ def test_initialize(serverapp, mock_extension, template_dir): ) ) def test_instance_creation_with_argv( - serverapp, trait_name, trait_value, mock_extension, @@ -62,10 +56,9 @@ def test_instance_creation_with_argv( def test_extensionapp_load_config_file( - extension_environ, config_file, - mock_extension, serverapp, + mock_extension, ): # Assert default config_file_paths is the same in the app and extension. assert mock_extension.config_file_paths == serverapp.config_file_paths diff --git a/tests/extension/test_manager.py b/tests/extension/test_manager.py index bfa349ec88..8ad38166f3 100644 --- a/tests/extension/test_manager.py +++ b/tests/extension/test_manager.py @@ -7,6 +7,11 @@ ExtensionModuleNotFound ) +# Use ServerApps environment because it monkeypatches +# jupyter_core.paths and provides a config directory +# that's not cross contaminating the user config directory. +pytestmark = pytest.mark.usefixtures("environ") + def test_extension_point_api(): # Import mock extension metadata diff --git a/tests/extension/test_serverextension.py b/tests/extension/test_serverextension.py index 009d5b059b..c18764c1dc 100644 --- a/tests/extension/test_serverextension.py +++ b/tests/extension/test_serverextension.py @@ -9,6 +9,12 @@ from jupyter_server.config_manager import BaseJSONConfigManager +# Use ServerApps environment because it monkeypatches +# jupyter_core.paths and provides a config directory +# that's not cross contaminating the user config directory. +pytestmark = pytest.mark.usefixtures("environ") + + def test_help_output(): check_help_all_output('jupyter_server.extension.serverextension') check_help_all_output('jupyter_server.extension.serverextension', ['enable']) diff --git a/tests/extension/test_utils.py b/tests/extension/test_utils.py index e12d665e99..54f48e17e3 100644 --- a/tests/extension/test_utils.py +++ b/tests/extension/test_utils.py @@ -2,6 +2,13 @@ from jupyter_server.extension.utils import validate_extension +# Use ServerApps environment because it monkeypatches +# jupyter_core.paths and provides a config directory +# that's not cross contaminating the user config directory. +pytestmark = pytest.mark.usefixtures("environ") + + + def test_validate_extension(): # enabled at sys level assert validate_extension('tests.extension.mockextensions.mockext_sys') diff --git a/tests/services/kernels/test_api.py b/tests/services/kernels/test_api.py index 068e4b28aa..71ac50bc0d 100644 --- a/tests/services/kernels/test_api.py +++ b/tests/services/kernels/test_api.py @@ -12,7 +12,6 @@ from jupyter_client.multikernelmanager import AsyncMultiKernelManager from jupyter_server.utils import url_path_join -from jupyter_server.services.kernels.kernelmanager import AsyncMappingKernelManager from ...utils import expected_http_error @@ -246,26 +245,3 @@ async def test_connection(fetch, ws_fetch, http_port, auth_header): model = json.loads(r.body.decode()) assert model['connections'] == 0 - -async def test_config2(serverapp): - assert serverapp.kernel_manager.allowed_message_types == [] - - -@pytest.mark.skipif( - sys.version_info < (3, 6), - reason="Kernel manager is AsyncMappingKernelManager, Python version < 3.6" -) -async def test_async_kernel_manager(configurable_serverapp): - argv = ['--ServerApp.kernel_manager_class=jupyter_server.services.kernels.kernelmanager.AsyncMappingKernelManager'] - app = configurable_serverapp(argv=argv) - assert isinstance(app.kernel_manager, AsyncMappingKernelManager) - - -@pytest.mark.skipif( - sys.version_info >= (3, 6), - reason="Testing AsyncMappingKernelManager on Python <=3.5" -) -async def test_async_kernel_manager_not_available_py35(configurable_serverapp): - argv = ['--ServerApp.kernel_manager_class=jupyter_server.services.kernels.kernelmanager.AsyncMappingKernelManager'] - with pytest.raises(ValueError): - app = configurable_serverapp(argv=argv) diff --git a/tests/services/kernels/test_config.py b/tests/services/kernels/test_config.py index ef6bd7709e..7539ae4b37 100644 --- a/tests/services/kernels/test_config.py +++ b/tests/services/kernels/test_config.py @@ -1,5 +1,7 @@ +import sys import pytest from traitlets.config import Config +from jupyter_server.services.kernels.kernelmanager import AsyncMappingKernelManager @pytest.fixture @@ -15,3 +17,26 @@ def server_config(): def test_config(serverapp): assert serverapp.kernel_manager.allowed_message_types == ['kernel_info_request'] + + +@pytest.mark.skipif( + sys.version_info < (3, 6), + reason="Kernel manager is AsyncMappingKernelManager, Python version < 3.6" +) +async def test_async_kernel_manager(configurable_serverapp): + argv = ['--ServerApp.kernel_manager_class=jupyter_server.services.kernels.kernelmanager.AsyncMappingKernelManager'] + app = configurable_serverapp(argv=argv) + assert isinstance(app.kernel_manager, AsyncMappingKernelManager) + + +@pytest.mark.skipif( + sys.version_info >= (3, 6), + reason="Testing AsyncMappingKernelManager on Python <=3.5" +) +@pytest.mark.parametrize( + "args", + [['--ServerApp.kernel_manager_class=jupyter_server.services.kernels.kernelmanager.AsyncMappingKernelManager']] +) +async def test_async_kernel_manager_not_available_py35(configurable_serverapp, args): + with pytest.raises(ValueError): + app = configurable_serverapp(argv=args) From f6c3cadd1f992e662d0caae3e2b21110b5d0013e Mon Sep 17 00:00:00 2001 From: Zsailer Date: Sun, 26 Jul 2020 20:50:40 -0700 Subject: [PATCH 23/30] handle sorting of dictionary keys properly --- jupyter_server/extension/manager.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/jupyter_server/extension/manager.py b/jupyter_server/extension/manager.py index f49081dfa9..e6947ff893 100644 --- a/jupyter_server/extension/manager.py +++ b/jupyter_server/extension/manager.py @@ -209,9 +209,8 @@ def enabled_extensions(self): """Dictionary with extension package names as keys and an ExtensionPackage objects as values. """ - # Sort enabled extensions before returning - out = sorted(self._enabled_extensions.items()) - return dict(out) + # Sort enabled extensions before + return self._enabled_extensions @property def extension_points(self): @@ -260,7 +259,7 @@ def link_all_extensions(self, serverapp): """ # Sort the extension names to enforce deterministic linking # order. - for name in self.enabled_extensions: + for name in sorted(self.enabled_extensions.keys()): self.link_extension(name, serverapp) def load_all_extensions(self, serverapp): @@ -269,6 +268,6 @@ def load_all_extensions(self, serverapp): """ # Sort the extension names to enforce deterministic loading # order. - for name in self.enabled_extensions: + for name in sorted(self.enabled_extensions.keys()): self.load_extension(name, serverapp) From 0003ade330561981e6bc75a9d32b00b2d34c54fb Mon Sep 17 00:00:00 2001 From: Zsailer Date: Mon, 27 Jul 2020 10:31:08 -0700 Subject: [PATCH 24/30] rename extension_paths to extension_points and add logging to reflect this change --- jupyter_server/extension/utils.py | 47 ++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/jupyter_server/extension/utils.py b/jupyter_server/extension/utils.py index d06f5903f3..84fa373f63 100644 --- a/jupyter_server/extension/utils.py +++ b/jupyter_server/extension/utils.py @@ -24,7 +24,7 @@ def get_extension_app_pkg(app_cls): raise NotAnExtensionApp("The ") -def get_loader(obj): +def get_loader(obj, logger=None): """Looks for _load_jupyter_server_extension as an attribute of the object or module. @@ -35,25 +35,58 @@ def get_loader(obj): func = getattr(obj, '_load_jupyter_server_extension') except AttributeError: func = getattr(obj, 'load_jupyter_server_extension') + except Exception: raise ExtensionLoadingError("_load_jupyter_server_extension function was not found.") return func -def get_metadata(package_name): +def get_metadata(package_name, logger=None): """Find the extension metadata from an extension package. + This looks for a `_jupyter_server_extension_points` function + that returns metadata about all extension points within a Jupyter + Server Extension pacakge. + If it doesn't exist, return a basic metadata packet given the module name. """ module = importlib.import_module(package_name) + + try: + return module._jupyter_server_extension_points + except AttributeError: + pass + + # For backwards compatibility, we temporarily allow + # _jupyter_server_extension_paths. We will remove in + # a later release of Jupyter Server. try: - return module._jupyter_server_extension_paths() + return module._jupyter_server_extension_paths except AttributeError: - return [{ - "module": package_name, - "name": package_name - }] + if logger: + logger.debug( + "A `_jupyter_server_extension_points` function was not " + "found in {name}. Instead, a `_jupyter_server_extension_paths` " + "function was found and will be used for now. This function " + "name will be deprecated in future releases " + "of Jupyter Server.".format(name=package_name) + ) + pass + + # Dynamically create metadata if the package doesn't + # provide it. + if logger: + logger.debug( + "A `_jupyter_server_extension_points` function was " + "not found in {name}, so Jupyter Server will look " + "for extension points in the extension pacakge's " + "root.".format(name=package_name) + ) + return [{ + "module": package_name, + "name": package_name + }] def validate_extension(name): From 3b0986042f922b3dfbb872c577265c46f1cfe5d1 Mon Sep 17 00:00:00 2001 From: Zsailer Date: Mon, 27 Jul 2020 12:21:58 -0700 Subject: [PATCH 25/30] update tests to changes in metadata function --- jupyter_server/extension/utils.py | 5 ++--- tests/extension/mockextensions/__init__.py | 2 +- tests/extension/test_manager.py | 8 ++++---- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/jupyter_server/extension/utils.py b/jupyter_server/extension/utils.py index 84fa373f63..6c59ed1cbc 100644 --- a/jupyter_server/extension/utils.py +++ b/jupyter_server/extension/utils.py @@ -35,7 +35,6 @@ def get_loader(obj, logger=None): func = getattr(obj, '_load_jupyter_server_extension') except AttributeError: func = getattr(obj, 'load_jupyter_server_extension') - except Exception: raise ExtensionLoadingError("_load_jupyter_server_extension function was not found.") return func @@ -54,7 +53,7 @@ def get_metadata(package_name, logger=None): module = importlib.import_module(package_name) try: - return module._jupyter_server_extension_points + return module._jupyter_server_extension_points() except AttributeError: pass @@ -62,7 +61,7 @@ def get_metadata(package_name, logger=None): # _jupyter_server_extension_paths. We will remove in # a later release of Jupyter Server. try: - return module._jupyter_server_extension_paths + return module._jupyter_server_extension_paths() except AttributeError: if logger: logger.debug( diff --git a/tests/extension/mockextensions/__init__.py b/tests/extension/mockextensions/__init__.py index 702dc3b123..5e41c308d8 100644 --- a/tests/extension/mockextensions/__init__.py +++ b/tests/extension/mockextensions/__init__.py @@ -6,7 +6,7 @@ # Function that makes these extensions discoverable # by the test functions. -def _jupyter_server_extension_paths(): +def _jupyter_server_extension_points(): return [ { 'module': 'tests.extension.mockextensions.app', diff --git a/tests/extension/test_manager.py b/tests/extension/test_manager.py index 8ad38166f3..68d6d6200f 100644 --- a/tests/extension/test_manager.py +++ b/tests/extension/test_manager.py @@ -15,10 +15,10 @@ def test_extension_point_api(): # Import mock extension metadata - from .mockextensions import _jupyter_server_extension_paths + from .mockextensions import _jupyter_server_extension_points # Testing the first path (which is an extension app). - metadata_list = _jupyter_server_extension_paths() + metadata_list = _jupyter_server_extension_points() point = metadata_list[0] module = point["module"] @@ -47,10 +47,10 @@ def test_extension_point_notfound_error(): def test_extension_package_api(): # Import mock extension metadata - from .mockextensions import _jupyter_server_extension_paths + from .mockextensions import _jupyter_server_extension_points # Testing the first path (which is an extension app). - metadata_list = _jupyter_server_extension_paths() + metadata_list = _jupyter_server_extension_points() path1 = metadata_list[0] app = path1["app"] From cbe843bd98f7cd6ec5c6bfebbfd4c572ae862937 Mon Sep 17 00:00:00 2001 From: Zsailer Date: Tue, 28 Jul 2020 16:00:12 -0700 Subject: [PATCH 26/30] add a basic extension config manager --- jupyter_server/extension/config.py | 65 +++++++++++++++++++++++++ jupyter_server/extension/manager.py | 74 +++++++++++++++++++++-------- jupyter_server/extension/utils.py | 11 +---- jupyter_server/serverapp.py | 12 ++--- tests/extension/test_config.py | 61 ++++++++++++++++++++++++ tests/extension/test_utils.py | 1 - 6 files changed, 189 insertions(+), 35 deletions(-) create mode 100644 jupyter_server/extension/config.py create mode 100644 tests/extension/test_config.py diff --git a/jupyter_server/extension/config.py b/jupyter_server/extension/config.py new file mode 100644 index 0000000000..2de52eea8f --- /dev/null +++ b/jupyter_server/extension/config.py @@ -0,0 +1,65 @@ + +from jupyter_server.services.config.manager import ConfigManager + + +DEFAULT_SECTION_NAME = "jupyter_server_config" + + +class ExtensionConfigManager(ConfigManager): + """A manager class to interface with Jupyter Server Extension config + found in a `config.d` folder. It is assumed that all configuration + files in this directory are JSON files. + """ + def get_jpserver_extensions( + self, + section_name=DEFAULT_SECTION_NAME + ): + """Return the jpserver_extensions field from all + config files found.""" + data = self.get(section_name) + return ( + data + .get("ServerApp", {}) + .get("jpserver_extensions", {}) + ) + + def enabled( + self, + name, + section_name=DEFAULT_SECTION_NAME, + include_root=True + ): + """Is the extension enabled?""" + extensions = self.get_jpserver_extensions(section_name) + try: + return extensions[name] + except KeyError: + return False + + def enable( + self, + name, + section_name=DEFAULT_SECTION_NAME, + ): + data = { + "ServerApp": { + "jpserver_extensions": { + name: True + } + } + } + self.update(section_name, data) + + def disable( + self, + name, + section_name=DEFAULT_SECTION_NAME + ): + data = { + "ServerApp": { + "jpserver_extensions": { + name: False + } + } + } + self.update(section_name, data) diff --git a/jupyter_server/extension/manager.py b/jupyter_server/extension/manager.py index e6947ff893..6f51443e19 100644 --- a/jupyter_server/extension/manager.py +++ b/jupyter_server/extension/manager.py @@ -84,12 +84,7 @@ def module(self): """ return self._module - def link(self, serverapp): - """Link the extension to a Jupyter ServerApp object. - - This looks for a `_link_jupyter_server_extension` function - in the extension's module or ExtensionApp class. - """ + def _get_linker(self): if self.app: linker = self.app._link_jupyter_server_extension else: @@ -100,11 +95,31 @@ def link(self, serverapp): # Otherwise return a dummy function. lambda serverapp: None ) + return linker + + def _get_loader(self): + loc = self.app + if not loc: + loc = self.module + loader = get_loader(loc) + return loader - # Capture output to return - out = linker(serverapp) - # Store that this extension has been linked - return out + def validate(self): + """Check that both a linker and loader exists.""" + try: + self.get_linker() + self.get_loader() + except Exception: + return False + + def link(self, serverapp): + """Link the extension to a Jupyter ServerApp object. + + This looks for a `_link_jupyter_server_extension` function + in the extension's module or ExtensionApp class. + """ + linker = self.get_linker() + return linker(serverapp) def load(self, serverapp): """Load the extension in a Jupyter ServerApp object. @@ -112,12 +127,7 @@ def load(self, serverapp): This looks for a `_load_jupyter_server_extension` function in the extension's module or ExtensionApp class. """ - # Use the ExtensionApp object to find a loading function - # if it exists. Otherwise, use the extension module given. - loc = self.app - if not loc: - loc = self.module - loader = get_loader(loc) + loader = self.get_loader() return loader(serverapp) @@ -143,7 +153,7 @@ def _validate_name(self, proposed): name = proposed['value'] self._extension_points = {} try: - self._metadata = get_metadata(name) + self._module, self._metadata = get_metadata(name) except ImportError: raise ExtensionModuleNotFound( "The module '{name}' could not be found. Are you " @@ -155,6 +165,16 @@ def _validate_name(self, proposed): self._extension_points[point.name] = point return name + @property + def module(self): + """Extension metadata loaded from the extension package.""" + return self._module + + @property + def version(self): + """Get the version of this package, if it's given. Otherwise, return an empty string""" + return getattr(self._module, "__version__", "") + @property def metadata(self): """Extension metadata loaded from the extension package.""" @@ -165,6 +185,13 @@ def extension_points(self): """A dictionary of extension points.""" return self._extension_points + def validate(self): + """Validate all extension points in this package.""" + for extension in self.extensions_points: + if not extension.validate(): + return False + return True + def link_point(self, point_name, serverapp): linked = self._linked_points.get(point_name, False) if not linked: @@ -191,7 +218,8 @@ class ExtensionManager(LoggingConfigurable): Usage: m = ExtensionManager(jpserver_extensions=extensions) """ - def __init__(self, *args, **kwargs): + def __init__(self, config_manager=None, *args, **kwargs): + super().__init__(*args, **kwargs) # The `enabled_extensions` attribute provides a dictionary # with extension (package) names mapped to their ExtensionPackage interface # (see above). This manager simplifies the interaction between the @@ -202,7 +230,9 @@ def __init__(self, *args, **kwargs): # extensions from being re-linked recursively unintentionally if another # extension attempts to link extensions again. self._linked_extensions = {} - super().__init__(*args, **kwargs) + self._config_manager = config_manager + if self._config_manager: + self.from_config_manager @property def enabled_extensions(self): @@ -221,6 +251,12 @@ def extension_points(self): for name, point in value.extension_points.items() } + def from_config_manager(self, config_manager): + """Add extensions found by an ExtensionConfigManager""" + self._config_manager = config_manager + jpserver_extensions = self._config_manager.get_jpserver_extensions() + self.from_jpserver_extensions(jpserver_extensions) + def from_jpserver_extensions(self, jpserver_extensions): """Add extensions from 'jpserver_extensions'-like dictionary.""" for name, enabled in jpserver_extensions.items(): diff --git a/jupyter_server/extension/utils.py b/jupyter_server/extension/utils.py index 6c59ed1cbc..bc2dcd4d6a 100644 --- a/jupyter_server/extension/utils.py +++ b/jupyter_server/extension/utils.py @@ -17,13 +17,6 @@ class NotAnExtensionApp(Exception): pass -def get_extension_app_pkg(app_cls): - """Get the Python package name - """ - if not isinstance(app_cls, "ExtensionApp"): - raise NotAnExtensionApp("The ") - - def get_loader(obj, logger=None): """Looks for _load_jupyter_server_extension as an attribute of the object or module. @@ -61,7 +54,7 @@ def get_metadata(package_name, logger=None): # _jupyter_server_extension_paths. We will remove in # a later release of Jupyter Server. try: - return module._jupyter_server_extension_paths() + return module, module._jupyter_server_extension_paths() except AttributeError: if logger: logger.debug( @@ -82,7 +75,7 @@ def get_metadata(package_name, logger=None): "for extension points in the extension pacakge's " "root.".format(name=package_name) ) - return [{ + return module, [{ "module": package_name, "name": package_name }] diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index a4f05803bd..70c703f44f 100755 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -102,6 +102,7 @@ from jupyter_server.extension.serverextension import ServerExtensionApp from jupyter_server.extension.manager import ExtensionManager +from jupyter_server.extension.config import ExtensionConfigManager #----------------------------------------------------------------------------- # Module globals @@ -1481,13 +1482,12 @@ def find_server_extensions(self): # This enables merging on keys, which we want for extension enabling. # Regular config loading only merges at the class level, # so each level clobbers the previous. - config_path = jupyter_config_path() - if self.config_dir not in config_path: + config_paths = jupyter_config_path() + if self.config_dir not in config_paths: # add self.config_dir to the front, if set manually - config_path.insert(0, self.config_dir) - manager = ConfigManager(read_config_path=config_path) - section = manager.get(self.config_file_name) - extensions = section.get('ServerApp', {}).get('jpserver_extensions', {}) + config_paths.insert(0, self.config_dir) + manager = ExtensionConfigManager(config_paths=config_paths) + extensions = manager.get_jpserver_extensions() for modulename, enabled in sorted(extensions.items()): if modulename not in self.jpserver_extensions: diff --git a/tests/extension/test_config.py b/tests/extension/test_config.py new file mode 100644 index 0000000000..4637c9783c --- /dev/null +++ b/tests/extension/test_config.py @@ -0,0 +1,61 @@ +import pytest + +from jupyter_core.paths import jupyter_config_path +from jupyter_server.extension.config import ( + ExtensionConfigManager, +) + +# Use ServerApps environment because it monkeypatches +# jupyter_core.paths and provides a config directory +# that's not cross contaminating the user config directory. +pytestmark = pytest.mark.usefixtures("environ") + + +@pytest.fixture +def configd(env_config_path): + """A pathlib.Path object that acts like a jupyter_server_config.d folder.""" + configd = env_config_path.joinpath('jupyter_server_config.d') + configd.mkdir() + return configd + + +ext1_json_config = """\ +{ + "ServerApp": { + "jpserver_extensions": { + "ext1_config": true + } + } +} +""" + +@pytest.fixture +def ext1_config(configd): + config = configd.joinpath("ext1_config.json") + config.write_text(ext1_json_config) + + +ext2_json_config = """\ +{ + "ServerApp": { + "jpserver_extensions": { + "ext2_config": false + } + } +} +""" + + +@pytest.fixture +def ext2_config(configd): + config = configd.joinpath("ext2_config.json") + config.write_text(ext2_json_config) + + +def test_list_extension_from_configd(ext1_config, ext2_config): + manager = ExtensionConfigManager( + read_config_path=jupyter_config_path() + ) + extensions = manager.get_jpserver_extensions() + assert "ext2_config" in extensions + assert "ext1_config" in extensions \ No newline at end of file diff --git a/tests/extension/test_utils.py b/tests/extension/test_utils.py index 54f48e17e3..66a89b86f7 100644 --- a/tests/extension/test_utils.py +++ b/tests/extension/test_utils.py @@ -8,7 +8,6 @@ pytestmark = pytest.mark.usefixtures("environ") - def test_validate_extension(): # enabled at sys level assert validate_extension('tests.extension.mockextensions.mockext_sys') From eca9e433078db8c5dbe43f9eaa814639b8931288 Mon Sep 17 00:00:00 2001 From: Zsailer Date: Fri, 31 Jul 2020 11:03:17 -0700 Subject: [PATCH 27/30] leverage the extension manager for enabling/disabling --- jupyter_server/config_manager.py | 1 - jupyter_server/extension/config.py | 16 +- jupyter_server/extension/manager.py | 54 +++--- jupyter_server/extension/serverextension.py | 172 +++++++++----------- jupyter_server/extension/utils.py | 2 +- jupyter_server/serverapp.py | 11 +- tests/extension/test_app.py | 2 +- tests/extension/test_manager.py | 4 +- tests/extension/test_serverextension.py | 4 +- 9 files changed, 120 insertions(+), 146 deletions(-) diff --git a/jupyter_server/config_manager.py b/jupyter_server/config_manager.py index 584df88709..53c9852a90 100644 --- a/jupyter_server/config_manager.py +++ b/jupyter_server/config_manager.py @@ -118,7 +118,6 @@ def set(self, section_name, data): # Generate the JSON up front, since it could raise an exception, # in order to avoid writing half-finished corrupted data to disk. json_content = json.dumps(data, indent=2) - if PY3: f = io.open(filename, 'w', encoding='utf-8') else: diff --git a/jupyter_server/extension/config.py b/jupyter_server/extension/config.py index 2de52eea8f..3091847abc 100644 --- a/jupyter_server/extension/config.py +++ b/jupyter_server/extension/config.py @@ -36,11 +36,7 @@ def enabled( except KeyError: return False - def enable( - self, - name, - section_name=DEFAULT_SECTION_NAME, - ): + def enable(self, name): data = { "ServerApp": { "jpserver_extensions": { @@ -48,13 +44,9 @@ def enable( } } } - self.update(section_name, data) + self.update(name, data) - def disable( - self, - name, - section_name=DEFAULT_SECTION_NAME - ): + def disable(self, name): data = { "ServerApp": { "jpserver_extensions": { @@ -62,4 +54,4 @@ def disable( } } } - self.update(section_name, data) + self.update(name, data) diff --git a/jupyter_server/extension/manager.py b/jupyter_server/extension/manager.py index 6f51443e19..4f1fedec7c 100644 --- a/jupyter_server/extension/manager.py +++ b/jupyter_server/extension/manager.py @@ -5,6 +5,7 @@ HasTraits, Dict, Unicode, + Bool, validate ) @@ -107,8 +108,8 @@ def _get_loader(self): def validate(self): """Check that both a linker and loader exists.""" try: - self.get_linker() - self.get_loader() + self._get_linker() + self._get_loader() except Exception: return False @@ -118,7 +119,7 @@ def link(self, serverapp): This looks for a `_link_jupyter_server_extension` function in the extension's module or ExtensionApp class. """ - linker = self.get_linker() + linker = self._get_linker() return linker(serverapp) def load(self, serverapp): @@ -127,7 +128,7 @@ def load(self, serverapp): This looks for a `_load_jupyter_server_extension` function in the extension's module or ExtensionApp class. """ - loader = self.get_loader() + loader = self._get_loader() return loader(serverapp) @@ -140,6 +141,7 @@ class ExtensionPackage(HasTraits): extpkg = ExtensionPackage(name=ext_name) """ name = Unicode(help="Name of the an importable Python package.") + enabled = Bool(False).tag(config=True) def __init__(self, *args, **kwargs): # Store extension points that have been linked. @@ -187,7 +189,7 @@ def extension_points(self): def validate(self): """Validate all extension points in this package.""" - for extension in self.extensions_points: + for extension in self.extension_points.values(): if not extension.validate(): return False return True @@ -224,7 +226,7 @@ def __init__(self, config_manager=None, *args, **kwargs): # with extension (package) names mapped to their ExtensionPackage interface # (see above). This manager simplifies the interaction between the # ServerApp and the extensions being appended. - self._enabled_extensions = {} + self._extensions = {} # The `_linked_extensions` attribute tracks when each extension # has been successfully linked to a ServerApp. This helps prevent # extensions from being re-linked recursively unintentionally if another @@ -232,19 +234,19 @@ def __init__(self, config_manager=None, *args, **kwargs): self._linked_extensions = {} self._config_manager = config_manager if self._config_manager: - self.from_config_manager + self.from_config_manager(self._config_manager) @property - def enabled_extensions(self): + def extensions(self): """Dictionary with extension package names as keys and an ExtensionPackage objects as values. """ # Sort enabled extensions before - return self._enabled_extensions + return self._extensions @property def extension_points(self): - extensions = self.enabled_extensions + extensions = self.extensions return { name: point for value in extensions.values() @@ -260,21 +262,20 @@ def from_config_manager(self, config_manager): def from_jpserver_extensions(self, jpserver_extensions): """Add extensions from 'jpserver_extensions'-like dictionary.""" for name, enabled in jpserver_extensions.items(): - if enabled: - self.add_extension(name) + self.add_extension(name, enabled=enabled) - def add_extension(self, extension_name): + def add_extension(self, extension_name, enabled=False): try: - extpkg = ExtensionPackage(name=extension_name) - self._enabled_extensions[extension_name] = extpkg - # Raise a warning if the extension cannot be loaded. + extpkg = ExtensionPackage(name=extension_name, enabled=enabled) + self._extensions[extension_name] = extpkg + # Raise a warning if the extension cannot be loaded. except Exception as e: self.log.warning(e) def link_extension(self, name, serverapp): linked = self._linked_extensions.get(name, False) - extension = self.enabled_extensions[name] - if not linked: + extension = self.extensions[name] + if not linked and extension.enabled: try: extension.link_all_points(serverapp) self.log.info("{name} | extension was successfully linked.".format(name=name)) @@ -282,12 +283,13 @@ def link_extension(self, name, serverapp): self.log.warning(e) def load_extension(self, name, serverapp): - extension = self.enabled_extensions.get(name) - try: - extension.load_all_points(serverapp) - self.log.info("{name} | extension was successfully loaded.".format(name=name)) - except Exception as e: - self.log.warning(e) + extension = self.extensions.get(name) + if extension.enabled: + try: + extension.load_all_points(serverapp) + self.log.info("{name} | extension was successfully loaded.".format(name=name)) + except Exception as e: + self.log.warning(e) def link_all_extensions(self, serverapp): """Link all enabled extensions @@ -295,7 +297,7 @@ def link_all_extensions(self, serverapp): """ # Sort the extension names to enforce deterministic linking # order. - for name in sorted(self.enabled_extensions.keys()): + for name in sorted(self.extensions.keys()): self.link_extension(name, serverapp) def load_all_extensions(self, serverapp): @@ -304,6 +306,6 @@ def load_all_extensions(self, serverapp): """ # Sort the extension names to enforce deterministic loading # order. - for name in sorted(self.enabled_extensions.keys()): + for name in sorted(self.extensions.keys()): self.load_extension(name, serverapp) diff --git a/jupyter_server/extension/serverextension.py b/jupyter_server/extension/serverextension.py index 7bd77bfe0b..62ea873885 100644 --- a/jupyter_server/extension/serverextension.py +++ b/jupyter_server/extension/serverextension.py @@ -17,10 +17,36 @@ SYSTEM_CONFIG_PATH ) from jupyter_server._version import __version__ -from jupyter_server.config_manager import BaseJSONConfigManager +from jupyter_server.extension.config import ExtensionConfigManager +from jupyter_server.extension.manager import ExtensionManager from .utils import validate_extension +def _get_config_dir(user=False, sys_prefix=False): + """Get the location of config files for the current context + + Returns the string to the environment + + Parameters + ---------- + + user : bool [default: False] + Get the user's .jupyter config directory + sys_prefix : bool [default: False] + Get sys.prefix, i.e. ~/.envs/my-env/etc/jupyter + """ + user = False if sys_prefix else user + if user and sys_prefix: + raise ArgumentConflict("Cannot specify more than one of user or sys_prefix") + if user: + extdir = jupyter_config_dir() + elif sys_prefix: + extdir = ENV_CONFIG_PATH[0] + else: + extdir = SYSTEM_CONFIG_PATH[0] + return extdir + + class ArgumentConflict(ValueError): pass @@ -72,30 +98,22 @@ def _log_format_default(self): """A default format for messages""" return "%(message)s" + @property + def config_dir(self): + return _get_config_dir(user=self.user, sys_prefix=self.sys_prefix) -def _get_config_dir(user=False, sys_prefix=False): - """Get the location of config files for the current context - - Returns the string to the environment - - Parameters - ---------- - - user : bool [default: False] - Get the user's .jupyter config directory - sys_prefix : bool [default: False] - Get sys.prefix, i.e. ~/.envs/my-env/etc/jupyter - """ - user = False if sys_prefix else user - if user and sys_prefix: - raise ArgumentConflict("Cannot specify more than one of user or sys_prefix") - if user: - extdir = jupyter_config_dir() - elif sys_prefix: - extdir = ENV_CONFIG_PATH[0] - else: - extdir = SYSTEM_CONFIG_PATH[0] - return extdir + def initialize(self, *args, **kwargs): + # Locate Server extension config in Jupyter Server's config.d + self.config_manager = ExtensionConfigManager( + read_config_path=[self.config_dir], + write_config_dir=os.path.join(self.config_dir, "jupyter_server_config.d"), + log=self.log + ) + self.extension_manager = ExtensionManager( + config_manager=self.config_manager, + log=self.log + ) + super().initialize(*args, **kwargs) # Constants for pretty print extension listing function. @@ -122,16 +140,14 @@ def toggle_server_extension_python( """ sys_prefix = False if user else sys_prefix config_dir = _get_config_dir(user=user, sys_prefix=sys_prefix) - cm = BaseJSONConfigManager(parent=parent, config_dir=config_dir) - cfg = cm.get("jupyter_server_config") - server_extensions = ( - cfg.setdefault("ServerApp", {}) - .setdefault("jpserver_extensions", {}) + manager = ExtensionConfigManager( + read_config_path=[config_dir], + write_config_dir=os.path.join(config_dir, "jupyter_server_config.d") ) - old_enabled = server_extensions.get(import_name, None) - new_enabled = enabled if enabled is not None else not old_enabled - server_extensions[import_name] = new_enabled - cm.update("jupyter_server_config", cfg) + if enabled: + manager.enable(import_name) + else: + manager.disable(import_name) # ---------------------------------------------------------------------- # Applications @@ -174,9 +190,6 @@ class ToggleServerExtensionApp(BaseExtensionApp): flags = flags - user = Bool(False, config=True, help="Whether to do a user install") - sys_prefix = Bool(True, config=True, help="Use the sys.prefix as the prefix") - python = Bool(False, config=True, help="Install from a Python package") _toggle_value = Bool() _toggle_pre_message = '' _toggle_post_message = '' @@ -195,51 +208,32 @@ def toggle_server_extension(self, import_name): """ try: self.log.info("{}: {}".format(self._toggle_pre_message.capitalize(), import_name)) + self.log.info("- Writing config: {}".format(self.config_dir)) # Validate the server extension. self.log.info(" - Validating {}...".format(import_name)) - version = validate_extension(import_name) - - # Toggle the server extension to active. - toggle_server_extension_python( - import_name, - self._toggle_value, - parent=self, - user=self.user, - sys_prefix=self.sys_prefix - ) + extension = self.extension_manager.extensions[import_name] + extension.validate() + version = extension.version self.log.info(" {} {} {}".format(import_name, version, GREEN_OK)) + # Toggle extension config. + config = self.config_manager + if self._toggle_value is True: + config.enable(import_name) + else: + config.disable(import_name) + # If successful, let's log. self.log.info(" - Extension successfully {}.".format(self._toggle_post_message)) except Exception as err: self.log.info(" {} Validation failed: {}".format(RED_X, err)) - def toggle_server_extension_python(self, package): - """Change the status of some server extensions in a Python package. - - Uses the value of `self._toggle_value`. - - Parameters - --------- - - package : str - Importable Python module exposing the - magic-named `_jupyter_server_extension_paths` function - """ - _, server_exts = _get_server_extension_metadata(package) - for server_ext in server_exts: - module = server_ext['module'] - self.toggle_server_extension(module) - def start(self): """Perform the App's actions as configured""" if not self.extra_args: sys.exit('Please specify a server extension/package to enable or disable') for arg in self.extra_args: - if self.python: - self.toggle_server_extension_python(arg) - else: - self.toggle_server_extension(arg) + self.toggle_server_extension(arg) class EnableServerExtensionApp(ToggleServerExtensionApp): @@ -281,34 +275,22 @@ def list_server_extensions(self): Enabled extensions are validated, potentially generating warnings. """ - config_dirs = jupyter_config_path() - - # Iterate over all locations where extensions might be named. - for config_dir in config_dirs: - cm = BaseJSONConfigManager(parent=self, config_dir=config_dir) - data = cm.get("jupyter_server_config") - server_extensions = ( - data.setdefault("ServerApp", {}) - .setdefault("jpserver_extensions", {}) - ) - if server_extensions: - self.log.info(u'config dir: {}'.format(config_dir)) - - # Iterate over packages listed in jpserver_extensions. - for pkg_name, enabled in server_extensions.items(): - # Attempt to get extension metadata - self.log.info(u' {} {}'.format( - pkg_name, - GREEN_ENABLED if enabled else RED_DISABLED)) - try: - self.log.info(" - Validating {}...".format(pkg_name)) - version = validate_extension(pkg_name) - self.log.info( - " {} {} {}".format(pkg_name, version, GREEN_OK) - ) - - except Exception as err: - self.log.warn(" {} {}".format(RED_X, err)) + self.log.info("Config dir: {}".format(self.config_dir)) + for name, extension in self.extension_manager.extensions.items(): + enabled = extension.enabled + # Attempt to get extension metadata + self.log.info(u' {} {}'.format( + name, + GREEN_ENABLED if enabled else RED_DISABLED)) + try: + self.log.info(" - Validating {}...".format(name)) + extension.validate() + version = extension.version + self.log.info( + " {} {} {}".format(name, version, GREEN_OK) + ) + except Exception as err: + self.log.warn(" {} {}".format(RED_X, err)) def start(self): """Perform the App's actions as configured""" diff --git a/jupyter_server/extension/utils.py b/jupyter_server/extension/utils.py index bc2dcd4d6a..9fcf257395 100644 --- a/jupyter_server/extension/utils.py +++ b/jupyter_server/extension/utils.py @@ -46,7 +46,7 @@ def get_metadata(package_name, logger=None): module = importlib.import_module(package_name) try: - return module._jupyter_server_extension_points() + return module, module._jupyter_server_extension_points() except AttributeError: pass diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index 70c703f44f..b89e3ce74b 100755 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -1486,7 +1486,7 @@ def find_server_extensions(self): if self.config_dir not in config_paths: # add self.config_dir to the front, if set manually config_paths.insert(0, self.config_dir) - manager = ExtensionConfigManager(config_paths=config_paths) + manager = ExtensionConfigManager(read_config_path=config_paths) extensions = manager.get_jpserver_extensions() for modulename, enabled in sorted(extensions.items()): @@ -1662,15 +1662,15 @@ def initialize(self, argv=None, find_extensions=True, new_httpserver=True): # Parse command line, load ServerApp config files, # and update ServerApp config. super(ServerApp, self).initialize(argv) + # Initialize all components of the ServerApp. + if self._dispatching: + return # Then, use extensions' config loading mechanism to # update config. ServerApp config takes precedence. if find_extensions: self.find_server_extensions() - self.init_server_extensions() - # Initialize all components of the ServerApp. self.init_logging() - if self._dispatching: - return + self.init_server_extensions() self.init_configurables() self.init_components() self.init_webapp() @@ -1682,7 +1682,6 @@ def initialize(self, argv=None, find_extensions=True, new_httpserver=True): self.init_mime_overrides() self.init_shutdown_no_activity() - def cleanup_kernels(self): """Shutdown all kernels. diff --git a/tests/extension/test_app.py b/tests/extension/test_app.py index 7303609e6f..e527e79b74 100644 --- a/tests/extension/test_app.py +++ b/tests/extension/test_app.py @@ -23,7 +23,7 @@ def server_config(template_dir): @pytest.fixture def mock_extension(extension_manager): name = "tests.extension.mockextensions" - pkg = extension_manager.enabled_extensions[name] + pkg = extension_manager.extensions[name] point = pkg.extension_points["mockextension"] app = point.app return app diff --git a/tests/extension/test_manager.py b/tests/extension/test_manager.py index 68d6d6200f..b076c43f25 100644 --- a/tests/extension/test_manager.py +++ b/tests/extension/test_manager.py @@ -72,6 +72,6 @@ def test_extension_manager_api(): } manager = ExtensionManager() manager.from_jpserver_extensions(jpserver_extensions) - assert len(manager.enabled_extensions) == 1 - assert "tests.extension.mockextensions" in manager.enabled_extensions + assert len(manager.extensions) == 1 + assert "tests.extension.mockextensions" in manager.extensions diff --git a/tests/extension/test_serverextension.py b/tests/extension/test_serverextension.py index c18764c1dc..dcf098b537 100644 --- a/tests/extension/test_serverextension.py +++ b/tests/extension/test_serverextension.py @@ -29,13 +29,13 @@ def get_config(sys_prefix=True): return data.get("ServerApp", {}).get("jpserver_extensions", {}) -def test_enable(): +def test_enable(env_config_path, extension_environ): toggle_server_extension_python('mock1', True) config = get_config() assert config['mock1'] -def test_disable(): +def test_disable(env_config_path, extension_environ): toggle_server_extension_python('mock1', True) toggle_server_extension_python('mock1', False) From 1ccf97bd2b53f65fac07986d99ce9770f1f1172b Mon Sep 17 00:00:00 2001 From: Zsailer Date: Fri, 31 Jul 2020 11:19:17 -0700 Subject: [PATCH 28/30] list all paths in server extension application --- jupyter_server/extension/serverextension.py | 87 +++++++++++++-------- 1 file changed, 56 insertions(+), 31 deletions(-) diff --git a/jupyter_server/extension/serverextension.py b/jupyter_server/extension/serverextension.py index 62ea873885..b7e7943495 100644 --- a/jupyter_server/extension/serverextension.py +++ b/jupyter_server/extension/serverextension.py @@ -47,6 +47,30 @@ def _get_config_dir(user=False, sys_prefix=False): return extdir +def _get_extmanager_for_context(user=False, sys_prefix=False): + """Get an extension manager pointing at the current context + + Returns the path to the current context and an ExtensionManager object. + + Parameters + ---------- + + user : bool [default: False] + Get the user's .jupyter config directory + sys_prefix : bool [default: False] + Get sys.prefix, i.e. ~/.envs/my-env/etc/jupyter + """ + config_dir = _get_config_dir(user=user, sys_prefix=sys_prefix) + config_manager = ExtensionConfigManager( + read_config_path=[config_dir], + write_config_dir=os.path.join(config_dir, "jupyter_server_config.d"), + ) + extension_manager = ExtensionManager( + config_manager=config_manager, + ) + return config_dir, extension_manager + + class ArgumentConflict(ValueError): pass @@ -102,19 +126,6 @@ def _log_format_default(self): def config_dir(self): return _get_config_dir(user=self.user, sys_prefix=self.sys_prefix) - def initialize(self, *args, **kwargs): - # Locate Server extension config in Jupyter Server's config.d - self.config_manager = ExtensionConfigManager( - read_config_path=[self.config_dir], - write_config_dir=os.path.join(self.config_dir, "jupyter_server_config.d"), - log=self.log - ) - self.extension_manager = ExtensionManager( - config_manager=self.config_manager, - log=self.log - ) - super().initialize(*args, **kwargs) - # Constants for pretty print extension listing function. # Window doesn't support coloring in the commandline @@ -206,12 +217,17 @@ def toggle_server_extension(self, import_name): Importable Python module (dotted-notation) exposing the magic-named `load_jupyter_server_extension` function """ + # Create an extension manager for this instance. + ext_manager, extension_manager = _get_extmanager_for_context( + user=self.user, + sys_prefix=self.sys_prefix + ) try: self.log.info("{}: {}".format(self._toggle_pre_message.capitalize(), import_name)) - self.log.info("- Writing config: {}".format(self.config_dir)) + self.log.info("- Writing config: {}".format(ext_manager)) # Validate the server extension. self.log.info(" - Validating {}...".format(import_name)) - extension = self.extension_manager.extensions[import_name] + extension = extension_manager.extensions[import_name] extension.validate() version = extension.version self.log.info(" {} {} {}".format(import_name, version, GREEN_OK)) @@ -275,22 +291,31 @@ def list_server_extensions(self): Enabled extensions are validated, potentially generating warnings. """ - self.log.info("Config dir: {}".format(self.config_dir)) - for name, extension in self.extension_manager.extensions.items(): - enabled = extension.enabled - # Attempt to get extension metadata - self.log.info(u' {} {}'.format( - name, - GREEN_ENABLED if enabled else RED_DISABLED)) - try: - self.log.info(" - Validating {}...".format(name)) - extension.validate() - version = extension.version - self.log.info( - " {} {} {}".format(name, version, GREEN_OK) - ) - except Exception as err: - self.log.warn(" {} {}".format(RED_X, err)) + configurations = ( + {"user":True, "sys_prefix": False}, + {"user":False, "sys_prefix": True}, + {"user":False, "sys_prefix": False} + ) + for option in configurations: + config_dir, ext_manager = _get_extmanager_for_context(**option) + self.log.info("Config dir: {}".format(config_dir)) + for name, extension in ext_manager.extensions.items(): + enabled = extension.enabled + # Attempt to get extension metadata + self.log.info(u' {} {}'.format( + name, + GREEN_ENABLED if enabled else RED_DISABLED)) + try: + self.log.info(" - Validating {}...".format(name)) + extension.validate() + version = extension.version + self.log.info( + " {} {} {}".format(name, version, GREEN_OK) + ) + except Exception as err: + self.log.warn(" {} {}".format(RED_X, err)) + # Add a blank line between paths. + self.log.info("") def start(self): """Perform the App's actions as configured""" From b0a759fa8116889cecda33a89ed6927b4667b6af Mon Sep 17 00:00:00 2001 From: Zsailer Date: Fri, 31 Jul 2020 11:20:07 -0700 Subject: [PATCH 29/30] minor styling fix --- jupyter_server/extension/serverextension.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jupyter_server/extension/serverextension.py b/jupyter_server/extension/serverextension.py index b7e7943495..a3637a2bc5 100644 --- a/jupyter_server/extension/serverextension.py +++ b/jupyter_server/extension/serverextension.py @@ -292,9 +292,9 @@ def list_server_extensions(self): Enabled extensions are validated, potentially generating warnings. """ configurations = ( - {"user":True, "sys_prefix": False}, - {"user":False, "sys_prefix": True}, - {"user":False, "sys_prefix": False} + {"user": True, "sys_prefix": False}, + {"user": False, "sys_prefix": True}, + {"user": False, "sys_prefix": False} ) for option in configurations: config_dir, ext_manager = _get_extmanager_for_context(**option) From 4d1f8e5b0572e14d774cb763649710566c50315a Mon Sep 17 00:00:00 2001 From: Zsailer Date: Fri, 31 Jul 2020 11:20:33 -0700 Subject: [PATCH 30/30] remove unused imports --- jupyter_server/extension/serverextension.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/jupyter_server/extension/serverextension.py b/jupyter_server/extension/serverextension.py index a3637a2bc5..f057c0310f 100644 --- a/jupyter_server/extension/serverextension.py +++ b/jupyter_server/extension/serverextension.py @@ -12,14 +12,12 @@ from jupyter_core.application import JupyterApp from jupyter_core.paths import ( jupyter_config_dir, - jupyter_config_path, ENV_CONFIG_PATH, SYSTEM_CONFIG_PATH ) from jupyter_server._version import __version__ from jupyter_server.extension.config import ExtensionConfigManager from jupyter_server.extension.manager import ExtensionManager -from .utils import validate_extension def _get_config_dir(user=False, sys_prefix=False):