From ee1383791c6d4c57d76eab834b4cbc187a8118b6 Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Thu, 8 Apr 2021 08:29:28 -0700 Subject: [PATCH] [sonic-py-common] Add 'general' module with load_module_from_source() function (#7167) #### Why I did it To eliminate the need to write duplicate code in order to import a Python module from a source file. #### How I did it Add `general` module to sonic-py-common, which contains a `load_module_from_source()` function which supports both Python 2 and 3. Call this new function in: - sonic-ctrmgrd/tests/container_test.py - sonic-ctrmgrd/tests/ctrmgr_tools_test.py - sonic-host-services/tests/determine-reboot-cause_test.py - sonic-host-services/tests/hostcfgd/hostcfgd_test.py - sonic-host-services/tests/procdockerstatsd_test.py - sonic-py-common/sonic_py_common/daemon_base.py --- src/sonic-ctrmgrd/setup.py | 1 + src/sonic-ctrmgrd/tests/common_test.py | 12 --------- src/sonic-ctrmgrd/tests/container_test.py | 6 +++-- src/sonic-ctrmgrd/tests/ctrmgr_tools_test.py | 3 ++- src/sonic-host-services/setup.py | 1 + .../tests/determine-reboot-cause_test.py | 9 ++----- .../tests/hostcfgd/hostcfgd_test.py | 14 ++++------ .../tests/procdockerstatsd_test.py | 9 ++----- .../sonic_py_common/daemon_base.py | 26 ++----------------- .../sonic_py_common/general.py | 25 ++++++++++++++++++ 10 files changed, 44 insertions(+), 62 deletions(-) create mode 100644 src/sonic-py-common/sonic_py_common/general.py diff --git a/src/sonic-ctrmgrd/setup.py b/src/sonic-ctrmgrd/setup.py index b53efc7615d1..85e02cae12ec 100644 --- a/src/sonic-ctrmgrd/setup.py +++ b/src/sonic-ctrmgrd/setup.py @@ -24,6 +24,7 @@ tests_require=[ 'pytest', 'pytest-cov', + 'sonic-py-common', ], install_requires=['netaddr', 'pyyaml'], license="GNU General Public License v3", diff --git a/src/sonic-ctrmgrd/tests/common_test.py b/src/sonic-ctrmgrd/tests/common_test.py index 5b3eae151cb0..54c877f11365 100755 --- a/src/sonic-ctrmgrd/tests/common_test.py +++ b/src/sonic-ctrmgrd/tests/common_test.py @@ -1,6 +1,4 @@ import copy -import importlib.machinery -import importlib.util import json import os import subprocess @@ -655,13 +653,3 @@ def create_remote_ctr_config_json(): s.write(str_conf) return fname - - -def load_mod_from_file(modname, fpath): - spec = importlib.util.spec_from_loader(modname, - importlib.machinery.SourceFileLoader(modname, fpath)) - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - sys.modules[modname] = mod - return mod - diff --git a/src/sonic-ctrmgrd/tests/container_test.py b/src/sonic-ctrmgrd/tests/container_test.py index 4d28a7761d29..146d41386de1 100755 --- a/src/sonic-ctrmgrd/tests/container_test.py +++ b/src/sonic-ctrmgrd/tests/container_test.py @@ -2,12 +2,14 @@ from unittest.mock import MagicMock, patch import pytest +from sonic_py_common.general import load_module_from_source from . import common_test -common_test.load_mod_from_file("docker", + +load_module_from_source("docker", os.path.join(os.path.dirname(os.path.realpath(__file__)), "mock_docker.py")) -container = common_test.load_mod_from_file("container", +container = load_module_from_source("container", os.path.join(os.path.dirname(os.path.realpath(__file__)), "../ctrmgr/container")) diff --git a/src/sonic-ctrmgrd/tests/ctrmgr_tools_test.py b/src/sonic-ctrmgrd/tests/ctrmgr_tools_test.py index d6ffa15ed68b..8ea5a5195fdf 100755 --- a/src/sonic-ctrmgrd/tests/ctrmgr_tools_test.py +++ b/src/sonic-ctrmgrd/tests/ctrmgr_tools_test.py @@ -3,10 +3,11 @@ from unittest.mock import MagicMock, patch import pytest +from sonic_py_common.general import load_module_from_source from . import common_test -common_test.load_mod_from_file("docker", +load_module_from_source("docker", os.path.join(os.path.dirname(os.path.realpath(__file__)), "mock_docker.py")) sys.path.append("ctrmgr") diff --git a/src/sonic-host-services/setup.py b/src/sonic-host-services/setup.py index e5431d034d9c..057ae6f18596 100644 --- a/src/sonic-host-services/setup.py +++ b/src/sonic-host-services/setup.py @@ -35,6 +35,7 @@ ], tests_require = [ 'pytest', + 'sonic-py-common' ], classifiers = [ 'Development Status :: 3 - Alpha', diff --git a/src/sonic-host-services/tests/determine-reboot-cause_test.py b/src/sonic-host-services/tests/determine-reboot-cause_test.py index 4eb95ee219ee..d9e999a5ce27 100644 --- a/src/sonic-host-services/tests/determine-reboot-cause_test.py +++ b/src/sonic-host-services/tests/determine-reboot-cause_test.py @@ -1,10 +1,9 @@ -import importlib.machinery -import importlib.util import sys import os import pytest import swsssdk +from sonic_py_common.general import load_module_from_source # TODO: Remove this if/else block once we no longer support Python 2 if sys.version_info.major == 3: @@ -31,11 +30,7 @@ # Load the file under test determine_reboot_cause_path = os.path.join(scripts_path, 'determine-reboot-cause') -loader = importlib.machinery.SourceFileLoader('determine_reboot_cause', determine_reboot_cause_path) -spec = importlib.util.spec_from_loader(loader.name, loader) -determine_reboot_cause = importlib.util.module_from_spec(spec) -loader.exec_module(determine_reboot_cause) -sys.modules['determine_reboot_cause'] = determine_reboot_cause +determine_reboot_cause = load_module_from_source('determine_reboot_cause', determine_reboot_cause_path) PROC_CMDLINE_CONTENTS = """\ diff --git a/src/sonic-host-services/tests/hostcfgd/hostcfgd_test.py b/src/sonic-host-services/tests/hostcfgd/hostcfgd_test.py index 57e7215715d7..2abbb7f7cad6 100644 --- a/src/sonic-host-services/tests/hostcfgd/hostcfgd_test.py +++ b/src/sonic-host-services/tests/hostcfgd/hostcfgd_test.py @@ -1,13 +1,13 @@ -import importlib.machinery -import importlib.util import os import sys import swsssdk from parameterized import parameterized +from sonic_py_common.general import load_module_from_source from unittest import TestCase, mock -from tests.hostcfgd.test_vectors import HOSTCFGD_TEST_VECTOR -from tests.hostcfgd.mock_configdb import MockConfigDb + +from .test_vectors import HOSTCFGD_TEST_VECTOR +from .mock_configdb import MockConfigDb swsssdk.ConfigDBConnector = MockConfigDb @@ -18,11 +18,7 @@ # Load the file under test hostcfgd_path = os.path.join(scripts_path, 'hostcfgd') -loader = importlib.machinery.SourceFileLoader('hostcfgd', hostcfgd_path) -spec = importlib.util.spec_from_loader(loader.name, loader) -hostcfgd = importlib.util.module_from_spec(spec) -loader.exec_module(hostcfgd) -sys.modules['hostcfgd'] = hostcfgd +hostcfgd = load_module_from_source('hostcfgd', hostcfgd_path) class TestHostcfgd(TestCase): diff --git a/src/sonic-host-services/tests/procdockerstatsd_test.py b/src/sonic-host-services/tests/procdockerstatsd_test.py index bb218e52ce2d..65c5a738ca67 100644 --- a/src/sonic-host-services/tests/procdockerstatsd_test.py +++ b/src/sonic-host-services/tests/procdockerstatsd_test.py @@ -1,10 +1,9 @@ -import importlib.machinery -import importlib.util import sys import os import pytest import swsssdk +from sonic_py_common.general import load_module_from_source from .mock_connector import MockConnector @@ -17,11 +16,7 @@ # Load the file under test procdockerstatsd_path = os.path.join(scripts_path, 'procdockerstatsd') -loader = importlib.machinery.SourceFileLoader('procdockerstatsd', procdockerstatsd_path) -spec = importlib.util.spec_from_loader(loader.name, loader) -procdockerstatsd = importlib.util.module_from_spec(spec) -loader.exec_module(procdockerstatsd) -sys.modules['procdockerstatsd'] = procdockerstatsd +procdockerstatsd = load_module_from_source('procdockerstatsd', procdockerstatsd_path) class TestProcDockerStatsDaemon(object): def test_convert_to_bytes(self): diff --git a/src/sonic-py-common/sonic_py_common/daemon_base.py b/src/sonic-py-common/sonic_py_common/daemon_base.py index 745b3c55ba96..86870ba1fdc0 100644 --- a/src/sonic-py-common/sonic_py_common/daemon_base.py +++ b/src/sonic-py-common/sonic_py_common/daemon_base.py @@ -2,6 +2,7 @@ import sys from . import device_info +from .general import load_module_from_source from .logger import Logger # @@ -25,29 +26,6 @@ def db_connect(db_name, namespace=EMPTY_NAMESPACE): return swsscommon.DBConnector(db_name, REDIS_TIMEOUT_MSECS, True, namespace) -# TODO: Consider moving this logic out of daemon_base and into antoher file -# so that it can be used by non-daemons. We can simply call that function here -# to retain backward compatibility. -def _load_module_from_file(module_name, file_path): - module = None - - # TODO: Remove this check once we no longer support Python 2 - if sys.version_info.major == 3: - import importlib.machinery - import importlib.util - loader = importlib.machinery.SourceFileLoader(module_name, file_path) - spec = importlib.util.spec_from_loader(loader.name, loader) - module = importlib.util.module_from_spec(spec) - loader.exec_module(module) - else: - import imp - module = imp.load_source(module_name, file_path) - - sys.modules[module_name] = module - - return module - - # # DaemonBase =================================================================== # @@ -92,7 +70,7 @@ def load_platform_util(self, module_name, class_name): try: module_file = "/".join([platform_path, "plugins", module_name + ".py"]) - module = _load_module_from_file(module_name, module_file) + module = load_module_from_source(module_name, module_file) except IOError as e: raise IOError("Failed to load platform module '%s': %s" % (module_name, str(e))) diff --git a/src/sonic-py-common/sonic_py_common/general.py b/src/sonic-py-common/sonic_py_common/general.py new file mode 100644 index 000000000000..9e04f3e214ee --- /dev/null +++ b/src/sonic-py-common/sonic_py_common/general.py @@ -0,0 +1,25 @@ +import sys + + +def load_module_from_source(module_name, file_path): + """ + This function will load the Python source file specified by + as a module named and return an instance of the module + """ + module = None + + # TODO: Remove this check once we no longer support Python 2 + if sys.version_info.major == 3: + import importlib.machinery + import importlib.util + loader = importlib.machinery.SourceFileLoader(module_name, file_path) + spec = importlib.util.spec_from_loader(loader.name, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + else: + import imp + module = imp.load_source(module_name, file_path) + + sys.modules[module_name] = module + + return module