EOS-27348: replace use of CortxConf class to MappedConf (Seagate#716)

* EOS-27348: Remove CortxConf

Signed-off-by: Rohit Dwivedi <[email protected]>

* renamed variable from cluster_conf_mapped to cluster_conf

Signed-off-by: Rohit Dwivedi <[email protected]>

Co-authored-by: Sachin Punadikar <[email protected]>
Signed-off-by: suryakumar.kumaravelan <[email protected]>

py-utils/src/support/cortx_support_bundle.py

@@ -23,7 +23,9 @@
 import os
 import traceback
 from argparse import RawTextHelpFormatter
-from cortx.utils.common.common import CortxConf
+
+from cortx.utils.const import CLUSTER_CONF
+from cortx.utils.conf_store import MappedConf
 from cortx.utils.support_framework import const
 from cortx.utils.support_framework import SupportBundle
 
@@ -174,10 +176,11 @@ def main():
     # Parse and Process Arguments
     try:
         args = parser.parse_args()
-        cluster_conf_path = args.cluster_conf_path[0] if args.cluster_conf_path else 'yaml:///etc/cortx/cluster.conf'
-        CortxConf.init(cluster_conf=cluster_conf_path)
-        log_path = CortxConf.get_log_path('support')
-        log_level = CortxConf.get('utils>log_level', 'INFO')
+        cluster_conf_path = args.cluster_conf_path[0] if args.cluster_conf_path else CLUSTER_CONF
+        cluster_conf = MappedConf(cluster_conf_path)
+        log_path = os.path.join(cluster_conf.get('log_dir'), \
+            f'utils/{Conf.machine_id}/support')
+        log_level = cluster_conf.get('utils>log_level', 'INFO')
         Log.init('support_bundle', log_path, level=log_level, backup_count=5, \
             file_size_in_mb=5)
         out = args.func(args)

py-utils/src/support/utils_support_bundle.py

@@ -23,10 +23,10 @@
 import errno
 import argparse
 
-from cortx.utils.common import CortxConf
-from cortx.utils.conf_store import Conf
 from cortx.utils.errors import UtilsError
+from cortx.utils.const import CLUSTER_CONF
 from cortx.utils.process import SimpleProcess
+from cortx.utils.conf_store import Conf, MappedConf
 
 
 class SupportBundleError(UtilsError):
@@ -43,7 +43,7 @@ class UtilsSupportBundle:
     _tmp_src = '/tmp/cortx/py-utils/'
 
     @staticmethod
-    def generate(bundle_id: str, target_path: str, cluster_conf: str, **filters):
+    def generate(bundle_id: str, target_path: str, cluster_conf_url: str, **filters):
         """ Generate a tar file. """
         duration = filters.get('duration', 'P5D')
         size_limit = filters.get('size_limit', '500MB')
@@ -52,9 +52,9 @@ def generate(bundle_id: str, target_path: str, cluster_conf: str, **filters):
         stacktrace = filters.get('stacktrace', False)
         # TODO process duration, size_limit, binlogs, coredumps and stacktrace
         # Find log dirs
-        CortxConf.init(cluster_conf=cluster_conf)
-        log_base = CortxConf.get_storage_path('log')
-        local_base = CortxConf.get_storage_path('local')
+        cluster_conf = MappedConf(cluster_conf_url)
+        log_base = cluster_conf.get('cortx>common>storage>log')
+        local_base = cluster_conf.get('cortx>common>storage>local')
         machine_id = Conf.machine_id
 
         if os.path.exists(UtilsSupportBundle._tmp_src):
@@ -113,7 +113,7 @@ def parse_args():
             nargs='?', default="/var/seagate/cortx/support_bundle/")
         parser.add_argument('-c', dest='cluster_conf',\
             help="Cluster config file path for Support Bundle",\
-            default='yaml:///etc/cortx/cluster.conf')
+            default=CLUSTER_CONF)
         parser.add_argument('-s', '--services', dest='services', nargs='+',\
             default='', help='List of services for Support Bundle')
         parser.add_argument('-d', '--duration', default='P5D', dest='duration',

py-utils/src/utils/const.py

@@ -15,6 +15,7 @@
 # For any questions about this software or licensing,
 # please email [email protected] or [email protected].
 
+CLUSTER_CONF = 'yaml:///etc/cortx/cluster.conf'
 CORTX_HA_INSTALL_PATH = "/etc/cortx/ha/"
 RULES_FILE_PATH = "rules_engine_schema.json"
 CONF_FILE_PATH = "decision_monitor_conf.json"

py-utils/src/utils/discovery/request_handler.py

@@ -15,23 +15,23 @@
 # For any questions about this software or licensing,
 # please email [email protected] or [email protected].
 
-import errno
 import os
-import psutil
 import re
 import time
+import errno
+import psutil
 from datetime import datetime
 
 from cortx.utils import const
+from cortx.utils.conf_store import MappedConf
+from cortx.utils.kv_store import KvStoreFactory
 from cortx.utils.discovery.error import DiscoveryError
 from cortx.utils.discovery.resource import Resource, ResourceFactory
-from cortx.utils.kv_store import KvStoreFactory
-from cortx.utils.common import CortxConf
 
 # Load cortx common config
 store_type = "json"
-CortxConf.init(cluster_conf='yaml:///etc/cortx/cluster.conf')
-local_storage_path = CortxConf.get_storage_path('local')
+cluster_conf = MappedConf(const.CLUSTER_CONF)
+local_storage_path = cluster_conf.get('cortx>common>storage>local')
 config_url = "%s://%s" % (store_type, os.path.join(local_storage_path, 'utils/conf/cortx.conf'))
 common_config = KvStoreFactory.get_instance(config_url)
 common_config.load()

py-utils/src/utils/setup/kafka/kafka_setup.py

@@ -14,16 +14,18 @@
 # For any questions about this software or licensing,
 # please email [email protected] or [email protected].
 
+import os
 import sys
 import errno
 import argparse
 import inspect
 import traceback
 
 from cortx.utils.log import Log
-from cortx.utils.common import CortxConf
 from cortx.utils.conf_store import Conf
 from cortx.utils.setup.kafka import Kafka
+from cortx.utils.const import CLUSTER_CONF
+from cortx.utils.conf_store import MappedConf
 from cortx.utils.setup.kafka import KafkaSetupError
 
 
@@ -85,7 +87,7 @@ def add_args(parser: str, cls: str, name: str):
         parser1 = parser.add_parser(cls.name, help='setup %s' % name)
         parser1.add_argument('--config', help='Conf Store URL', type=str)
         parser1.add_argument('--cluster_conf', help='cluster.conf url',
-            type=str, default='yaml:///etc/cortx/cluster.conf')
+            type=str, default=CLUSTER_CONF)
         cls._add_extended_args(parser1)
         parser1.add_argument('args', nargs='*', default=[], help='args')
         parser1.set_defaults(command=cls)
@@ -205,10 +207,11 @@ def main(argv: dict):
         Conf.load(kafka_config, command.url)
         kafka_servers = Conf.get(kafka_config, 'cortx>software>kafka>servers')
         # Get log path and initialise Log
-        CortxConf.init(cluster_conf=command.cluster_conf)
-        log_dir = CortxConf.get_storage_path('log')
-        log_path = CortxConf.get_log_path(base_dir=log_dir)
-        log_level = CortxConf.get('utils>log_level', 'INFO')
+        cluster_conf = MappedConf(command.cluster_conf)
+        log_dir = cluster_conf.get('cortx>common>storage>log')
+        log_dir = log_dir if log_dir else cluster_conf.get('log_dir')
+        log_path = os.path.join(log_dir, f'utils/{Conf.machine_id}')
+        log_level = cluster_conf.get('utils>log_level', 'INFO')
         Log.init('kafka_setup', log_path, level=log_level, backup_count=5, \
             file_size_in_mb=5)
 

py-utils/src/utils/setup/openldap/setupcmd.py

@@ -29,7 +29,8 @@
 from cortx.utils.validator.v_path import PathV
 from cortx.utils.process import SimpleProcess
 from cortx.utils.log import Log
-from cortx.utils.common import CortxConf
+from cortx.utils.conf_store import MappedConf
+from cortx.utils.const import CLUSTER_CONF
 
 class OpenldapPROVError(Exception):
 
@@ -51,8 +52,8 @@ class SetupCmd(object):
   rootdn_pass_key = 'cluster_config>rootdn_password'
   cluster_id_key = 'cluster_config>cluster_id'
   Log.init('OpenldapProvisioning','/var/log/cortx/utils/openldap',level='DEBUG')
-  CortxConf.init(cluster_conf='yaml:///etc/cortx/cluster.conf')
-  util_install_path = CortxConf.get('install_path')
+  cluster_conf = MappedConf(CLUSTER_CONF)
+  util_install_path = cluster_conf.get('install_path')
   openldap_prov_config = path.join(util_install_path, "cortx/utils/conf", "openldap_prov_config.yaml")
   openldap_config_file = path.join(util_install_path, "cortx/utils/conf", "openldap_config.yaml")
   utils_tmp_dir = path.join(util_install_path, "cortx/utils/tmp")

py-utils/src/utils/support_framework/bundle_generate.py

@@ -16,16 +16,17 @@
 # please email [email protected] or [email protected].
 
 import os
-import asyncio
 import shutil
-from datetime import datetime
+import asyncio
 from typing import List
-from cortx.utils.schema.payload import Yaml, Tar
-from cortx.utils.support_framework import const
+from datetime import datetime
+
+from cortx.utils.log import Log
 from cortx.utils.process import SimpleProcess
+from cortx.utils.conf_store import MappedConf
+from cortx.utils.support_framework import const
+from cortx.utils.schema.payload import Yaml, Tar
 from cortx.utils.conf_store.conf_store import Conf
-from cortx.utils.log import Log
-from cortx.utils.common import CortxConf
 
 ERROR = 'error'
 INFO = 'info'
@@ -97,7 +98,7 @@ async def _exc_components_cmd(commands: List, bundle_id: str, path: str, \
         for command in commands:
         # SB Framework will not parse additional filters until all the components
         # accept filters in their respective support bundle scripts.
-    
+
         #    Log.info(f"Executing command -> {command} -b {bundle_id} -t {path}"
         #        f" -c {config_url} -s {services} --duration {duration}"
         #        f" --size_limit {size_limit} --binlogs {binlogs}"
@@ -106,7 +107,7 @@ async def _exc_components_cmd(commands: List, bundle_id: str, path: str, \
         #    cmd_proc = SimpleProcess(f"{command} -b {bundle_id} -t {path} -c {config_url}"
         #        f" -s {services} --duration {duration} --size_limit {size_limit}"
         #        f" --binlogs {binlogs} --coredumps {coredumps} --stacktrace {stacktrace}")
-        
+
             Log.info(f"Executing command -> {command} -b {bundle_id} -t {path}"
                 f" -c {config_url} -s {services}")
 
@@ -128,9 +129,10 @@ async def init(bundle_obj, node_id, config_url, **kwargs):
         command:        cli Command Object :type: command
         return:         None
         """
-        CortxConf.init(cluster_conf=config_url)
-        log_path = CortxConf.get_log_path('support')
-        log_level = CortxConf.get('utils>log_level', 'INFO')
+        cluster_conf = MappedConf(config_url)
+        log_path = os.path.join(cluster_conf.get('log_dir'), \
+            f'utils/{Conf.machine_id}/support')
+        log_level = cluster_conf.get('utils>log_level', 'INFO')
         Log.init('support_bundle_node', log_path, level=log_level, \
             backup_count=5, file_size_in_mb=5)
         bundle_id = bundle_obj.bundle_id
@@ -150,7 +152,7 @@ async def init(bundle_obj, node_id, config_url, **kwargs):
             f"{node_name}, {const.SB_COMMENT}: {comment}, "
             f"{const.SB_COMPONENTS}: {components_list}, {const.SOS_COMP}"))
         # Read support_bundle.Yaml and Check's If It Exists.
-        cmd_setup_file = os.path.join(CortxConf.get('install_path'),\
+        cmd_setup_file = os.path.join(cluster_conf.get('install_path'),\
             const.SUPPORT_YAML)
         try:
             support_bundle_config = Yaml(cmd_setup_file).load()

py-utils/src/utils/support_framework/support_bundle.py

@@ -143,7 +143,7 @@ async def _generate_bundle(command):
             raise BundleError(errno.EINVAL, "Bundle ID already exists,"
                 "Please use Unique Bundle ID")
 
-        cortx_config_store = MappedConf(config_url)
+        cluster_conf = MappedConf(config_url)
         # Get Node ID
         node_id = Conf.machine_id
         if node_id is None:
@@ -160,11 +160,11 @@ async def _generate_bundle(command):
         Conf.set(const.SB_INDEX, f'{node_id}>{bundle_id}', data)
         Conf.save(const.SB_INDEX)
 
-        node_name = cortx_config_store.get(f'node>{node_id}>name')
+        node_name = cluster_conf.get(f'node>{node_id}>name')
         Log.info(f'Starting SB Generation on {node_id}:{node_name}')
         # Get required SB size per component
         components_list, service_per_comp = SupportBundle._get_component_and_services(
-            cortx_config_store, node_id, components)
+            cluster_conf, node_id, components)
 
         if not components_list:
             Log.warn(f"No component specified for {node_name} in CORTX config")

py-utils/src/utils/utils_server/utils_server.py

@@ -15,15 +15,17 @@
 # For any questions about this software or licensing,
 # please email [email protected] or [email protected].
 
-import argparse
+import os
 import errno
-from argparse import RawTextHelpFormatter
+import argparse
 from aiohttp import web
+from argparse import RawTextHelpFormatter
 
 from cortx.utils.log import Log
-from cortx.utils.common import CortxConf
 from cortx.utils.conf_store import Conf
 from cortx.utils.errors import UtilsError
+from cortx.utils.const import CLUSTER_CONF
+from cortx.utils.conf_store import MappedConf
 from cortx.utils.message_bus import MessageBus
 
 
@@ -70,24 +72,23 @@ def __init__(self, message_server_endpoints, message_server_port=28300):
     parser = argparse.ArgumentParser(description='Utils server CLI',
         formatter_class=RawTextHelpFormatter)
     parser.add_argument('-c', '--config', dest='cluster_conf',\
-        help="Cluster config file path for Support Bundle",\
-        default='yaml:///etc/cortx/cluster.conf')
+        help="Cluster config file path for Support Bundle", \
+        default=CLUSTER_CONF)
     args=parser.parse_args()
-    cluster_conf = args.cluster_conf
-    Conf.load('config', cluster_conf, skip_reload=True)
-    CortxConf.init(cluster_conf=cluster_conf)
+    cluster_conf_url = args.cluster_conf
+    cluster_conf = MappedConf(cluster_conf_url)
     # Get the log path
-    log_dir = CortxConf.get_storage_path('log')
+    log_dir = cluster_conf.get('cortx>common>storage>log')
     if not log_dir:
         raise UtilsServerError(errno.EINVAL, "Fail to initialize logger."+\
             " Unable to find log_dir path entry")
-    utils_log_path = CortxConf.get_log_path('utils_server', base_dir=log_dir)
+    utils_log_path = os.path.join(log_dir, f'utils/{Conf.machine_id}/utils_server')
     # Get the log level
-    log_level = CortxConf.get('utils>log_level', 'INFO')
+    log_level = cluster_conf.get('utils>log_level', 'INFO')
     Log.init('utils_server', utils_log_path, level=log_level, backup_count=5, \
         file_size_in_mb=5)
-    message_bus_backend = Conf.get('config', 'cortx>utils>message_bus_backend')
-    message_server_endpoints = Conf.get('config',\
-            f'cortx>external>{message_bus_backend}>endpoints')
-    message_server_port = Conf.get('config', 'cortx>utils>message_server_port')
+    message_bus_backend = cluster_conf.get('cortx>utils>message_bus_backend')
+    message_server_endpoints = cluster_conf.get(
+        f'cortx>external>{message_bus_backend}>endpoints')
+    message_server_port = cluster_conf.get('cortx>utils>message_server_port')
     MessageServer(message_server_endpoints, message_server_port)

py-utils/test/discovery/test_discovery.py

@@ -21,14 +21,14 @@
 
 from cortx.utils.conf_store import Conf
 from cortx.utils.discovery import Discovery
-from cortx.utils.discovery.error import DiscoveryError
+from cortx.utils.conf_store import MappedConf
 from cortx.utils.kv_store import KvStoreFactory
-from cortx.utils.common import CortxConf
+from cortx.utils.discovery.error import DiscoveryError
 
 # Load cortx common config
 store_type = "json"
-CortxConf.init(cluster_conf='yaml:///etc/cortx/cluster.conf')
-local_storage_path = CortxConf.get_storage_path('local')
+cluster_conf = MappedConf('yaml:///etc/cortx/cluster.conf')
+local_storage_path = cluster_conf.get('cortx>common>storage>local')
 config_url = "%s://%s" % (store_type, os.path.join(local_storage_path, 'utils/conf/cortx.conf'))
 common_config = KvStoreFactory.get_instance(config_url)
 common_config.load()