From c37b3ea77be7125c23c24d4c54c568287d3c6870 Mon Sep 17 00:00:00 2001 From: rohit-k-dwivedi <83213072+rohit-k-dwivedi@users.noreply.github.com> Date: Fri, 21 Jan 2022 19:53:39 +0530 Subject: [PATCH] EOS-27348: replace use of CortxConf class to MappedConf (#716) * EOS-27348: Remove CortxConf Signed-off-by: Rohit Dwivedi * renamed variable from cluster_conf_mapped to cluster_conf Signed-off-by: Rohit Dwivedi Co-authored-by: Sachin Punadikar Signed-off-by: suryakumar.kumaravelan --- py-utils/src/support/cortx_support_bundle.py | 13 +++++--- py-utils/src/support/utils_support_bundle.py | 14 ++++----- py-utils/src/utils/const.py | 1 + .../src/utils/discovery/request_handler.py | 12 +++---- py-utils/src/utils/setup/kafka/kafka_setup.py | 15 +++++---- py-utils/src/utils/setup/openldap/setupcmd.py | 7 +++-- .../support_framework/bundle_generate.py | 26 +++++++++------- .../utils/support_framework/support_bundle.py | 6 ++-- .../src/utils/utils_server/utils_server.py | 31 ++++++++++--------- py-utils/test/discovery/test_discovery.py | 8 ++--- 10 files changed, 72 insertions(+), 61 deletions(-) diff --git a/py-utils/src/support/cortx_support_bundle.py b/py-utils/src/support/cortx_support_bundle.py index e929cf0fa..95f927c1b 100644 --- a/py-utils/src/support/cortx_support_bundle.py +++ b/py-utils/src/support/cortx_support_bundle.py @@ -23,7 +23,9 @@ import os import traceback from argparse import RawTextHelpFormatter -from cortx.utils.common.common import CortxConf + +from cortx.utils.const import CLUSTER_CONF +from cortx.utils.conf_store import MappedConf from cortx.utils.support_framework import const from cortx.utils.support_framework import SupportBundle @@ -174,10 +176,11 @@ def main(): # Parse and Process Arguments try: args = parser.parse_args() - cluster_conf_path = args.cluster_conf_path[0] if args.cluster_conf_path else 'yaml:///etc/cortx/cluster.conf' - CortxConf.init(cluster_conf=cluster_conf_path) - log_path = CortxConf.get_log_path('support') - log_level = CortxConf.get('utils>log_level', 'INFO') + cluster_conf_path = args.cluster_conf_path[0] if args.cluster_conf_path else CLUSTER_CONF + cluster_conf = MappedConf(cluster_conf_path) + log_path = os.path.join(cluster_conf.get('log_dir'), \ + f'utils/{Conf.machine_id}/support') + log_level = cluster_conf.get('utils>log_level', 'INFO') Log.init('support_bundle', log_path, level=log_level, backup_count=5, \ file_size_in_mb=5) out = args.func(args) diff --git a/py-utils/src/support/utils_support_bundle.py b/py-utils/src/support/utils_support_bundle.py index 339e825d8..863103359 100644 --- a/py-utils/src/support/utils_support_bundle.py +++ b/py-utils/src/support/utils_support_bundle.py @@ -23,10 +23,10 @@ import errno import argparse -from cortx.utils.common import CortxConf -from cortx.utils.conf_store import Conf from cortx.utils.errors import UtilsError +from cortx.utils.const import CLUSTER_CONF from cortx.utils.process import SimpleProcess +from cortx.utils.conf_store import Conf, MappedConf class SupportBundleError(UtilsError): @@ -43,7 +43,7 @@ class UtilsSupportBundle: _tmp_src = '/tmp/cortx/py-utils/' @staticmethod - def generate(bundle_id: str, target_path: str, cluster_conf: str, **filters): + def generate(bundle_id: str, target_path: str, cluster_conf_url: str, **filters): """ Generate a tar file. """ duration = filters.get('duration', 'P5D') size_limit = filters.get('size_limit', '500MB') @@ -52,9 +52,9 @@ def generate(bundle_id: str, target_path: str, cluster_conf: str, **filters): stacktrace = filters.get('stacktrace', False) # TODO process duration, size_limit, binlogs, coredumps and stacktrace # Find log dirs - CortxConf.init(cluster_conf=cluster_conf) - log_base = CortxConf.get_storage_path('log') - local_base = CortxConf.get_storage_path('local') + cluster_conf = MappedConf(cluster_conf_url) + log_base = cluster_conf.get('cortx>common>storage>log') + local_base = cluster_conf.get('cortx>common>storage>local') machine_id = Conf.machine_id if os.path.exists(UtilsSupportBundle._tmp_src): @@ -113,7 +113,7 @@ def parse_args(): nargs='?', default="/var/seagate/cortx/support_bundle/") parser.add_argument('-c', dest='cluster_conf',\ help="Cluster config file path for Support Bundle",\ - default='yaml:///etc/cortx/cluster.conf') + default=CLUSTER_CONF) parser.add_argument('-s', '--services', dest='services', nargs='+',\ default='', help='List of services for Support Bundle') parser.add_argument('-d', '--duration', default='P5D', dest='duration', diff --git a/py-utils/src/utils/const.py b/py-utils/src/utils/const.py index 71231de20..a40ae83eb 100644 --- a/py-utils/src/utils/const.py +++ b/py-utils/src/utils/const.py @@ -15,6 +15,7 @@ # For any questions about this software or licensing, # please email opensource@seagate.com or cortx-questions@seagate.com. +CLUSTER_CONF = 'yaml:///etc/cortx/cluster.conf' CORTX_HA_INSTALL_PATH = "/etc/cortx/ha/" RULES_FILE_PATH = "rules_engine_schema.json" CONF_FILE_PATH = "decision_monitor_conf.json" diff --git a/py-utils/src/utils/discovery/request_handler.py b/py-utils/src/utils/discovery/request_handler.py index 41b98f0dc..205147ec7 100644 --- a/py-utils/src/utils/discovery/request_handler.py +++ b/py-utils/src/utils/discovery/request_handler.py @@ -15,23 +15,23 @@ # For any questions about this software or licensing, # please email opensource@seagate.com or cortx-questions@seagate.com. -import errno import os -import psutil import re import time +import errno +import psutil from datetime import datetime from cortx.utils import const +from cortx.utils.conf_store import MappedConf +from cortx.utils.kv_store import KvStoreFactory from cortx.utils.discovery.error import DiscoveryError from cortx.utils.discovery.resource import Resource, ResourceFactory -from cortx.utils.kv_store import KvStoreFactory -from cortx.utils.common import CortxConf # Load cortx common config store_type = "json" -CortxConf.init(cluster_conf='yaml:///etc/cortx/cluster.conf') -local_storage_path = CortxConf.get_storage_path('local') +cluster_conf = MappedConf(const.CLUSTER_CONF) +local_storage_path = cluster_conf.get('cortx>common>storage>local') config_url = "%s://%s" % (store_type, os.path.join(local_storage_path, 'utils/conf/cortx.conf')) common_config = KvStoreFactory.get_instance(config_url) common_config.load() diff --git a/py-utils/src/utils/setup/kafka/kafka_setup.py b/py-utils/src/utils/setup/kafka/kafka_setup.py index b024a1df7..1d91538c0 100755 --- a/py-utils/src/utils/setup/kafka/kafka_setup.py +++ b/py-utils/src/utils/setup/kafka/kafka_setup.py @@ -14,6 +14,7 @@ # For any questions about this software or licensing, # please email opensource@seagate.com or cortx-questions@seagate.com. +import os import sys import errno import argparse @@ -21,9 +22,10 @@ import traceback from cortx.utils.log import Log -from cortx.utils.common import CortxConf from cortx.utils.conf_store import Conf from cortx.utils.setup.kafka import Kafka +from cortx.utils.const import CLUSTER_CONF +from cortx.utils.conf_store import MappedConf from cortx.utils.setup.kafka import KafkaSetupError @@ -85,7 +87,7 @@ def add_args(parser: str, cls: str, name: str): parser1 = parser.add_parser(cls.name, help='setup %s' % name) parser1.add_argument('--config', help='Conf Store URL', type=str) parser1.add_argument('--cluster_conf', help='cluster.conf url', - type=str, default='yaml:///etc/cortx/cluster.conf') + type=str, default=CLUSTER_CONF) cls._add_extended_args(parser1) parser1.add_argument('args', nargs='*', default=[], help='args') parser1.set_defaults(command=cls) @@ -205,10 +207,11 @@ def main(argv: dict): Conf.load(kafka_config, command.url) kafka_servers = Conf.get(kafka_config, 'cortx>software>kafka>servers') # Get log path and initialise Log - CortxConf.init(cluster_conf=command.cluster_conf) - log_dir = CortxConf.get_storage_path('log') - log_path = CortxConf.get_log_path(base_dir=log_dir) - log_level = CortxConf.get('utils>log_level', 'INFO') + cluster_conf = MappedConf(command.cluster_conf) + log_dir = cluster_conf.get('cortx>common>storage>log') + log_dir = log_dir if log_dir else cluster_conf.get('log_dir') + log_path = os.path.join(log_dir, f'utils/{Conf.machine_id}') + log_level = cluster_conf.get('utils>log_level', 'INFO') Log.init('kafka_setup', log_path, level=log_level, backup_count=5, \ file_size_in_mb=5) diff --git a/py-utils/src/utils/setup/openldap/setupcmd.py b/py-utils/src/utils/setup/openldap/setupcmd.py index e78cd0336..ca7c59f8e 100644 --- a/py-utils/src/utils/setup/openldap/setupcmd.py +++ b/py-utils/src/utils/setup/openldap/setupcmd.py @@ -29,7 +29,8 @@ from cortx.utils.validator.v_path import PathV from cortx.utils.process import SimpleProcess from cortx.utils.log import Log -from cortx.utils.common import CortxConf +from cortx.utils.conf_store import MappedConf +from cortx.utils.const import CLUSTER_CONF class OpenldapPROVError(Exception): @@ -51,8 +52,8 @@ class SetupCmd(object): rootdn_pass_key = 'cluster_config>rootdn_password' cluster_id_key = 'cluster_config>cluster_id' Log.init('OpenldapProvisioning','/var/log/cortx/utils/openldap',level='DEBUG') - CortxConf.init(cluster_conf='yaml:///etc/cortx/cluster.conf') - util_install_path = CortxConf.get('install_path') + cluster_conf = MappedConf(CLUSTER_CONF) + util_install_path = cluster_conf.get('install_path') openldap_prov_config = path.join(util_install_path, "cortx/utils/conf", "openldap_prov_config.yaml") openldap_config_file = path.join(util_install_path, "cortx/utils/conf", "openldap_config.yaml") utils_tmp_dir = path.join(util_install_path, "cortx/utils/tmp") diff --git a/py-utils/src/utils/support_framework/bundle_generate.py b/py-utils/src/utils/support_framework/bundle_generate.py index 5ff8a3170..8013ef8ed 100644 --- a/py-utils/src/utils/support_framework/bundle_generate.py +++ b/py-utils/src/utils/support_framework/bundle_generate.py @@ -16,16 +16,17 @@ # please email opensource@seagate.com or cortx-questions@seagate.com. import os -import asyncio import shutil -from datetime import datetime +import asyncio from typing import List -from cortx.utils.schema.payload import Yaml, Tar -from cortx.utils.support_framework import const +from datetime import datetime + +from cortx.utils.log import Log from cortx.utils.process import SimpleProcess +from cortx.utils.conf_store import MappedConf +from cortx.utils.support_framework import const +from cortx.utils.schema.payload import Yaml, Tar from cortx.utils.conf_store.conf_store import Conf -from cortx.utils.log import Log -from cortx.utils.common import CortxConf ERROR = 'error' INFO = 'info' @@ -97,7 +98,7 @@ async def _exc_components_cmd(commands: List, bundle_id: str, path: str, \ for command in commands: # SB Framework will not parse additional filters until all the components # accept filters in their respective support bundle scripts. - + # Log.info(f"Executing command -> {command} -b {bundle_id} -t {path}" # f" -c {config_url} -s {services} --duration {duration}" # f" --size_limit {size_limit} --binlogs {binlogs}" @@ -106,7 +107,7 @@ async def _exc_components_cmd(commands: List, bundle_id: str, path: str, \ # cmd_proc = SimpleProcess(f"{command} -b {bundle_id} -t {path} -c {config_url}" # f" -s {services} --duration {duration} --size_limit {size_limit}" # f" --binlogs {binlogs} --coredumps {coredumps} --stacktrace {stacktrace}") - + Log.info(f"Executing command -> {command} -b {bundle_id} -t {path}" f" -c {config_url} -s {services}") @@ -128,9 +129,10 @@ async def init(bundle_obj, node_id, config_url, **kwargs): command: cli Command Object :type: command return: None """ - CortxConf.init(cluster_conf=config_url) - log_path = CortxConf.get_log_path('support') - log_level = CortxConf.get('utils>log_level', 'INFO') + cluster_conf = MappedConf(config_url) + log_path = os.path.join(cluster_conf.get('log_dir'), \ + f'utils/{Conf.machine_id}/support') + log_level = cluster_conf.get('utils>log_level', 'INFO') Log.init('support_bundle_node', log_path, level=log_level, \ backup_count=5, file_size_in_mb=5) bundle_id = bundle_obj.bundle_id @@ -150,7 +152,7 @@ async def init(bundle_obj, node_id, config_url, **kwargs): f"{node_name}, {const.SB_COMMENT}: {comment}, " f"{const.SB_COMPONENTS}: {components_list}, {const.SOS_COMP}")) # Read support_bundle.Yaml and Check's If It Exists. - cmd_setup_file = os.path.join(CortxConf.get('install_path'),\ + cmd_setup_file = os.path.join(cluster_conf.get('install_path'),\ const.SUPPORT_YAML) try: support_bundle_config = Yaml(cmd_setup_file).load() diff --git a/py-utils/src/utils/support_framework/support_bundle.py b/py-utils/src/utils/support_framework/support_bundle.py index e8975f9c9..be9cb30f4 100644 --- a/py-utils/src/utils/support_framework/support_bundle.py +++ b/py-utils/src/utils/support_framework/support_bundle.py @@ -143,7 +143,7 @@ async def _generate_bundle(command): raise BundleError(errno.EINVAL, "Bundle ID already exists," "Please use Unique Bundle ID") - cortx_config_store = MappedConf(config_url) + cluster_conf = MappedConf(config_url) # Get Node ID node_id = Conf.machine_id if node_id is None: @@ -160,11 +160,11 @@ async def _generate_bundle(command): Conf.set(const.SB_INDEX, f'{node_id}>{bundle_id}', data) Conf.save(const.SB_INDEX) - node_name = cortx_config_store.get(f'node>{node_id}>name') + node_name = cluster_conf.get(f'node>{node_id}>name') Log.info(f'Starting SB Generation on {node_id}:{node_name}') # Get required SB size per component components_list, service_per_comp = SupportBundle._get_component_and_services( - cortx_config_store, node_id, components) + cluster_conf, node_id, components) if not components_list: Log.warn(f"No component specified for {node_name} in CORTX config") diff --git a/py-utils/src/utils/utils_server/utils_server.py b/py-utils/src/utils/utils_server/utils_server.py index 78ce4f09c..27d98928c 100644 --- a/py-utils/src/utils/utils_server/utils_server.py +++ b/py-utils/src/utils/utils_server/utils_server.py @@ -15,15 +15,17 @@ # For any questions about this software or licensing, # please email opensource@seagate.com or cortx-questions@seagate.com. -import argparse +import os import errno -from argparse import RawTextHelpFormatter +import argparse from aiohttp import web +from argparse import RawTextHelpFormatter from cortx.utils.log import Log -from cortx.utils.common import CortxConf from cortx.utils.conf_store import Conf from cortx.utils.errors import UtilsError +from cortx.utils.const import CLUSTER_CONF +from cortx.utils.conf_store import MappedConf from cortx.utils.message_bus import MessageBus @@ -70,24 +72,23 @@ def __init__(self, message_server_endpoints, message_server_port=28300): parser = argparse.ArgumentParser(description='Utils server CLI', formatter_class=RawTextHelpFormatter) parser.add_argument('-c', '--config', dest='cluster_conf',\ - help="Cluster config file path for Support Bundle",\ - default='yaml:///etc/cortx/cluster.conf') + help="Cluster config file path for Support Bundle", \ + default=CLUSTER_CONF) args=parser.parse_args() - cluster_conf = args.cluster_conf - Conf.load('config', cluster_conf, skip_reload=True) - CortxConf.init(cluster_conf=cluster_conf) + cluster_conf_url = args.cluster_conf + cluster_conf = MappedConf(cluster_conf_url) # Get the log path - log_dir = CortxConf.get_storage_path('log') + log_dir = cluster_conf.get('cortx>common>storage>log') if not log_dir: raise UtilsServerError(errno.EINVAL, "Fail to initialize logger."+\ " Unable to find log_dir path entry") - utils_log_path = CortxConf.get_log_path('utils_server', base_dir=log_dir) + utils_log_path = os.path.join(log_dir, f'utils/{Conf.machine_id}/utils_server') # Get the log level - log_level = CortxConf.get('utils>log_level', 'INFO') + log_level = cluster_conf.get('utils>log_level', 'INFO') Log.init('utils_server', utils_log_path, level=log_level, backup_count=5, \ file_size_in_mb=5) - message_bus_backend = Conf.get('config', 'cortx>utils>message_bus_backend') - message_server_endpoints = Conf.get('config',\ - f'cortx>external>{message_bus_backend}>endpoints') - message_server_port = Conf.get('config', 'cortx>utils>message_server_port') + message_bus_backend = cluster_conf.get('cortx>utils>message_bus_backend') + message_server_endpoints = cluster_conf.get( + f'cortx>external>{message_bus_backend}>endpoints') + message_server_port = cluster_conf.get('cortx>utils>message_server_port') MessageServer(message_server_endpoints, message_server_port) \ No newline at end of file diff --git a/py-utils/test/discovery/test_discovery.py b/py-utils/test/discovery/test_discovery.py index eadb735da..d4f5f16f4 100644 --- a/py-utils/test/discovery/test_discovery.py +++ b/py-utils/test/discovery/test_discovery.py @@ -21,14 +21,14 @@ from cortx.utils.conf_store import Conf from cortx.utils.discovery import Discovery -from cortx.utils.discovery.error import DiscoveryError +from cortx.utils.conf_store import MappedConf from cortx.utils.kv_store import KvStoreFactory -from cortx.utils.common import CortxConf +from cortx.utils.discovery.error import DiscoveryError # Load cortx common config store_type = "json" -CortxConf.init(cluster_conf='yaml:///etc/cortx/cluster.conf') -local_storage_path = CortxConf.get_storage_path('local') +cluster_conf = MappedConf('yaml:///etc/cortx/cluster.conf') +local_storage_path = cluster_conf.get('cortx>common>storage>local') config_url = "%s://%s" % (store_type, os.path.join(local_storage_path, 'utils/conf/cortx.conf')) common_config = KvStoreFactory.get_instance(config_url) common_config.load()