Skip to content
This repository has been archived by the owner on Feb 8, 2024. It is now read-only.

EOS-27348: Utils Mini Provisioning Setup: Removed usage of cortxconf class from utils setup #719

Merged
merged 7 commits into from
Jan 24, 2022
Merged

EOS-27348: Utils Mini Provisioning Setup: Removed usage of cortxconf class from utils setup #719

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
8ecbc0b
Removed usage of cortxconf class from utils setup
suryakumar1024 Jan 20, 2022
e4b92a1
Code optimize
suryakumar1024 Jan 20, 2022
448e6a6
Code opt
suryakumar1024 Jan 21, 2022
cd35d12
Merge branch 'main' into utils_setup_cc_fix
suryakumar1024 Jan 21, 2022
f77185b
Utils setup feedback changes
suryakumar1024 Jan 21, 2022
659e2ae
Code optimize
suryakumar1024 Jan 21, 2022
e01a400
Merge branch 'main' into utils_setup_cc_fix
Jan 24, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 7 additions & 105 deletions py-utils/src/setup/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,66 +31,14 @@
from cortx.utils.validator.error import VError
from cortx.utils.validator.v_confkeys import ConfKeysV
from cortx.utils.service.service_handler import Service
from cortx.utils.common import CortxConf


class Utils:
""" Represents Utils and Performs setup related actions """
utils_path = '/opt/seagate/cortx/utils'

# Utils private methods


@staticmethod
def _delete_files(files: list):
"""
Deletes the passed list of files

Args:
files ([str]): List of files to be deleted

Raises:
SetupError: If unable to delete file
"""
for each_file in files:
if os.path.exists(each_file):
try:
os.remove(each_file)
except OSError as e:
raise SetupError(e.errno, "Error deleting file %s, \
%s", each_file, e)

@staticmethod
def _get_utils_path() -> str:
""" Gets install path from cortx.conf and returns utils path """
install_path = CortxConf.get(key='install_path')
if not install_path:
local_storage_path = CortxConf.get_storage_path('local')
config_url = "%s://%s" % ('json', \
os.path.join(local_storage_path, 'utils/conf/cortx.conf'))
raise SetupError(errno.EINVAL, "install_path not found in %s",\
config_url)
return install_path

def _set_to_conf_file(key, value):
""" Add key value pair to cortx.conf file """
CortxConf.set(key, value)
CortxConf.save()

# Utils private methods
@staticmethod
def _get_from_conf_file(key) -> str:
""" Fetch and return value for the key from cortx.conf file """
val = CortxConf.get(key=key)

if not val:
local_storage_path = CortxConf.get_storage_path('local')
config_url = "%s://%s" % ('json', \
os.path.join(local_storage_path, 'utils/conf/cortx.conf'))
raise SetupError(errno.EINVAL, "Value for key: %s, not found in \
%s", key, config_url)

return val

@staticmethod
def _copy_cluster_map(config_path: str):
Conf.load('cluster', config_path, skip_reload=True)
Expand Down Expand Up @@ -124,24 +72,23 @@ def validate(phase: str):
def post_install(config_path: str):
""" Performs post install operations """
# Check required python packages
install_path = Utils._get_from_conf_file('install_path')
utils_path = install_path + '/cortx/utils'
with open(f"{utils_path}/conf/python_requirements.txt") as file:
# TODO: Remove this python package check as RPM installation
# doing the same.
with open(f"{Utils.utils_path}/conf/python_requirements.txt") as file:
req_pack = []
for package in file.readlines():
pack = package.strip().split('==')
req_pack.append(f"{pack[0]} ({pack[1]})")
try:
with open(f"{utils_path}/conf/python_requirements.ext.txt") as extfile :
with open(f"{Utils.utils_path}/conf/python_requirements.ext.txt") as extfile :
for package in extfile.readlines():
pack = package.strip().split('==')
req_pack.append(f"{pack[0]} ({pack[1]})")
except Exception:
Log.info("Not found: "+f"{utils_path}/conf/python_requirements.ext.txt")
Log.info("Not found: "+f"{Utils.utils_path}/conf/python_requirements.ext.txt")

PkgV().validate(v_type='pip3s', args=req_pack)
default_sb_path = '/var/log/cortx/support_bundle'
Utils._set_to_conf_file('support>local_path', default_sb_path)
os.makedirs(default_sb_path, exist_ok=True)

post_install_template_index = 'post_install_index'
Expand All @@ -162,37 +109,10 @@ def config(config_path: str):
# Load required files
config_template_index = 'config'
Conf.load(config_template_index, config_path)
# Configure log_dir for utils
log_dir = CortxConf.get_storage_path('log')
if log_dir is not None:
CortxConf.set('log_dir', log_dir)
CortxConf.save()

# set cluster nodename:hostname mapping to cluster.conf
Utils._copy_cluster_map(config_path)
Utils._configure_rsyslog()

# get shared storage from cluster.conf and set it to cortx.conf
shared_storage = CortxConf.get_storage_path('shared', none_allowed=True)
if shared_storage:
Utils._set_to_conf_file('support>shared_path', shared_storage)

# temporary fix for a common message bus log file
# The issue happend when some user other than root:root is trying
# to write logs in these log dir/files. This needs to be removed soon!
log_dir = CortxConf.get('log_dir', '/var/log')
utils_log_dir = CortxConf.get_log_path(base_dir=log_dir)
#message_bus
os.makedirs(os.path.join(utils_log_dir, 'message_bus'), exist_ok=True)
os.chmod(os.path.join(utils_log_dir, 'message_bus'), 0o0777)
Path(os.path.join(utils_log_dir,'message_bus/message_bus.log')) \
.touch(exist_ok=True)
os.chmod(os.path.join(utils_log_dir,'message_bus/message_bus.log'), 0o0666)
#iem
os.makedirs(os.path.join(utils_log_dir, 'iem'), exist_ok=True)
os.chmod(os.path.join(utils_log_dir, 'iem'), 0o0777)
Path(os.path.join(utils_log_dir, 'iem/iem.log')).touch(exist_ok=True)
os.chmod(os.path.join(utils_log_dir, 'iem/iem.log'), 0o0666)
return 0

@staticmethod
Expand Down Expand Up @@ -225,13 +145,11 @@ def test(config_path: str, plan: str):
try:
Log.info("Validating cortx-py-utils-test rpm")
PkgV().validate('rpms', ['cortx-py-utils-test'])
install_path = Utils._get_from_conf_file('install_path')
utils_path = install_path + '/cortx/utils'
import cortx.utils.test as test_dir
plan_path = os.path.join(os.path.dirname(test_dir.__file__), \
'plans/', plan + '.pln')
Log.info("Running test plan: %s", plan)
cmd = "%s/bin/run_test -c %s -t %s" %(utils_path, config_path, plan_path)
cmd = "%s/bin/run_test -c %s -t %s" %(Utils.utils_path, config_path, plan_path)
_output, _err, _rc = SimpleProcess(cmd).run(realtime_output=True)
if _rc != 0:
Log.error("Py-utils Test Failed")
Expand Down Expand Up @@ -269,15 +187,6 @@ def reset(config_path: str):
except Exception as e:
raise SetupError(errors.ERR_OP_FAILED, "Internal error, can not \
reset Message Bus. %s", e)
# Clear the logs
utils_log_path = CortxConf.get_log_path()
if os.path.exists(utils_log_path):
cmd = "find %s -type f -name '*.log' -exec truncate -s 0 {} +" % utils_log_path
cmd_proc = SimpleProcess(cmd)
_, stderr, rc = cmd_proc.run()
if rc != 0:
raise SetupError(errors.ERR_OP_FAILED, \
"Can not reset log files. %s", stderr)
return 0

@staticmethod
Expand All @@ -304,13 +213,6 @@ def cleanup(pre_factory: bool, config_path: str):
raise SetupError(errors.ERR_OP_FAILED, "Can not cleanup Message \
Bus. %s", e)

if pre_factory:
# deleting all log files as part of pre-factory cleanup
utils_log_path = CortxConf.get_log_path()
cortx_utils_log_regex = f'{utils_log_path}/**/*.log'
log_files = glob.glob(cortx_utils_log_regex, recursive=True)
Utils._delete_files(log_files)

return 0

@staticmethod
Expand Down
27 changes: 6 additions & 21 deletions py-utils/src/setup/utils_setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
from cortx.setup.utils import Utils
from cortx.utils.log import Log
from cortx.setup.utils import SetupError
from cortx.utils import const


class Cmd:
Expand Down Expand Up @@ -235,33 +236,17 @@ def process(self):


def main():
from cortx.utils.conf_store import Conf
tmpl_file_index = 'tmpl_index'
from cortx.utils.conf_store import Conf, MappedConf
argv = sys.argv

# Get the log path
tmpl_file = argv[3]
from cortx.utils.common import CortxConf
Conf.load(tmpl_file_index, tmpl_file, skip_reload=True)
local_storage_path = Conf.get(tmpl_file_index, 'cortx>common>storage>local')
cortx_config_file = os.path.join(f'{local_storage_path}', 'utils/conf/cortx.conf')
if not os.path.exists(cortx_config_file):
import shutil
# copy local conf file as cortx.conf
try:
os.makedirs(f'{local_storage_path}/utils/conf', exist_ok=True)
shutil.copy('/opt/seagate/cortx/utils/conf/cortx.conf.sample', \
f'{local_storage_path}/utils/conf/cortx.conf')
except OSError as e:
raise SetupError(e.errno, "Failed to create %s %s", \
cortx_config_file, e)

CortxConf.init(cluster_conf=tmpl_file)
log_dir = CortxConf.get_storage_path('log')
utils_log_path = CortxConf.get_log_path(base_dir=log_dir)
Conf.load('cluster_config', tmpl_file)
log_dir = Conf.get('cluster_config', 'cortx>common>storage>log')
utils_log_path = os.path.join(log_dir, f'utils/{Conf.machine_id}')

# Get the log level
log_level = CortxConf.get('utils>log_level', 'INFO')
log_level = Conf.get('cluster_config', 'utils>log_level', 'INFO')

Log.init('utils_setup', utils_log_path, level=log_level, backup_count=5, \
file_size_in_mb=5)
Expand Down