From 4f1d97cdb53e1dfcf06588756b4d38089a31ec8c Mon Sep 17 00:00:00 2001 From: mattiagiupponi <51856725+mattiagiupponi@users.noreply.github.com> Date: Wed, 17 Mar 2021 10:00:52 +0100 Subject: [PATCH] [Fixes #7089] Delete existing table on restore command (#7090) * [Fixes #7089] Delete existing table on restore command * [Fixes #7089] Add cascade delete, --soft-restore and preserve_geoserver_resources * [Fixes #7089] Only --soft-reset is mantained --- geonode/br/management/commands/restore.py | 27 ++++++++++++----- geonode/br/management/commands/utils/utils.py | 29 +++++++++++++++++-- 2 files changed, 47 insertions(+), 9 deletions(-) diff --git a/geonode/br/management/commands/restore.py b/geonode/br/management/commands/restore.py index de6ef8dacd1..691d9d2ee52 100755 --- a/geonode/br/management/commands/restore.py +++ b/geonode/br/management/commands/restore.py @@ -152,6 +152,14 @@ def add_arguments(self, parser): help='Skips activation of the Read Only mode in restore procedure execution.' ) + parser.add_argument( + '--soft-reset', + action='store_true', + dest='soft_reset', + default=False, + help='If True, preserve geoserver resources and tables' + ) + def handle(self, **options): skip_read_only = options.get('skip_read_only') config = Configuration.load() @@ -184,6 +192,7 @@ def execute_restore(self, **options): backup_files_dir = options.get('backup_files_dir') with_logs = options.get('with_logs') notify = options.get('notify') + soft_reset = options.get('soft_reset') # choose backup_file from backup_files_dir, if --backup-files-dir was provided if backup_files_dir: @@ -288,10 +297,10 @@ def execute_restore(self, **options): print(("[Sanity Check] Full Write Access to '{}' ...".format(target_folder))) chmod_tree(target_folder) self.restore_geoserver_backup(config, settings, target_folder, - skip_geoserver_info, skip_geoserver_security, ignore_errors) + skip_geoserver_info, skip_geoserver_security, ignore_errors, soft_reset) self.prepare_geoserver_gwc_config(config, settings) self.restore_geoserver_raster_data(config, settings, target_folder) - self.restore_geoserver_vector_data(config, settings, target_folder) + self.restore_geoserver_vector_data(config, settings, target_folder, soft_reset) print("Restoring geoserver external resources") self.restore_geoserver_externals(config, settings, target_folder) except Exception as exception: @@ -299,9 +308,9 @@ def execute_restore(self, **options): with tempfile.TemporaryDirectory(dir=temp_dir_path) as restore_folder: recovery_folder = extract_archive(recovery_file, restore_folder) self.restore_geoserver_backup(config, settings, recovery_folder, - skip_geoserver_info, skip_geoserver_security, ignore_errors) + skip_geoserver_info, skip_geoserver_security, ignore_errors, soft_reset) self.restore_geoserver_raster_data(config, settings, recovery_folder) - self.restore_geoserver_vector_data(config, settings, recovery_folder) + self.restore_geoserver_vector_data(config, settings, recovery_folder, soft_reset) self.restore_geoserver_externals(config, settings, recovery_folder) if notify: restore_notification.apply_async( @@ -613,7 +622,7 @@ def check_backup_ini_settings(self, backup_file: str) -> str: return None - def restore_geoserver_backup(self, config, settings, target_folder, skip_geoserver_info, skip_geoserver_security, ignore_errors): + def restore_geoserver_backup(self, config, settings, target_folder, skip_geoserver_info, skip_geoserver_security, ignore_errors, soft_reset): """Restore GeoServer Catalog""" url = settings.OGC_SERVER['default']['LOCATION'] user = settings.OGC_SERVER['default']['USER'] @@ -628,6 +637,7 @@ def restore_geoserver_backup(self, config, settings, target_folder, skip_geoserv # Best Effort Restore: 'options': {'option': ['BK_BEST_EFFORT=true']} _options = [ + 'BK_PURGE_RESOURCES={}'.format('true' if not soft_reset else 'false'), 'BK_CLEANUP_TEMP=true', 'BK_SKIP_SETTINGS={}'.format('true' if skip_geoserver_info else 'false'), 'BK_SKIP_SECURITY={}'.format('true' if skip_geoserver_security else 'false'), @@ -759,7 +769,7 @@ def restore_geoserver_raster_data(self, config, settings, target_folder): print(('Skipping geoserver raster data restore: ' + 'directory "{}" not found.'.format(gs_data_folder))) - def restore_geoserver_vector_data(self, config, settings, target_folder): + def restore_geoserver_vector_data(self, config, settings, target_folder, soft_reset): """Restore Vectorial Data from DB""" if (config.gs_dump_vector_data): @@ -777,8 +787,11 @@ def restore_geoserver_vector_data(self, config, settings, target_folder): ogc_db_host = settings.DATABASES[datastore]['HOST'] ogc_db_port = settings.DATABASES[datastore]['PORT'] + if not soft_reset: + utils.remove_existing_tables(ogc_db_name, ogc_db_user, ogc_db_port, ogc_db_host, ogc_db_passwd) + utils.restore_db(config, ogc_db_name, ogc_db_user, ogc_db_port, - ogc_db_host, ogc_db_passwd, gs_data_folder) + ogc_db_host, ogc_db_passwd, gs_data_folder, soft_reset) def restore_geoserver_externals(self, config, settings, target_folder): """Restore external references from XML files""" diff --git a/geonode/br/management/commands/utils/utils.py b/geonode/br/management/commands/utils/utils.py index 9d31fe967a9..537b8a7d7d8 100644 --- a/geonode/br/management/commands/utils/utils.py +++ b/geonode/br/management/commands/utils/utils.py @@ -289,7 +289,7 @@ def dump_db(config, db_name, db_user, db_port, db_host, db_passwd, target_folder conn.commit() -def restore_db(config, db_name, db_user, db_port, db_host, db_passwd, source_folder): +def restore_db(config, db_name, db_user, db_port, db_host, db_passwd, source_folder, preserve_tables): """Restore Full DB into target folder""" db_host = db_host if db_host is not None else 'localhost' db_port = db_port if db_port is not None else 5432 @@ -302,10 +302,11 @@ def restore_db(config, db_name, db_user, db_port, db_host, db_passwd, source_fol if any(fn.endswith(ext) for ext in included_extenstions)] for table in file_names: logger.info("Restoring GeoServer Vectorial Data : {}:{} ".format(db_name, os.path.splitext(table)[0])) - pg_rstcmd = 'PGPASSWORD="' + db_passwd + '" ' + config.pg_restore_cmd + ' -c -h ' + db_host + \ + pg_rstcmd = 'PGPASSWORD="' + db_passwd + '" ' + config.pg_restore_cmd + ' -h ' + db_host + \ ' -p ' + str(db_port) + ' -U ' + db_user + ' --role=' + db_user + \ ' -F c -t "' + os.path.splitext(table)[0] + '" ' +\ os.path.join(source_folder, table) + ' -d ' + db_name + pg_rstcmd += " -c" if preserve_tables else "" os.system(pg_rstcmd) except Exception: @@ -319,6 +320,30 @@ def restore_db(config, db_name, db_user, db_port, db_host, db_passwd, source_fol conn.commit() +def remove_existing_tables(db_name, db_user, db_port, db_host, db_passwd): + conn = get_db_conn(db_name, db_user, db_port, db_host, db_passwd) + curs = conn.cursor() + table_list = """SELECT tablename from pg_tables where tableowner = '%s'""" % (db_user) + + try: + curs.execute(table_list) + pg_all_tables = [table[0] for table in curs.fetchall()] + for pg_table in pg_all_tables: + logger.info("Dropping existing GeoServer Vectorial Data : {}:{} ".format(db_name, pg_table)) + curs.execute(f"DROP TABLE {pg_table} CASCADE") + + conn.commit() + except Exception: + try: + conn.rollback() + except Exception: + pass + + traceback.print_exc() + curs.close() + conn.close() + + def confirm(prompt=None, resp=False): """prompts for yes or no response from the user. Returns True for yes and False for no.