Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Fixes #7089] Delete existing table on restore command #7090

Merged
merged 3 commits into from
Mar 17, 2021
Merged

[Fixes #7089] Delete existing table on restore command #7090

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
dc93266
[Fixes #7089] Delete existing table on restore command
mattiagiupponi Mar 16, 2021
49b0ffd
[Fixes #7089] Add cascade delete, --soft-restore and preserve_geoserv…
mattiagiupponi Mar 16, 2021
5043136
[Fixes #7089] Only --soft-reset is mantained
mattiagiupponi Mar 16, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 20 additions & 7 deletions geonode/br/management/commands/restore.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,14 @@ def add_arguments(self, parser):
help='Skips activation of the Read Only mode in restore procedure execution.'
)

parser.add_argument(
'--soft-reset',
action='store_true',
dest='soft_reset',
default=False,
help='If True, preserve geoserver resources and tables'
)

def handle(self, **options):
skip_read_only = options.get('skip_read_only')
config = Configuration.load()
Expand Down Expand Up @@ -184,6 +192,7 @@ def execute_restore(self, **options):
backup_files_dir = options.get('backup_files_dir')
with_logs = options.get('with_logs')
notify = options.get('notify')
soft_reset = options.get('soft_reset')

# choose backup_file from backup_files_dir, if --backup-files-dir was provided
if backup_files_dir:
Expand Down Expand Up @@ -288,20 +297,20 @@ def execute_restore(self, **options):
print(("[Sanity Check] Full Write Access to '{}' ...".format(target_folder)))
chmod_tree(target_folder)
self.restore_geoserver_backup(config, settings, target_folder,
skip_geoserver_info, skip_geoserver_security, ignore_errors)
skip_geoserver_info, skip_geoserver_security, ignore_errors, soft_reset)
self.prepare_geoserver_gwc_config(config, settings)
self.restore_geoserver_raster_data(config, settings, target_folder)
self.restore_geoserver_vector_data(config, settings, target_folder)
self.restore_geoserver_vector_data(config, settings, target_folder, soft_reset)
print("Restoring geoserver external resources")
self.restore_geoserver_externals(config, settings, target_folder)
except Exception as exception:
if recovery_file:
with tempfile.TemporaryDirectory(dir=temp_dir_path) as restore_folder:
recovery_folder = extract_archive(recovery_file, restore_folder)
self.restore_geoserver_backup(config, settings, recovery_folder,
skip_geoserver_info, skip_geoserver_security, ignore_errors)
skip_geoserver_info, skip_geoserver_security, ignore_errors, soft_reset)
self.restore_geoserver_raster_data(config, settings, recovery_folder)
self.restore_geoserver_vector_data(config, settings, recovery_folder)
self.restore_geoserver_vector_data(config, settings, recovery_folder, soft_reset)
self.restore_geoserver_externals(config, settings, recovery_folder)
if notify:
restore_notification.apply_async(
Expand Down Expand Up @@ -613,7 +622,7 @@ def check_backup_ini_settings(self, backup_file: str) -> str:

return None

def restore_geoserver_backup(self, config, settings, target_folder, skip_geoserver_info, skip_geoserver_security, ignore_errors):
def restore_geoserver_backup(self, config, settings, target_folder, skip_geoserver_info, skip_geoserver_security, ignore_errors, soft_reset):
"""Restore GeoServer Catalog"""
url = settings.OGC_SERVER['default']['LOCATION']
user = settings.OGC_SERVER['default']['USER']
Expand All @@ -628,6 +637,7 @@ def restore_geoserver_backup(self, config, settings, target_folder, skip_geoserv

# Best Effort Restore: 'options': {'option': ['BK_BEST_EFFORT=true']}
_options = [
'BK_PURGE_RESOURCES={}'.format('true' if not soft_reset else 'false'),
'BK_CLEANUP_TEMP=true',
'BK_SKIP_SETTINGS={}'.format('true' if skip_geoserver_info else 'false'),
'BK_SKIP_SECURITY={}'.format('true' if skip_geoserver_security else 'false'),
Expand Down Expand Up @@ -759,7 +769,7 @@ def restore_geoserver_raster_data(self, config, settings, target_folder):
print(('Skipping geoserver raster data restore: ' +
'directory "{}" not found.'.format(gs_data_folder)))

def restore_geoserver_vector_data(self, config, settings, target_folder):
def restore_geoserver_vector_data(self, config, settings, target_folder, soft_reset):
"""Restore Vectorial Data from DB"""
if (config.gs_dump_vector_data):

Expand All @@ -777,8 +787,11 @@ def restore_geoserver_vector_data(self, config, settings, target_folder):
ogc_db_host = settings.DATABASES[datastore]['HOST']
ogc_db_port = settings.DATABASES[datastore]['PORT']

if not soft_reset:
utils.remove_existing_tables(ogc_db_name, ogc_db_user, ogc_db_port, ogc_db_host, ogc_db_passwd)

utils.restore_db(config, ogc_db_name, ogc_db_user, ogc_db_port,
ogc_db_host, ogc_db_passwd, gs_data_folder)
ogc_db_host, ogc_db_passwd, gs_data_folder, soft_reset)

def restore_geoserver_externals(self, config, settings, target_folder):
"""Restore external references from XML files"""
Expand Down
29 changes: 27 additions & 2 deletions geonode/br/management/commands/utils/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -289,7 +289,7 @@ def dump_db(config, db_name, db_user, db_port, db_host, db_passwd, target_folder
conn.commit()


def restore_db(config, db_name, db_user, db_port, db_host, db_passwd, source_folder):
def restore_db(config, db_name, db_user, db_port, db_host, db_passwd, source_folder, preserve_tables):
"""Restore Full DB into target folder"""
db_host = db_host if db_host is not None else 'localhost'
db_port = db_port if db_port is not None else 5432
Expand All @@ -302,10 +302,11 @@ def restore_db(config, db_name, db_user, db_port, db_host, db_passwd, source_fol
if any(fn.endswith(ext) for ext in included_extenstions)]
for table in file_names:
logger.info("Restoring GeoServer Vectorial Data : {}:{} ".format(db_name, os.path.splitext(table)[0]))
pg_rstcmd = 'PGPASSWORD="' + db_passwd + '" ' + config.pg_restore_cmd + ' -c -h ' + db_host + \
pg_rstcmd = 'PGPASSWORD="' + db_passwd + '" ' + config.pg_restore_cmd + ' -h ' + db_host + \
' -p ' + str(db_port) + ' -U ' + db_user + ' --role=' + db_user + \
' -F c -t "' + os.path.splitext(table)[0] + '" ' +\
os.path.join(source_folder, table) + ' -d ' + db_name
pg_rstcmd += " -c" if preserve_tables else ""
os.system(pg_rstcmd)

except Exception:
Expand All @@ -319,6 +320,30 @@ def restore_db(config, db_name, db_user, db_port, db_host, db_passwd, source_fol
conn.commit()


def remove_existing_tables(db_name, db_user, db_port, db_host, db_passwd):
conn = get_db_conn(db_name, db_user, db_port, db_host, db_passwd)
curs = conn.cursor()
table_list = """SELECT tablename from pg_tables where tableowner = '%s'""" % (db_user)

try:
curs.execute(table_list)
pg_all_tables = [table[0] for table in curs.fetchall()]
for pg_table in pg_all_tables:
logger.info("Dropping existing GeoServer Vectorial Data : {}:{} ".format(db_name, pg_table))
curs.execute(f"DROP TABLE {pg_table} CASCADE")

conn.commit()
except Exception:
try:
conn.rollback()
except Exception:
pass

traceback.print_exc()
curs.close()
conn.close()


def confirm(prompt=None, resp=False):
"""prompts for yes or no response from the user. Returns True for yes and
False for no.
Expand Down