Skip to content

Commit

Permalink
[Fixes GeoNode#7089] Delete existing table on restore command (GeoNod…
Browse files Browse the repository at this point in the history 
…e#7090)

* [Fixes GeoNode#7089] Delete existing table on restore command

* [Fixes GeoNode#7089] Add cascade delete, --soft-restore and preserve_geoserver_resources

* [Fixes GeoNode#7089] Only --soft-reset is mantained

(cherry picked from commit 4f1d97c)

# Conflicts:
#	geonode/br/management/commands/restore.py
#	geonode/br/management/commands/utils/utils.py
  • Loading branch information
@mattiagiupponi @afabiani
mattiagiupponi authored and afabiani committed Mar 17, 2021
1 parent 445dc3e commit c6bfd80
Show file tree
Hide file tree
Showing 2 changed files with 60 additions and 18 deletions.
36 changes: 26 additions & 10 deletions geonode/br/management/commands/restore.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import requests
import tempfile
import warnings
import logging
from typing import Union
from datetime import datetime

Expand All @@ -52,6 +53,9 @@
from django.db.utils import IntegrityError


logger = logging.getLogger(__name__)


class Command(BaseCommand):

help = 'Restore the GeoNode application data'
Expand Down Expand Up @@ -148,6 +152,14 @@ def add_arguments(self, parser):
help='Skips activation of the Read Only mode in restore procedure execution.'
)

parser.add_argument(
'--soft-reset',
action='store_true',
dest='soft_reset',
default=False,
help='If True, preserve geoserver resources and tables'
)

def handle(self, **options):
skip_read_only = options.get('skip_read_only')
config = Configuration.load()
Expand Down Expand Up @@ -180,6 +192,7 @@ def execute_restore(self, **options):
backup_files_dir = options.get('backup_files_dir')
with_logs = options.get('with_logs')
notify = options.get('notify')
soft_reset = options.get('soft_reset')

# choose backup_file from backup_files_dir, if --backup-files-dir was provided
if backup_files_dir:
Expand Down Expand Up @@ -284,20 +297,20 @@ def execute_restore(self, **options):
print(("[Sanity Check] Full Write Access to '{}' ...".format(target_folder)))
chmod_tree(target_folder)
self.restore_geoserver_backup(config, settings, target_folder,
skip_geoserver_info, skip_geoserver_security, ignore_errors)
skip_geoserver_info, skip_geoserver_security, ignore_errors, soft_reset)
self.prepare_geoserver_gwc_config(config, settings)
self.restore_geoserver_raster_data(config, settings, target_folder)
self.restore_geoserver_vector_data(config, settings, target_folder)
self.restore_geoserver_vector_data(config, settings, target_folder, soft_reset)
print("Restoring geoserver external resources")
self.restore_geoserver_externals(config, settings, target_folder)
except Exception as exception:
if recovery_file:
with tempfile.TemporaryDirectory(dir=temp_dir_path) as restore_folder:
recovery_folder = extract_archive(recovery_file, restore_folder)
self.restore_geoserver_backup(config, settings, recovery_folder,
skip_geoserver_info, skip_geoserver_security, ignore_errors)
skip_geoserver_info, skip_geoserver_security, ignore_errors, soft_reset)
self.restore_geoserver_raster_data(config, settings, recovery_folder)
self.restore_geoserver_vector_data(config, settings, recovery_folder)
self.restore_geoserver_vector_data(config, settings, recovery_folder, soft_reset)
self.restore_geoserver_externals(config, settings, recovery_folder)
if notify:
restore_notification.apply_async(
Expand Down Expand Up @@ -349,11 +362,11 @@ def execute_restore(self, **options):
call_command('loaddata', fixture_file, app_label=app_name)
except IntegrityError as e:
traceback.print_exc()
print("WARNING: The fixture '"+dump_name+"' fails on integrity check and import is aborted after all fixtures have been checked.")
logger.warning("WARNING: The fixture '"+dump_name+"' fails on integrity check and import is aborted after all fixtures have been checked.")
abortlater = True
except Exception:
traceback.print_exc()
print("WARNING: No valid fixture data found for '"+dump_name+"'.")
logger.warning("WARNING: No valid fixture data found for '"+dump_name+"'.")
# helpers.load_fixture(app_name, fixture_file)
raise
try:
Expand Down Expand Up @@ -609,7 +622,7 @@ def check_backup_ini_settings(self, backup_file: str) -> str:

return None

def restore_geoserver_backup(self, config, settings, target_folder, skip_geoserver_info, skip_geoserver_security, ignore_errors):
def restore_geoserver_backup(self, config, settings, target_folder, skip_geoserver_info, skip_geoserver_security, ignore_errors, soft_reset):
"""Restore GeoServer Catalog"""
url = settings.OGC_SERVER['default']['LOCATION']
user = settings.OGC_SERVER['default']['USER']
Expand All @@ -624,7 +637,7 @@ def restore_geoserver_backup(self, config, settings, target_folder, skip_geoserv

# Best Effort Restore: 'options': {'option': ['BK_BEST_EFFORT=true']}
_options = [
'BK_PURGE_RESOURCES=true',
'BK_PURGE_RESOURCES={}'.format('true' if not soft_reset else 'false'),
'BK_CLEANUP_TEMP=true',
'BK_SKIP_SETTINGS={}'.format('true' if skip_geoserver_info else 'false'),
'BK_SKIP_SECURITY={}'.format('true' if skip_geoserver_security else 'false'),
Expand Down Expand Up @@ -756,7 +769,7 @@ def restore_geoserver_raster_data(self, config, settings, target_folder):
print(('Skipping geoserver raster data restore: ' +
'directory "{}" not found.'.format(gs_data_folder)))

def restore_geoserver_vector_data(self, config, settings, target_folder):
def restore_geoserver_vector_data(self, config, settings, target_folder, soft_reset):
"""Restore Vectorial Data from DB"""
if (config.gs_dump_vector_data):

Expand All @@ -774,8 +787,11 @@ def restore_geoserver_vector_data(self, config, settings, target_folder):
ogc_db_host = settings.DATABASES[datastore]['HOST']
ogc_db_port = settings.DATABASES[datastore]['PORT']

if not soft_reset:
utils.remove_existing_tables(ogc_db_name, ogc_db_user, ogc_db_port, ogc_db_host, ogc_db_passwd)

utils.restore_db(config, ogc_db_name, ogc_db_user, ogc_db_port,
ogc_db_host, ogc_db_passwd, gs_data_folder)
ogc_db_host, ogc_db_passwd, gs_data_folder, soft_reset)

def restore_geoserver_externals(self, config, settings, target_folder):
"""Restore external references from XML files"""
Expand Down
42 changes: 34 additions & 8 deletions geonode/br/management/commands/utils/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,12 @@

import os
import re
import six
import sys
import hashlib
import psycopg2
import traceback
import dateutil.parser
import logging

from configparser import ConfigParser

Expand All @@ -38,6 +38,7 @@
TEMPLATE_DIRS = 'template_dirs'
LOCALE_PATHS = 'locale_dirs'
EXTERNAL_ROOT = 'external'
logger = logging.getLogger(__name__)


def option(parser):
Expand Down Expand Up @@ -232,7 +233,7 @@ def flush_db(db_name, db_user, db_port, db_host, db_passwd):
for table in pg_tables:
if table[0] == 'br_restoredbackup':
continue
print("Flushing Data : " + table[0])
logger.info("Flushing Data : " + table[0])
curs.execute("TRUNCATE " + table[0] + " CASCADE;")

except Exception:
Expand Down Expand Up @@ -271,7 +272,7 @@ def dump_db(config, db_name, db_user, db_port, db_host, db_passwd, target_folder
pg_tables = pg_all_tables

for table in pg_tables:
print("Dumping GeoServer Vectorial Data : {}:{}".format(db_name, table))
logger.info("Dumping GeoServer Vectorial Data : {}:{}".format(db_name, table))
os.system('PGPASSWORD="' + db_passwd + '" ' + config.pg_dump_cmd + ' -h ' + db_host +
' -p ' + str(db_port) + ' -U ' + db_user + ' -F c -b' +
' -t \'"' + str(table) + '"\' -f ' +
Expand All @@ -288,7 +289,7 @@ def dump_db(config, db_name, db_user, db_port, db_host, db_passwd, target_folder
conn.commit()


def restore_db(config, db_name, db_user, db_port, db_host, db_passwd, source_folder):
def restore_db(config, db_name, db_user, db_port, db_host, db_passwd, source_folder, preserve_tables):
"""Restore Full DB into target folder"""
db_host = db_host if db_host is not None else 'localhost'
db_port = db_port if db_port is not None else 5432
Expand All @@ -300,11 +301,12 @@ def restore_db(config, db_name, db_user, db_port, db_host, db_passwd, source_fol
file_names = [fn for fn in os.listdir(source_folder)
if any(fn.endswith(ext) for ext in included_extenstions)]
for table in file_names:
print("Restoring GeoServer Vectorial Data : {}:{} ".format(db_name, os.path.splitext(table)[0]))
pg_rstcmd = 'PGPASSWORD="' + db_passwd + '" ' + config.pg_restore_cmd + ' -c -h ' + db_host + \
logger.info("Restoring GeoServer Vectorial Data : {}:{} ".format(db_name, os.path.splitext(table)[0]))
pg_rstcmd = 'PGPASSWORD="' + db_passwd + '" ' + config.pg_restore_cmd + ' -h ' + db_host + \
' -p ' + str(db_port) + ' -U ' + db_user + ' --role=' + db_user + \
' -F c -t "' + os.path.splitext(table)[0] + '" ' +\
os.path.join(source_folder, table) + ' -d ' + db_name
pg_rstcmd += " -c" if preserve_tables else ""
os.system(pg_rstcmd)

except Exception:
Expand All @@ -318,6 +320,30 @@ def restore_db(config, db_name, db_user, db_port, db_host, db_passwd, source_fol
conn.commit()


def remove_existing_tables(db_name, db_user, db_port, db_host, db_passwd):
conn = get_db_conn(db_name, db_user, db_port, db_host, db_passwd)
curs = conn.cursor()
table_list = """SELECT tablename from pg_tables where tableowner = '%s'""" % (db_user)

try:
curs.execute(table_list)
pg_all_tables = [table[0] for table in curs.fetchall()]
for pg_table in pg_all_tables:
logger.info("Dropping existing GeoServer Vectorial Data : {}:{} ".format(db_name, pg_table))
curs.execute(f"DROP TABLE {pg_table} CASCADE")

conn.commit()
except Exception:
try:
conn.rollback()
except Exception:
pass

traceback.print_exc()
curs.close()
conn.close()


def confirm(prompt=None, resp=False):
"""prompts for yes or no response from the user. Returns True for yes and
False for no.
Expand Down Expand Up @@ -345,10 +371,10 @@ def confirm(prompt=None, resp=False):
prompt = '%s [%s]|%s: ' % (prompt, 'n', 'y')

while True:
ans = six.moves.input(prompt)
ans = input(prompt)
if not ans:
return resp
if ans not in ['y', 'Y', 'n', 'N']:
if ans not in {'y', 'Y', 'n', 'N'}:
print('please enter y or n.')
continue
if ans == 'y' or ans == 'Y':
Expand Down

0 comments on commit c6bfd80

Please sign in to comment.