diff --git a/.env b/.env index 9cba58ed754..4a1224902b4 100644 --- a/.env +++ b/.env @@ -25,6 +25,8 @@ GEONODE_GEODATABASE=geonode_data GEONODE_GEODATABASE_PASSWORD=geonode_data GEONODE_DATABASE_SCHEMA=public GEONODE_GEODATABASE_SCHEMA=public +DATABASE_HOST=db +DATABASE_PORT=5432 DATABASE_URL=postgis://geonode:geonode@db:5432/geonode GEODATABASE_URL=postgis://geonode_data:geonode_data@db:5432/geonode_data GEONODE_DB_CONN_MAX_AGE=0 diff --git a/geonode/api/resourcebase_api.py b/geonode/api/resourcebase_api.py index e5a94d98ede..09d47d4ab7c 100644 --- a/geonode/api/resourcebase_api.py +++ b/geonode/api/resourcebase_api.py @@ -340,12 +340,11 @@ def build_haystack_filters(self, parameters): if len(subtypes) > 0: types.append("layer") - sqs = SearchQuerySet().narrow("subtype:%s" % - ','.join(map(str, subtypes))) + sqs = SearchQuerySet().narrow(f"subtype:{','.join(map(str, subtypes))}") if len(types) > 0: sqs = (SearchQuerySet() if sqs is None else sqs).narrow( - "type:%s" % ','.join(map(str, types))) + f"type:{','.join(map(str, types))}") # Filter by Query Params # haystack bug? if boosted fields aren't included in the @@ -417,7 +416,7 @@ def build_haystack_filters(self, parameters): if owner: sqs = ( SearchQuerySet() if sqs is None else sqs).narrow( - "owner__username:%s" % ','.join(map(str, owner))) + f"owner__username:{','.join(map(str, owner))}") # filter by date if date_start: diff --git a/geonode/api/tests.py b/geonode/api/tests.py index c8be98e5070..6d5c63217c5 100644 --- a/geonode/api/tests.py +++ b/geonode/api/tests.py @@ -209,7 +209,7 @@ def setUp(self): self.passwd = 'admin' self._user = get_user_model().objects.get(username=self.user) self.token = get_or_create_token(self._user) - self.auth_header = 'Bearer {}'.format(self.token) + self.auth_header = f'Bearer {self.token}' self.list_url = reverse( 'api_dispatch_list', kwargs={ @@ -402,21 +402,21 @@ def to_date(val): return val.date().strftime(fstring) d1 = to_date(now - step) - filter_url = self.list_url + '?date__exact={}'.format(d1) + filter_url = self.list_url + f'?date__exact={d1}' resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) self.assertEqual(len(self.deserialize(resp)['objects']), 0) d3 = to_date(now - (3 * step)) - filter_url = self.list_url + '?date__gte={}'.format(d3) + filter_url = self.list_url + f'?date__gte={d3}' resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) self.assertEqual(len(self.deserialize(resp)['objects']), 0) d4 = to_date(now - (4 * step)) - filter_url = self.list_url + '?date__range={},{}'.format(d4, to_date(now)) + filter_url = self.list_url + f'?date__range={d4},{to_date(now)}' resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) diff --git a/geonode/api/views.py b/geonode/api/views.py index 7698e1c27b4..824c5f49d21 100644 --- a/geonode/api/views.py +++ b/geonode/api/views.py @@ -141,7 +141,7 @@ def verify_token(request): token_info, content_type="application/json" ) - response["Authorization"] = ("Bearer %s" % access_token) + response["Authorization"] = f"Bearer {access_token}" return response else: return HttpResponse( diff --git a/geonode/base/forms.py b/geonode/base/forms.py index 586dc1b47ca..9e81add3eda 100644 --- a/geonode/base/forms.py +++ b/geonode/base/forms.py @@ -503,7 +503,7 @@ def clean_keywords(self): _kk = _kk.replace('%u', r'\u').encode('unicode-escape').replace( b'\\\\u', b'\\u').decode('unicode-escape') if '%u' in _kk else _kk - _hk = HierarchicalKeyword.objects.filter(name__iexact='%s' % _kk.strip()) + _hk = HierarchicalKeyword.objects.filter(name__iexact=f'{_kk.strip()}') if _hk and len(_hk) > 0: _unsescaped_kwds.append(str(_hk[0])) else: diff --git a/geonode/base/management/commands/delete_resources.py b/geonode/base/management/commands/delete_resources.py index 538b4382093..49d29575f86 100644 --- a/geonode/base/management/commands/delete_resources.py +++ b/geonode/base/management/commands/delete_resources.py @@ -22,7 +22,6 @@ import json # import needed to resolve model filters: -from django.db.models import Q from django.db import transaction, IntegrityError from django.core.management.base import BaseCommand, CommandError @@ -76,10 +75,12 @@ def handle(self, **options): # check argument set if all(config is None for config in {layer_filters, map_filters, document_filters, config_path}): - raise CommandError("No configuration provided. Please specify any of the following arguments: '-l', '-m', '-d', '-c'.") + raise CommandError( + "No configuration provided. Please specify any of the following arguments: '-l', '-m', '-d', '-c'.") if any([layer_filters, map_filters, document_filters]) and config_path: - raise CommandError("Too many configuration options provided. Please use either '-c' or '-l', '-m', '-d' arguments.") + raise CommandError( + "Too many configuration options provided. Please use either '-c' or '-l', '-m', '-d' arguments.") # check config_file, if it exists if config_path: @@ -102,7 +103,12 @@ def handle(self, **options): # or 'filters' is not set in config OR it is not an JSON object or not isinstance(config.get('filters'), dict) # or all filters are empty - or not any([config.get('filters').get('layer'), config.get('filters').get('map'), config.get('filters').get('document')]) + or not any( + [ + config.get('filters').get('layer'), + config.get('filters').get('map'), + config.get('filters').get('document') + ]) ): print('Nothing to be done... exiting delete_resources command.') return diff --git a/geonode/base/management/commands/fixoauthuri.py b/geonode/base/management/commands/fixoauthuri.py index ca3c16f4afd..81a6e57f626 100644 --- a/geonode/base/management/commands/fixoauthuri.py +++ b/geonode/base/management/commands/fixoauthuri.py @@ -53,16 +53,12 @@ def handle(self, *args, **options): force_exec = options.get('force_exec') target_address = options.get('target_address') - from django.conf import settings client_id = None client_secret = None if check_ogc_backend(geoserver.BACKEND_PACKAGE): from geonode.geoserver.helpers import ogc_server_settings - redirect_uris = '%s\n%s\n%s' % ( - ogc_server_settings.LOCATION, - ogc_server_settings.public_url, - "{0}/geoserver/".format(target_address)) + redirect_uris = f'{ogc_server_settings.LOCATION}\n{ogc_server_settings.public_url}\n{target_address}/geoserver/' # noqa if Application.objects.filter(name='GeoServer').exists(): Application.objects.filter(name='GeoServer').update(redirect_uris=redirect_uris) if force_exec: @@ -86,4 +82,4 @@ def handle(self, *args, **options): client_secret=client_secret, user=get_user_model().objects.filter(is_superuser=True)[0] ) - return '%s,%s' % (client_id, client_secret) + return f'{client_id},{client_secret}' diff --git a/geonode/base/management/commands/helpers.py b/geonode/base/management/commands/helpers.py index c0fb331867b..fe049229726 100644 --- a/geonode/base/management/commands/helpers.py +++ b/geonode/base/management/commands/helpers.py @@ -46,9 +46,9 @@ def confirm(prompt=None, resp=False): prompt = 'Confirm' if resp: - prompt = '%s [%s]|%s: ' % (prompt, 'y', 'n') + prompt = f'{prompt} [y]|n: ' else: - prompt = '%s [%s]|%s: ' % (prompt, 'n', 'y') + prompt = f'{prompt} [n]|y: ' while True: ans = input(prompt) diff --git a/geonode/base/management/commands/load_thesaurus.py b/geonode/base/management/commands/load_thesaurus.py index 01af25d3f5c..80cb00377c8 100644 --- a/geonode/base/management/commands/load_thesaurus.py +++ b/geonode/base/management/commands/load_thesaurus.py @@ -93,7 +93,7 @@ def load_thesaurus(self, input_file, name, store): raise CommandError("ConceptScheme not found in file") titles = scheme.findall('dc:title', ns) - + default_lang = getattr(settings, 'THESAURUS_DEFAULT_LANG', None) available_lang = get_all_lang_available_with_title(titles, LANG_ATTRIB) thesaurus_title = determinate_value(available_lang, default_lang) @@ -102,7 +102,7 @@ def load_thesaurus(self, input_file, name, store): date_issued = scheme.find('dcterms:issued', ns).text about = scheme.attrib.get(ABOUT_ATTRIB) - print('Thesaurus "{}" issued on {}'.format(thesaurus_title, date_issued)) + print(f'Thesaurus "{thesaurus_title}" issued at {date_issued}') thesaurus = Thesaurus() thesaurus.identifier = name @@ -133,7 +133,7 @@ def load_thesaurus(self, input_file, name, store): available_lang = get_all_lang_available_with_title(concepts, LANG_ATTRIB) alt_label = determinate_value(available_lang, default_lang) - print('Concept {} ({})'.format(alt_label, about)) + print(f'Concept {alt_label} ({about})') tk = ThesaurusKeyword() tk.thesaurus = thesaurus @@ -147,7 +147,7 @@ def load_thesaurus(self, input_file, name, store): lang = pref_label.attrib.get(LANG_ATTRIB) label = pref_label.text - print(' Label {}: {}'.format(lang, label)) + print(f' Label {lang}: {label}') tkl = ThesaurusKeywordLabel() tkl.keyword = tk @@ -174,18 +174,20 @@ def create_fake_thesaurus(self, name): tk.alt_label = keyword + '_alt' tk.save() - for l in ['it', 'en', 'es']: + for _l in ['it', 'en', 'es']: tkl = ThesaurusKeywordLabel() tkl.keyword = tk - tkl.lang = l - tkl.label = keyword + "_l_" + l + "_t_" + name + tkl.lang = _l + tkl.label = keyword + "_l_" + _l + "_t_" + name tkl.save() + def get_all_lang_available_with_title(items: List, LANG_ATTRIB: str): return [(item.attrib.get(LANG_ATTRIB), item.text) for item in items] + def determinate_value(available_lang: List, default_lang: str): - sorted_lang = sorted(available_lang, key= lambda lang: '' if lang[0] is None else lang[0]) + sorted_lang = sorted(available_lang, key=lambda lang: '' if lang[0] is None else lang[0]) for item in sorted_lang: if item[0] is None: return item[1] diff --git a/geonode/base/management/commands/migrate_baseurl.py b/geonode/base/management/commands/migrate_baseurl.py index 2ef3d96e117..c6befd74f56 100644 --- a/geonode/base/management/commands/migrate_baseurl.py +++ b/geonode/base/management/commands/migrate_baseurl.py @@ -74,8 +74,8 @@ def handle(self, **options): if not target_address or len(target_address) == 0: raise CommandError("Target Address '--target-address' is mandatory") - print("This will change all Maps, Layers, \ -Styles and Links Base URLs from [%s] to [%s]." % (source_address, target_address)) + print(f"This will change all Maps, Layers, \ +Styles and Links Base URLs from [{source_address}] to [{target_address}].") print("The operation may take some time, depending on the amount of Layer on GeoNode.") message = 'You want to proceed?' @@ -83,42 +83,42 @@ def handle(self, **options): try: _cnt = Map.objects.filter(thumbnail_url__icontains=source_address).update( thumbnail_url=Func( - F('thumbnail_url'),Value(source_address),Value(target_address),function='replace')) - logger.info("Updated %s Maps" % _cnt) + F('thumbnail_url'), Value(source_address), Value(target_address), function='replace')) + logger.info(f"Updated {_cnt} Maps") _cnt = MapLayer.objects.filter(ows_url__icontains=source_address).update( ows_url=Func( - F('ows_url'),Value(source_address),Value(target_address),function='replace')) + F('ows_url'), Value(source_address), Value(target_address), function='replace')) MapLayer.objects.filter(layer_params__icontains=source_address).update( layer_params=Func( - F('layer_params'),Value(source_address),Value(target_address),function='replace')) - logger.info("Updated %s MapLayers" % _cnt) + F('layer_params'), Value(source_address), Value(target_address), function='replace')) + logger.info(f"Updated {_cnt} MapLayers") _cnt = Layer.objects.filter(thumbnail_url__icontains=source_address).update( thumbnail_url=Func( - F('thumbnail_url'),Value(source_address),Value(target_address),function='replace')) - logger.info("Updated %s Layers" % _cnt) + F('thumbnail_url'), Value(source_address), Value(target_address), function='replace')) + logger.info(f"Updated {_cnt} Layers") _cnt = Style.objects.filter(sld_url__icontains=source_address).update( sld_url=Func( - F('sld_url'),Value(source_address),Value(target_address),function='replace')) - logger.info("Updated %s Styles" % _cnt) + F('sld_url'), Value(source_address), Value(target_address), function='replace')) + logger.info(f"Updated {_cnt} Styles") _cnt = Link.objects.filter(url__icontains=source_address).update( url=Func( - F('url'),Value(source_address),Value(target_address),function='replace')) - logger.info("Updated %s Links" % _cnt) + F('url'), Value(source_address), Value(target_address), function='replace')) + logger.info(f"Updated {_cnt} Links") _cnt = ResourceBase.objects.filter(thumbnail_url__icontains=source_address).update( thumbnail_url=Func( - F('thumbnail_url'),Value(source_address),Value(target_address),function='replace')) + F('thumbnail_url'), Value(source_address), Value(target_address), function='replace')) _cnt += ResourceBase.objects.filter(csw_anytext__icontains=source_address).update( csw_anytext=Func( F('csw_anytext'), Value(source_address), Value(target_address), function='replace')) _cnt += ResourceBase.objects.filter(metadata_xml__icontains=source_address).update( metadata_xml=Func( F('metadata_xml'), Value(source_address), Value(target_address), function='replace')) - logger.info("Updated %s ResourceBases" % _cnt) + logger.info(f"Updated {_cnt} ResourceBases") site = Site.objects.get_current() if site: @@ -132,7 +132,7 @@ def handle(self, **options): _cnt = Application.objects.filter(name='GeoServer').update( redirect_uris=Func( F('redirect_uris'), Value(source_address), Value(target_address), function='replace')) - logger.info("Updated %s OAUth2 Redirect URIs" % _cnt) + logger.info(f"Updated {_cnt} OAUth2 Redirect URIs") finally: print("...done!") diff --git a/geonode/base/management/commands/migrate_notifications.py b/geonode/base/management/commands/migrate_notifications.py index 1cb8a96f3d7..19a10a7a861 100644 --- a/geonode/base/management/commands/migrate_notifications.py +++ b/geonode/base/management/commands/migrate_notifications.py @@ -43,7 +43,7 @@ def handle(self, **options): try: c.execute('select medium, send, notice_type_id, user_id from notification_noticesetting;') except ProgrammingError as err: - log.error("No table for notification app, exiting") + log.error(f"No table for notification app, exiting: {err}") # no source of data, bye! return diff --git a/geonode/base/management/commands/set_all_layers_alternate.py b/geonode/base/management/commands/set_all_layers_alternate.py index 666b93c44de..5d2ea1c72a4 100644 --- a/geonode/base/management/commands/set_all_layers_alternate.py +++ b/geonode/base/management/commands/set_all_layers_alternate.py @@ -66,15 +66,13 @@ def handle(self, *args, **options): all_layers = all_layers.filter(owner__username=username) for index, layer in enumerate(all_layers): - logger.info("[%s / %s] Checking 'alternate' of Layer [%s] ..." % ((index + 1), len(all_layers), layer.name)) + logger.info(f"[{(index + 1)} / {len(all_layers)}] Checking 'alternate' of Layer [{layer.name}] ...") try: if not layer.alternate: layer.alternate = layer.typename layer.save() except Exception as e: - # import traceback - # traceback.print_exc() if ignore_errors: - logger.error("[ERROR] Layer [%s] couldn't be updated" % (layer.name)) + logger.error(f"[ERROR] Layer [{layer.name}] couldn't be updated") else: raise e diff --git a/geonode/base/management/commands/set_all_layers_metadata.py b/geonode/base/management/commands/set_all_layers_metadata.py index a2247aca807..c8bd7365ccd 100644 --- a/geonode/base/management/commands/set_all_layers_metadata.py +++ b/geonode/base/management/commands/set_all_layers_metadata.py @@ -23,13 +23,8 @@ from geonode.layers.models import Layer from geonode import geoserver # noqa from geonode.catalogue.models import catalogue_post_save -from geonode.geoserver.helpers import ogc_server_settings import logging - -logger = logging.getLogger(__name__) - - from geonode.utils import ( check_ogc_backend, set_resource_default_links @@ -42,6 +37,8 @@ if check_ogc_backend(geoserver.BACKEND_PACKAGE): from geonode.geoserver.helpers import set_attributes_from_geoserver as set_attributes +logger = logging.getLogger(__name__) + class Command(BaseCommand): help = 'Resets Metadata Attributes and Schema to All Layers' @@ -110,7 +107,7 @@ def handle(self, *args, **options): all_layers = all_layers.filter(owner__username=username) for index, layer in enumerate(all_layers): - print("[%s / %s] Updating Layer [%s] ..." % ((index + 1), len(all_layers), layer.name)) + print(f"[{(index + 1)} / {len(all_layers)}] Updating Layer [{layer.name}] ...") try: # recalculate the layer statistics set_attributes(layer, overwrite=True) @@ -128,7 +125,7 @@ def handle(self, *args, **options): import traceback traceback.print_exc() if ignore_errors: - logger.error("[ERROR] Layer [%s] couldn't be updated" % (layer.name)) + logger.error(f"[ERROR] Layer [{layer.name}] couldn't be updated") else: raise e diff --git a/geonode/base/management/commands/set_all_layers_public.py b/geonode/base/management/commands/set_all_layers_public.py index 3c8b432d086..199acea9470 100644 --- a/geonode/base/management/commands/set_all_layers_public.py +++ b/geonode/base/management/commands/set_all_layers_public.py @@ -36,7 +36,7 @@ def handle(self, *args, **options): all_layers = Layer.objects.all() for index, layer in enumerate(all_layers): - print("[%s / %s] Setting public permissions to Layer [%s] ..." % ((index + 1), len(all_layers), layer.name)) + print(f"[{(index + 1)} / {len(all_layers)}] Setting public permissions to Layer [{layer.name}] ...") try: use_geofence = settings.OGC_SERVER['default'].get( "GEOFENCE_SECURITY_ENABLED", False) @@ -53,4 +53,4 @@ def handle(self, *args, **options): perm_spec["users"]["AnonymousUser"] = ['view_resourcebase', 'download_resourcebase'] layer.set_permissions(perm_spec) except Exception: - logger.error("[ERROR] Layer [%s] couldn't be updated" % (layer.name)) + logger.error(f"[ERROR] Layer [{layer.name}] couldn't be updated") diff --git a/geonode/base/models.py b/geonode/base/models.py index fde5de26d45..4cbcf8413ea 100644 --- a/geonode/base/models.py +++ b/geonode/base/models.py @@ -126,8 +126,7 @@ def clean(self): # only allow this if we are updating the same contact if self.contact != contacts.get(): raise ValidationError( - 'There can be only one %s for a given resource' % - self.role) + f'There can be only one {self.role} for a given resource') if self.contact is None: # verify that any unbound contact is only associated to one # resource @@ -184,7 +183,7 @@ class SpatialRepresentationType(models.Model): is_choice = models.BooleanField(default=True) def __str__(self): - return "{0}".format(self.gn_description) + return str(self.gn_description) class Meta: ordering = ("identifier",) @@ -231,7 +230,7 @@ class Region(MPTTModel): default='EPSG:4326') def __str__(self): - return "{0}".format(self.name) + return str(self.name) @property def bbox(self): @@ -280,7 +279,7 @@ class RestrictionCodeType(models.Model): is_choice = models.BooleanField(default=True) def __str__(self): - return "{0}".format(self.gn_description) + return str(self.gn_description) class Meta: ordering = ("identifier",) @@ -296,7 +295,7 @@ class License(models.Model): license_text = models.TextField(null=True, blank=True) def __str__(self): - return "{0}".format(self.name) + return str(self.name) @property def name_long(self): @@ -419,10 +418,10 @@ class TaggedContentItem(ItemBase): def tags_for(cls, model, instance=None): if instance is not None: return cls.tag_model().objects.filter(**{ - '%s__content_object' % cls.tag_relname(): instance + f'{cls.tag_relname()}__content_object': instance }) return cls.tag_model().objects.filter(**{ - '%s__content_object__isnull' % cls.tag_relname(): False + f'{cls.tag_relname()}__content_object__isnull': False }).distinct() @@ -485,7 +484,7 @@ class Thesaurus(models.Model): facet = models.BooleanField(default=True) def __str__(self): - return "{0}".format(self.identifier) + return str(self.identifier) class Meta: ordering = ("identifier",) @@ -506,7 +505,7 @@ class ThesaurusKeywordLabel(models.Model): keyword = models.ForeignKey('ThesaurusKeyword', related_name='keyword', on_delete=models.CASCADE) def __str__(self): - return "{0}".format(self.label) + return str(self.label) class Meta: ordering = ("keyword", "lang") @@ -530,7 +529,7 @@ class ThesaurusKeyword(models.Model): thesaurus = models.ForeignKey('Thesaurus', related_name='thesaurus', on_delete=models.CASCADE) def __str__(self): - return "{0}".format(self.alt_label) + return str(self.alt_label) @property def labels(self): @@ -554,7 +553,7 @@ class ThesaurusLabel(models.Model): thesaurus = models.ForeignKey('Thesaurus', related_name='rel_thesaurus', on_delete=models.CASCADE) def __str__(self): - return "{0}".format(self.label) + return str(self.label) class Meta: ordering = ("lang",) @@ -916,7 +915,7 @@ def __init__(self, *args, **kwargs): super(ResourceBase, self).__init__(*args, **kwargs) def __str__(self): - return "{0}".format(self.title) + return str(self.title) def _remove_html_tags(self, attribute_str): _attribute_str = attribute_str @@ -962,7 +961,7 @@ def save(self, notify=False, *args, **kwargs): if self.pk is None and self.title: # Resource Created - notice_type_label = '%s_created' % self.class_name.lower() + notice_type_label = f'{self.class_name.lower()}_created' recipients = get_notification_recipients(notice_type_label, resource=self) send_notification(recipients, notice_type_label, {'resource': self}) elif self.pk: @@ -976,7 +975,7 @@ def save(self, notify=False, *args, **kwargs): self.set_workflow_perms(approved=True) # Send "approved" notification - notice_type_label = '%s_approved' % self.class_name.lower() + notice_type_label = f'{self.class_name.lower()}_approved' recipients = get_notification_recipients(notice_type_label, resource=self) send_notification(recipients, notice_type_label, {'resource': self}) _notification_sent = True @@ -988,14 +987,14 @@ def save(self, notify=False, *args, **kwargs): self.set_workflow_perms(published=True) # Send "published" notification - notice_type_label = '%s_published' % self.class_name.lower() + notice_type_label = f'{self.class_name.lower()}_published' recipients = get_notification_recipients(notice_type_label, resource=self) send_notification(recipients, notice_type_label, {'resource': self}) _notification_sent = True # Updated Notifications Here if not _notification_sent: - notice_type_label = '%s_updated' % self.class_name.lower() + notice_type_label = f'{self.class_name.lower()}_updated' recipients = get_notification_recipients(notice_type_label, resource=self) send_notification(recipients, notice_type_label, {'resource': self}) @@ -1008,7 +1007,7 @@ def delete(self, notify=True, *args, **kwargs): Send a notification when a layer, map or document is deleted """ if hasattr(self, 'class_name') and notify: - notice_type_label = '%s_deleted' % self.class_name.lower() + notice_type_label = f'{self.class_name.lower()}_deleted' recipients = get_notification_recipients(notice_type_label, resource=self) send_notification(recipients, notice_type_label, {'resource': self}) @@ -1062,7 +1061,7 @@ def bbox(self): match = re.match(r'^(EPSG:)?(?P\d{4,6})$', self.srid) srid = int(match.group('srid')) if match else 4326 bbox = BBOXHelper(self.bbox_polygon.extent) - return [bbox.xmin, bbox.xmax, bbox.ymin, bbox.ymax, "EPSG:{}".format(srid)] + return [bbox.xmin, bbox.xmax, bbox.ymin, bbox.ymax, f"EPSG:{srid}"] bbox = BBOXHelper.from_xy([-180, 180, -90, 90]) return [bbox.xmin, bbox.xmax, bbox.ymin, bbox.ymax, "EPSG:4326"] @@ -1082,11 +1081,7 @@ def ll_bbox_string(self): if self.bbox_polygon: bbox = BBOXHelper.from_xy(self.ll_bbox[:4]) - return "{x0:.7f},{y0:.7f},{x1:.7f},{y1:.7f}".format( - x0=bbox.xmin, - y0=bbox.ymin, - x1=bbox.xmax, - y1=bbox.ymax) + return f"{bbox.xmin:.7f},{bbox.ymin:.7f},{bbox.xmax:.7f},{bbox.ymax:.7f}" bbox = BBOXHelper.from_xy([-180, 180, -90, 90]) return [bbox.xmin, bbox.xmax, bbox.ymin, bbox.ymax, "EPSG:4326"] @@ -1097,11 +1092,7 @@ def bbox_string(self): if self.bbox_polygon: bbox = BBOXHelper.from_xy(self.bbox[:4]) - return "{x0:.7f},{y0:.7f},{x1:.7f},{y1:.7f}".format( - x0=bbox.xmin, - y0=bbox.ymin, - x1=bbox.xmax, - y1=bbox.ymax) + return f"{bbox.xmin:.7f},{bbox.ymin:.7f},{bbox.xmax:.7f},{bbox.ymax:.7f}" bbox = BBOXHelper.from_xy([-180, 180, -90, 90]) return [bbox.xmin, bbox.xmax, bbox.ymin, bbox.ymax, "EPSG:4326"] @@ -1205,7 +1196,7 @@ def metadata_completeness(self): if not field.identifier: continue filled_fields.append(field) - return '{}%'.format(len(filled_fields) * 100 / len(required_fields)) + return f'{len(filled_fields) * 100 / len(required_fields)}%' @property def instance_is_processed(self): @@ -1342,12 +1333,10 @@ def set_bounds_from_bbox(self, bbox, srid): if not bbox or len(bbox) < 4: raise ValidationError( - 'Bounding Box cannot be empty %s for a given resource' % - self.name) + f'Bounding Box cannot be empty {self.name} for a given resource') if not srid: raise ValidationError( - 'Projection cannot be empty %s for a given resource' % - self.name) + f'Projection cannot be empty {self.name} for a given resource') self.srid = srid self.set_bbox_polygon( @@ -1387,8 +1376,8 @@ def download_links(self): else: _link_type = 'WWW:DOWNLOAD-1.0-http--download' if self.storeType == 'remoteStore' and link.extension in ('html'): - _link_type = 'WWW:DOWNLOAD-%s' % self.remote_service.type - description = '%s (%s Format)' % (self.title, link.name) + _link_type = f'WWW:DOWNLOAD-{self.remote_service.type}' + description = f'{self.title} ({link.name} Format)' links.append( (self.title, description, @@ -1734,7 +1723,7 @@ class Link(models.Model): objects = LinkManager() def __str__(self): - return "{0} link".format(self.link_type) + return f"{self.link_type} link" class MenuPlaceholder(models.Model): @@ -1746,7 +1735,7 @@ class MenuPlaceholder(models.Model): ) def __str__(self): - return "{0}".format(self.name) + return str(self.name) class Menu(models.Model): @@ -1765,7 +1754,7 @@ class Menu(models.Model): ) def __str__(self): - return "{0}".format(self.title) + return str(self.title) class Meta: unique_together = ( @@ -1818,7 +1807,7 @@ def __hash__(self): return hash(self.url) def __str__(self): - return "{0}".format(self.title) + return str(self.title) class Meta: unique_together = ( diff --git a/geonode/base/templatetags/base_tags.py b/geonode/base/templatetags/base_tags.py index 4430f8f0a20..614f1550c72 100644 --- a/geonode/base/templatetags/base_tags.py +++ b/geonode/base/templatetags/base_tags.py @@ -347,7 +347,7 @@ def get_context_resourcetype(context): resource_types = ['layers', 'maps', 'geoapps', 'documents', 'search', 'people', 'groups/categories', 'groups'] for resource_type in resource_types: - if "/{0}/".format(resource_type) in c_path: + if f"/{resource_type}/" in c_path: return resource_type return 'error' diff --git a/geonode/base/tests.py b/geonode/base/tests.py index d4efa13a987..651bca9c951 100644 --- a/geonode/base/tests.py +++ b/geonode/base/tests.py @@ -266,52 +266,38 @@ def test_get_menu_placeholder_0(self): self.assertIn( self.menu_0_0.title, rendered, - 'Expected "{}" string in the rendered template'.format( - self.menu_0_0.title - ) + f'Expected "{self.menu_0_0.title}" string in the rendered template' ) self.assertIn( self.menu_item_0_0_0.title, rendered, - 'Expected "{}" string in the rendered template'.format( - self.menu_item_0_0_0.title - ) + f'Expected "{self.menu_item_0_0_0.title}" string in the rendered template' ) self.assertIn( self.menu_item_0_0_1.title, rendered, - 'Expected "{}" string in the rendered template'.format( - self.menu_item_0_0_1.title - ) + f'Expected "{self.menu_item_0_0_1.title}" string in the rendered template' ) # second menu self.assertIn( self.menu_0_1.title, rendered, - 'Expected "{}" string in the rendered template'.format( - self.menu_0_1.title - ) + f'Expected "{self.menu_0_1.title}" string in the rendered template' ) self.assertIn( self.menu_item_0_1_0.title, rendered, - 'Expected "{}" string in the rendered template'.format( - self.menu_item_0_1_0.title - ) + f'Expected "{self.menu_item_0_1_0.title}" string in the rendered template' ) self.assertIn( self.menu_item_0_1_1.title, rendered, - 'Expected "{}" string in the rendered template'.format( - self.menu_item_0_1_1.title - ) + f'Expected "{self.menu_item_0_1_1.title}" string in the rendered template' ) self.assertIn( self.menu_item_0_1_2.title, rendered, - 'Expected "{}" string in the rendered template'.format( - self.menu_item_0_1_2.title - ) + f'Expected "{self.menu_item_0_1_2.title}" string in the rendered template' ) # menu_placeholder_1 # first menu @@ -319,23 +305,17 @@ def test_get_menu_placeholder_0(self): self.assertNotIn( self.menu_1_0.title, rendered, - 'No "{}" string expected in the rendered template'.format( - self.menu_1_0.title - ) + f'No "{self.menu_1_0.title}" string expected in the rendered template' ) self.assertNotIn( self.menu_item_1_0_0.title, rendered, - 'No "{}" string expected in the rendered template'.format( - self.menu_item_1_0_0.title - ) + f'No "{self.menu_item_1_0_0.title}" string expected in the rendered template' ) self.assertNotIn( self.menu_item_1_0_1.title, rendered, - 'No "{}" string expected in the rendered template'.format( - self.menu_item_1_0_1.title - ) + f'No "{self.menu_item_1_0_1.title}" string expected in the rendered template' ) def test_get_menu_placeholder_1(self): @@ -348,52 +328,38 @@ def test_get_menu_placeholder_1(self): self.assertNotIn( self.menu_0_0.title, rendered, - 'No "{}" string expected in the rendered template'.format( - self.menu_0_0.title - ) + f'No "{self.menu_0_0.title}" string expected in the rendered template' ) self.assertNotIn( self.menu_item_0_0_0.title, rendered, - 'No "{}" string expected in the rendered template'.format( - self.menu_item_0_0_0.title - ) + f'No "{self.menu_item_0_0_0.title}" string expected in the rendered template' ) self.assertNotIn( self.menu_item_0_0_1.title, rendered, - 'No "{}" string expected in the rendered template'.format( - self.menu_item_0_0_1.title - ) + f'No "{self.menu_item_0_0_1.title}" string expected in the rendered template' ) # second menu self.assertNotIn( self.menu_0_1.title, rendered, - 'No "{}" string expected in the rendered template'.format( - self.menu_0_1.title - ) + f'No "{self.menu_0_1.title}" string expected in the rendered template' ) self.assertNotIn( self.menu_item_0_1_0.title, rendered, - 'No "{}" string expected in the rendered template'.format( - self.menu_item_0_1_0.title - ) + f'No "{self.menu_item_0_1_0.title}" string expected in the rendered template' ) self.assertNotIn( self.menu_item_0_1_1.title, rendered, - 'No "{}" string expected in the rendered template'.format( - self.menu_item_0_1_1.title - ) + f'No "{self.menu_item_0_1_1.title}" string expected in the rendered template' ) self.assertNotIn( self.menu_item_0_1_2.title, rendered, - 'No "{}" string expected in the rendered template'.format( - self.menu_item_0_1_2.title - ) + f'No "{self.menu_item_0_1_2.title}" string expected in the rendered template' ) # menu_placeholder_1 # first menu @@ -401,23 +367,17 @@ def test_get_menu_placeholder_1(self): self.assertIn( self.menu_1_0.title, rendered, - 'Expected "{}" string in the rendered template'.format( - self.menu_1_0.title - ) + f'Expected "{self.menu_1_0.title}" string in the rendered template' ) self.assertIn( self.menu_item_1_0_0.title, rendered, - 'Expected "{}" string in the rendered template'.format( - self.menu_item_1_0_0.title - ) + f'Expected "{self.menu_item_1_0_0.title}" string in the rendered template' ) self.assertIn( self.menu_item_1_0_1.title, rendered, - 'Expected "{}" string in the rendered template'.format( - self.menu_item_1_0_1.title - ) + f'Expected "{self.menu_item_1_0_1.title}" string in the rendered template' ) def test_render_nav_menu_placeholder_0(self): @@ -430,52 +390,38 @@ def test_render_nav_menu_placeholder_0(self): self.assertIn( self.menu_0_0.title, rendered, - 'Expected "{}" string in the rendered template'.format( - self.menu_0_0.title - ) + f'Expected "{self.menu_0_0.title}" string in the rendered template' ) self.assertIn( self.menu_item_0_0_0.title, rendered, - 'Expected "{}" string in the rendered template'.format( - self.menu_item_0_0_0.title - ) + f'Expected "{self.menu_item_0_0_0.title}" string in the rendered template' ) self.assertIn( self.menu_item_0_0_1.title, rendered, - 'Expected "{}" string in the rendered template'.format( - self.menu_item_0_0_1.title - ) + f'Expected "{self.menu_item_0_0_1.title}" string in the rendered template' ) # second menu self.assertIn( self.menu_0_1.title, rendered, - 'Expected "{}" string in the rendered template'.format( - self.menu_0_1.title - ) + f'Expected "{self.menu_0_1.title}" string in the rendered template' ) self.assertIn( self.menu_item_0_1_0.title, rendered, - 'Expected "{}" string in the rendered template'.format( - self.menu_item_0_1_0.title - ) + f'Expected "{self.menu_item_0_1_0.title}" string in the rendered template' ) self.assertIn( self.menu_item_0_1_1.title, rendered, - 'Expected "{}" string in the rendered template'.format( - self.menu_item_0_1_1.title - ) + f'Expected "{self.menu_item_0_1_1.title}" string in the rendered template' ) self.assertIn( self.menu_item_0_1_2.title, rendered, - 'Expected "{}" string in the rendered template'.format( - self.menu_item_0_1_2.title - ) + f'Expected "{self.menu_item_0_1_2.title}" string in the rendered template' ) # menu_placeholder_1 # first menu @@ -483,23 +429,17 @@ def test_render_nav_menu_placeholder_0(self): self.assertNotIn( self.menu_1_0.title, rendered, - 'No "{}" string expected in the rendered template'.format( - self.menu_1_0.title - ) + f'No "{self.menu_1_0.title}" string expected in the rendered template' ) self.assertNotIn( self.menu_item_1_0_0.title, rendered, - 'No "{}" string expected in the rendered template'.format( - self.menu_item_1_0_0.title - ) + f'No "{self.menu_item_1_0_0.title}" string expected in the rendered template' ) self.assertNotIn( self.menu_item_1_0_1.title, rendered, - 'No "{}" string expected in the rendered template'.format( - self.menu_item_1_0_1.title - ) + f'No "{self.menu_item_1_0_1.title}" string expected in the rendered template' ) def test_render_nav_menu_placeholder_1(self): @@ -512,52 +452,38 @@ def test_render_nav_menu_placeholder_1(self): self.assertNotIn( self.menu_0_0.title, rendered, - 'No "{}" string expected in the rendered template'.format( - self.menu_0_0.title - ) + f'No "{self.menu_0_0.title}" string expected in the rendered template' ) self.assertNotIn( self.menu_item_0_0_0.title, rendered, - 'No "{}" string expected in the rendered template'.format( - self.menu_item_0_0_0.title - ) + f'No "{self.menu_item_0_0_0.title}" string expected in the rendered template' ) self.assertNotIn( self.menu_item_0_0_1.title, rendered, - 'No "{}" string expected in the rendered template'.format( - self.menu_item_0_0_1.title - ) + f'No "{self.menu_item_0_0_1.title}" string expected in the rendered template' ) # second menu self.assertNotIn( self.menu_0_1.title, rendered, - 'No "{}" string expected in the rendered template'.format( - self.menu_0_1.title - ) + f'No "{self.menu_0_1.title}" string expected in the rendered template' ) self.assertNotIn( self.menu_item_0_1_0.title, rendered, - 'No "{}" string expected in the rendered template'.format( - self.menu_item_0_1_0.title - ) + f'No "{self.menu_item_0_1_0.title}" string expected in the rendered template' ) self.assertNotIn( self.menu_item_0_1_1.title, rendered, - 'No "{}" string expected in the rendered template'.format( - self.menu_item_0_1_1.title - ) + f'No "{self.menu_item_0_1_1.title}" string expected in the rendered template' ) self.assertNotIn( self.menu_item_0_1_2.title, rendered, - 'No "{}" string expected in the rendered template'.format( - self.menu_item_0_1_2.title - ) + f'No "{self.menu_item_0_1_2.title}" string expected in the rendered template' ) # menu_placeholder_1 # first menu @@ -565,23 +491,17 @@ def test_render_nav_menu_placeholder_1(self): self.assertIn( self.menu_1_0.title, rendered, - 'Expected "{}" string in the rendered template'.format( - self.menu_1_0.title - ) + f'Expected "{self.menu_1_0.title}" string in the rendered template' ) self.assertIn( self.menu_item_1_0_0.title, rendered, - 'Expected "{}" string in the rendered template'.format( - self.menu_item_1_0_0.title - ) + f'Expected "{self.menu_item_1_0_0.title}" string in the rendered template' ) self.assertIn( self.menu_item_1_0_1.title, rendered, - 'Expected "{}" string in the rendered template'.format( - self.menu_item_1_0_1.title - ) + f'Expected "{self.menu_item_1_0_1.title}" string in the rendered template' ) diff --git a/geonode/base/utils.py b/geonode/base/utils.py index 87587927356..8d0d9abf793 100644 --- a/geonode/base/utils.py +++ b/geonode/base/utils.py @@ -78,7 +78,7 @@ def delete_orphaned_thumbs(): remove_thumb(filename) deleted.append(filename) except NotImplementedError as e: - logger.error("Failed to delete orphaned thumbnail '{}': {}".format(filename, e)) + logger.error(f"Failed to delete orphaned thumbnail '{filename}': {e}") return deleted diff --git a/geonode/base/views.py b/geonode/base/views.py index 113079e311f..905dfd90907 100644 --- a/geonode/base/views.py +++ b/geonode/base/views.py @@ -72,7 +72,7 @@ def user_and_group_permission(request, model): ids = request.POST.get("ids") if "cancel" in request.POST or not ids: return HttpResponseRedirect( - '/admin/{}/{}/'.format(model_class._meta.app_label, model) + f'/admin/{model_class._meta.app_label}/{model}/' ) if request.method == 'POST': @@ -104,7 +104,7 @@ def user_and_group_permission(request, model): (permissions_names, resources_names, users_usernames, groups_names, delete_flag)) return HttpResponseRedirect( - '/admin/{}/{}/'.format(model_class._meta.app_label, model) + f'/admin/{model_class._meta.app_label}/{model}/' ) form = UserAndGroupPermissionsForm({ @@ -135,7 +135,7 @@ def batch_modify(request, model): if "cancel" in request.POST or not ids: return HttpResponseRedirect( - '/admin/{model}s/{model}/'.format(model=model.lower()) + f'/admin/{model.lower()}s/{model.lower()}/' ) if request.method == 'POST': @@ -177,7 +177,7 @@ def get_or_create(keyword): keywords_through.objects.bulk_create(new_keywords, ignore_conflicts=True) return HttpResponseRedirect( - '/admin/{model}s/{model}/'.format(model=model.lower()) + f'/admin/{model.lower()}s/{model.lower()}/' ) return render( request, diff --git a/geonode/base/widgets.py b/geonode/base/widgets.py index 067a0937e21..f0b57415ea0 100644 --- a/geonode/base/widgets.py +++ b/geonode/base/widgets.py @@ -17,7 +17,7 @@ def value_from_datadict(self, data, files, name): try: value = super(TaggitSelect2Custom, self).value_from_datadict(data, files, name) if value and ',' not in value: - value = '%s,' % value + value = f'{value},' return value except TypeError: return "" diff --git a/geonode/br/management/__init__.py b/geonode/br/management/__init__.py index 735fc7c7f22..b0fb2f81c70 100644 --- a/geonode/br/management/__init__.py +++ b/geonode/br/management/__init__.py @@ -16,4 +16,4 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -######################################################################### \ No newline at end of file +######################################################################### diff --git a/geonode/br/management/commands/__init__.py b/geonode/br/management/commands/__init__.py index 735fc7c7f22..b0fb2f81c70 100644 --- a/geonode/br/management/commands/__init__.py +++ b/geonode/br/management/commands/__init__.py @@ -16,4 +16,4 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -######################################################################### \ No newline at end of file +######################################################################### diff --git a/geonode/br/management/commands/backup.py b/geonode/br/management/commands/backup.py index 634e7c64908..666626cee02 100644 --- a/geonode/br/management/commands/backup.py +++ b/geonode/br/management/commands/backup.py @@ -23,7 +23,6 @@ import time import shutil import requests -import traceback import re import logging @@ -158,11 +157,10 @@ def execute_backup(self, **options): if app_name == 'br': continue - logger.info("Dumping '"+app_name+"' into '"+dump_name+".json'.") + logger.info(f"Dumping '{app_name}' into '{dump_name}.json'.") # Point stdout at a file for dumping data to. - with open(os.path.join(target_folder, dump_name+'.json'), 'w') as output: + with open(os.path.join(target_folder, f'{dump_name}.json'), 'w') as output: call_command('dumpdata', app_name, format='json', indent=2, stdout=output) - # Store Media Root media_root = settings.MEDIA_ROOT @@ -172,7 +170,7 @@ def execute_backup(self, **options): copy_tree(media_root, media_folder, ignore=utils.ignore_time(config.gs_data_dt_filter[0], config.gs_data_dt_filter[1])) - print("Saved Media Files from '"+media_root+"'.") + print(f"Saved Media Files from '{media_root}'.") # Store Static Root static_root = settings.STATIC_ROOT @@ -182,7 +180,7 @@ def execute_backup(self, **options): copy_tree(static_root, static_folder, ignore=utils.ignore_time(config.gs_data_dt_filter[0], config.gs_data_dt_filter[1])) - print("Saved Static Root from '"+static_root+"'.") + print(f"Saved Static Root from '{static_root}'.") # Store Static Folders static_folders = settings.STATICFILES_DIRS @@ -194,8 +192,10 @@ def execute_backup(self, **options): # skip dumping of static files of apps not located under LOCAL_ROOT path # (check to prevent saving files from site-packages in project-template based GeoNode projects) - if getattr(settings, 'LOCAL_ROOT', None) and not static_files_folder.startswith(settings.LOCAL_ROOT): - print(f"Skipping static directory: {static_files_folder}. It's not located under LOCAL_ROOT path: {settings.LOCAL_ROOT}.") + if getattr(settings, 'LOCAL_ROOT', None) and \ + not static_files_folder.startswith(settings.LOCAL_ROOT): + print(f"Skipping static directory: {static_files_folder}. " + f"It's not located under LOCAL_ROOT path: {settings.LOCAL_ROOT}.") continue static_folder = os.path.join(static_files_folders, @@ -205,7 +205,7 @@ def execute_backup(self, **options): copy_tree(static_files_folder, static_folder, ignore=utils.ignore_time(config.gs_data_dt_filter[0], config.gs_data_dt_filter[1])) - print("Saved Static Files from '"+static_files_folder+"'.") + print(f"Saved Static Files from '{static_files_folder}'.") # Store Template Folders template_folders = [] @@ -224,8 +224,10 @@ def execute_backup(self, **options): # skip dumping of template files of apps not located under LOCAL_ROOT path # (check to prevent saving files from site-packages in project-template based GeoNode projects) - if getattr(settings, 'LOCAL_ROOT', None) and not template_files_folder.startswith(settings.LOCAL_ROOT): - print(f"Skipping template directory: {template_files_folder}. It's not located under LOCAL_ROOT path: {settings.LOCAL_ROOT}.") + if getattr(settings, 'LOCAL_ROOT', None) and \ + not template_files_folder.startswith(settings.LOCAL_ROOT): + print(f"Skipping template directory: {template_files_folder}. " + f"It's not located under LOCAL_ROOT path: {settings.LOCAL_ROOT}.") continue template_folder = os.path.join(template_files_folders, @@ -235,7 +237,7 @@ def execute_backup(self, **options): copy_tree(template_files_folder, template_folder, ignore=utils.ignore_time(config.gs_data_dt_filter[0], config.gs_data_dt_filter[1])) - print("Saved Template Files from '"+template_files_folder+"'.") + print(f"Saved Template Files from '{template_files_folder}'.") # Store Locale Folders locale_folders = settings.LOCALE_PATHS @@ -247,8 +249,10 @@ def execute_backup(self, **options): # skip dumping of locale files of apps not located under LOCAL_ROOT path # (check to prevent saving files from site-packages in project-template based GeoNode projects) - if getattr(settings, 'LOCAL_ROOT', None) and not locale_files_folder.startswith(settings.LOCAL_ROOT): - logger.info(f"Skipping locale directory: {locale_files_folder}. It's not located under LOCAL_ROOT path: {settings.LOCAL_ROOT}.") + if getattr(settings, 'LOCAL_ROOT', None) and \ + not locale_files_folder.startswith(settings.LOCAL_ROOT): + logger.info(f"Skipping locale directory: {locale_files_folder}. " + f"It's not located under LOCAL_ROOT path: {settings.LOCAL_ROOT}.") continue locale_folder = os.path.join(locale_files_folders, @@ -258,20 +262,20 @@ def execute_backup(self, **options): copy_tree(locale_files_folder, locale_folder, ignore=utils.ignore_time(config.gs_data_dt_filter[0], config.gs_data_dt_filter[1])) - logger.info("Saved Locale Files from '"+locale_files_folder+"'.") + logger.info(f"Saved Locale Files from '{locale_files_folder}'.") # Create Final ZIP Archive - backup_archive = os.path.join(backup_dir, dir_time_suffix+'.zip') + backup_archive = os.path.join(backup_dir, f'{dir_time_suffix}.zip') zip_dir(target_folder, backup_archive) # Generate a md5 hash of a backup archive and save it - backup_md5_file = os.path.join(backup_dir, dir_time_suffix+'.md5') + backup_md5_file = os.path.join(backup_dir, f'{dir_time_suffix}.md5') zip_archive_md5 = utils.md5_file_hash(backup_archive) with open(backup_md5_file, 'w') as md5_file: md5_file.write(zip_archive_md5) # Generate the ini file with the current settings used by the backup command - backup_ini_file = os.path.join(backup_dir, dir_time_suffix + '.ini') + backup_ini_file = os.path.join(backup_dir, f'{dir_time_suffix}.ini') with open(backup_ini_file, 'w') as configfile: config.config_parser.write(configfile) @@ -279,11 +283,11 @@ def execute_backup(self, **options): try: shutil.rmtree(target_folder) except Exception: - logger.warning("WARNING: Could not be possible to delete the temp folder: '" + str(target_folder) + "'") + logger.warning(f"WARNING: Could not be possible to delete the temp folder: '{target_folder}'") print("Backup Finished. Archive generated.") - return str(os.path.join(backup_dir, dir_time_suffix+'.zip')) + return str(os.path.join(backup_dir, f'{dir_time_suffix}.zip')) def create_geoserver_backup(self, config, settings, target_folder, ignore_errors): # Create GeoServer Backup @@ -292,12 +296,12 @@ def create_geoserver_backup(self, config, settings, target_folder, ignore_errors passwd = settings.OGC_SERVER['default']['PASSWORD'] geoserver_bk_file = os.path.join(target_folder, 'geoserver_catalog.zip') - logger.info("Dumping 'GeoServer Catalog ["+url+"]' into '"+geoserver_bk_file+"'.") - r = requests.put(url + 'rest/reset/', + logger.info(f"Dumping 'GeoServer Catalog [{url}]' into '{geoserver_bk_file}'.") + r = requests.put(f'{url}rest/reset/', auth=HTTPBasicAuth(user, passwd)) if r.status_code != 200: raise ValueError('Could not reset GeoServer catalog!') - r = requests.put(url + 'rest/reload/', + r = requests.put(f'{url}rest/reload/', auth=HTTPBasicAuth(user, passwd)) if r.status_code != 200: raise ValueError('Could not reload GeoServer catalog!') @@ -309,8 +313,8 @@ def create_geoserver_backup(self, config, settings, target_folder, ignore_errors 'BK_CLEANUP_TEMP=true', 'BK_SKIP_SETTINGS=false', 'BK_SKIP_SECURITY=false', - 'BK_BEST_EFFORT={}'.format('true' if ignore_errors else 'false'), - 'exclude.file.path={}'.format(config.gs_exclude_file_path) + f'BK_BEST_EFFORT={("true" if ignore_errors else "false")}', + f'exclude.file.path={config.gs_exclude_file_path}' ] data = {'backup': {'archiveFile': geoserver_bk_file, 'overwrite': 'true', 'options': {'option': _options}}} @@ -318,19 +322,19 @@ def create_geoserver_backup(self, config, settings, target_folder, ignore_errors 'Accept': 'application/json', 'Content-type': 'application/json' } - r = requests.post(url + 'rest/br/backup/', data=json.dumps(data), + r = requests.post(f'{url}rest/br/backup/', data=json.dumps(data), headers=headers, auth=HTTPBasicAuth(user, passwd)) if r.status_code in (200, 201, 406): try: - r = requests.get(url + 'rest/br/backup.json', + r = requests.get(f'{url}rest/br/backup.json', headers=headers, auth=HTTPBasicAuth(user, passwd), timeout=10) if (r.status_code == 200): gs_backup = r.json() _url = urlparse(gs_backup['backups']['backup'][len(gs_backup['backups']['backup']) - 1]['href']) - _url = '{}?{}'.format(urljoin(url, _url.path), _url.query) + _url = f'{urljoin(url, _url.path)}?{_url.query}' r = requests.get(_url, headers=headers, auth=HTTPBasicAuth(user, passwd), @@ -344,7 +348,7 @@ def create_geoserver_backup(self, config, settings, target_folder, ignore_errors raise ValueError(error_backup.format(url, r.status_code, r.text)) gs_bk_exec_id = gs_backup['backup']['execution']['id'] - r = requests.get(url + 'rest/br/backup/' + str(gs_bk_exec_id) + '.json', + r = requests.get(f'{url}rest/br/backup/{gs_bk_exec_id}.json', headers=headers, auth=HTTPBasicAuth(user, passwd), timeout=10) @@ -355,7 +359,7 @@ def create_geoserver_backup(self, config, settings, target_folder, ignore_errors while (gs_bk_exec_status != 'COMPLETED' and gs_bk_exec_status != 'FAILED'): if (gs_bk_exec_progress != gs_bk_exec_progress_updated): gs_bk_exec_progress_updated = gs_bk_exec_progress - r = requests.get(url + 'rest/br/backup/' + str(gs_bk_exec_id) + '.json', + r = requests.get(f'{url}rest/br/backup/{gs_bk_exec_id}.json', headers=headers, auth=HTTPBasicAuth(user, passwd), timeout=10) @@ -368,7 +372,7 @@ def create_geoserver_backup(self, config, settings, target_folder, ignore_errors gs_bk_exec_status = gs_backup['backup']['execution']['status'] gs_bk_exec_progress = gs_backup['backup']['execution']['progress'] - print(str(gs_bk_exec_status) + ' - ' + gs_bk_exec_progress) + print(f'{gs_bk_exec_status} - {gs_bk_exec_progress}') time.sleep(3) else: raise ValueError(error_backup.format(url, r.status_code, r.text)) @@ -379,7 +383,7 @@ def create_geoserver_backup(self, config, settings, target_folder, ignore_errors os.chmod(geoserver_bk_file, _permissions) status = os.stat(geoserver_bk_file) if oct(status.st_mode & 0o777) != str(oct(_permissions)): - raise Exception("Could not update permissions of {}".format(geoserver_bk_file)) + raise Exception(f"Could not update permissions of {geoserver_bk_file}") else: raise ValueError(error_backup.format(url, r.status_code, r.text)) @@ -390,22 +394,22 @@ def dump_geoserver_raster_data(self, config, settings, target_folder): gs_data_root = os.path.join(config.gs_data_dir, 'geonode') if not os.path.isabs(gs_data_root): gs_data_root = os.path.join(settings.PROJECT_ROOT, '..', gs_data_root) - logger.info("Dumping GeoServer Uploaded Data from '"+gs_data_root+"'.") + logger.info(f"Dumping GeoServer Uploaded Data from '{gs_data_root}'.") if os.path.exists(gs_data_root): gs_data_folder = os.path.join(target_folder, 'gs_data_dir', 'geonode') if not os.path.exists(gs_data_folder): os.makedirs(gs_data_folder) copy_tree(gs_data_root, gs_data_folder, ignore=utils.ignore_time(config.gs_data_dt_filter[0], config.gs_data_dt_filter[1])) - logger.info("Dumped GeoServer Uploaded Data from '"+gs_data_root+"'.") + logger.info(f"Dumped GeoServer Uploaded Data from '{gs_data_root}'.") else: - logger.info("Skipped GeoServer Uploaded Data '"+gs_data_root+"'.") + logger.info(f"Skipped GeoServer Uploaded Data '{gs_data_root}'.") # Dump '$config.gs_data_dir/data/geonode' gs_data_root = os.path.join(config.gs_data_dir, 'data', 'geonode') if not os.path.isabs(gs_data_root): gs_data_root = os.path.join(settings.PROJECT_ROOT, '..', gs_data_root) - logger.info("Dumping GeoServer Uploaded Data from '"+gs_data_root+"'.") + logger.info(f"Dumping GeoServer Uploaded Data from '{gs_data_root}'.") if os.path.exists(gs_data_root): gs_data_folder = os.path.join(target_folder, 'gs_data_dir', 'data', 'geonode') if not os.path.exists(gs_data_folder): @@ -413,9 +417,9 @@ def dump_geoserver_raster_data(self, config, settings, target_folder): copy_tree(gs_data_root, gs_data_folder, ignore=utils.ignore_time(config.gs_data_dt_filter[0], config.gs_data_dt_filter[1])) - logger.info("Dumped GeoServer Uploaded Data from '" + gs_data_root + "'.") + logger.info(f"Dumped GeoServer Uploaded Data from '{gs_data_root}'.") else: - logger.info("Skipped GeoServer Uploaded Data '"+gs_data_root+"'.") + logger.info(f"Skipped GeoServer Uploaded Data '{gs_data_root}'.") def dump_geoserver_vector_data(self, config, settings, target_folder): if (config.gs_dump_vector_data): @@ -455,7 +459,7 @@ def copy_external_resource(abspath): if not os.path.isdir(external_path) and os.path.exists(external_path): shutil.copy2(abspath, external_path) except shutil.SameFileError: - logger.warning("WARNING: {} and {} are the same file!".format(abspath, external_path)) + logger.warning(f"WARNING: {abspath} and {external_path} are the same file!") def match_filename(key, text, regexp=re.compile("^(.+)$")): if key in ('filename', ): @@ -469,7 +473,7 @@ def match_filename(key, text, regexp=re.compile("^(.+)$")): if os.path.exists(abspath): return abspath except Exception: - logger.warning("WARNING: Error while trying to dump {}".format(text)) + logger.warning(f"WARNING: Error while trying to dump {text}") return def match_fileurl(key, text, regexp=re.compile("^file:(.+)$")): @@ -484,7 +488,7 @@ def match_fileurl(key, text, regexp=re.compile("^file:(.+)$")): if os.path.exists(abspath): return abspath except Exception: - logger.warning("WARNING: Error while trying to dump {}".format(text)) + logger.warning(f"WARNING: Error while trying to dump {text}") return def dump_external_resources_from_xml(path): diff --git a/geonode/br/management/commands/restore.py b/geonode/br/management/commands/restore.py index 691d9d2ee52..0219a171358 100755 --- a/geonode/br/management/commands/restore.py +++ b/geonode/br/management/commands/restore.py @@ -18,17 +18,17 @@ # ######################################################################### -import json -import traceback import os +import json import time import uuid import shutil +import logging import zipfile import requests import tempfile import warnings -import logging +import traceback from typing import Union from datetime import datetime @@ -132,7 +132,7 @@ def add_arguments(self, parser): action='store_true', dest='with_logs', default=False, - help="Compares the backup file with restoration logs, and applies it only, if it hasn't been already restored" + help="Compares the backup file with restoration logs, and applies it only, if it hasn't been already restored" # noqa ) parser.add_argument( @@ -192,7 +192,7 @@ def execute_restore(self, **options): backup_files_dir = options.get('backup_files_dir') with_logs = options.get('with_logs') notify = options.get('notify') - soft_reset = options.get('soft_reset') + soft_reset = options.get('soft_reset') # choose backup_file from backup_files_dir, if --backup-files-dir was provided if backup_files_dir: @@ -239,7 +239,7 @@ def execute_restore(self, **options): # otherwise default tmp directory is chosen temp_dir_path = backup_files_dir if os.path.exists(backup_files_dir) else None - restore_folder = os.path.join(temp_dir_path, '{}{}'.format('tmp', str(uuid.uuid4())[:4])) + restore_folder = os.path.join(temp_dir_path, f'tmp{str(uuid.uuid4())[:4]}') try: os.makedirs(restore_folder) except Exception as e: @@ -268,36 +268,37 @@ def execute_restore(self, **options): locale_files_folders = os.path.join(target_folder, utils.LOCALE_PATHS) try: - print(("[Sanity Check] Full Write Access to '{}' ...".format(restore_folder))) + print((f"[Sanity Check] Full Write Access to '{restore_folder}' ...")) chmod_tree(restore_folder) - print(("[Sanity Check] Full Write Access to '{}' ...".format(media_root))) + print((f"[Sanity Check] Full Write Access to '{media_root}' ...")) chmod_tree(media_root) - print(("[Sanity Check] Full Write Access to '{}' ...".format(static_root))) + print((f"[Sanity Check] Full Write Access to '{static_root}' ...")) chmod_tree(static_root) for static_files_folder in static_folders: - print(("[Sanity Check] Full Write Access to '{}' ...".format(static_files_folder))) + print((f"[Sanity Check] Full Write Access to '{static_files_folder}' ...")) chmod_tree(static_files_folder) for template_files_folder in template_folders: - print(("[Sanity Check] Full Write Access to '{}' ...".format(template_files_folder))) + print((f"[Sanity Check] Full Write Access to '{template_files_folder}' ...")) chmod_tree(template_files_folder) for locale_files_folder in locale_folders: - print(("[Sanity Check] Full Write Access to '{}' ...".format(locale_files_folder))) + print((f"[Sanity Check] Full Write Access to '{locale_files_folder}' ...")) chmod_tree(locale_files_folder) except Exception as exception: if notify: restore_notification.apply_async( (admin_emails, backup_file, backup_md5, str(exception))) - print("...Sanity Checks on Folder failed. Please make sure that the current user has full WRITE access to the above folders (and sub-folders or files).") + print("...Sanity Checks on Folder failed. Please make sure that the current user has full WRITE access to the above folders (and sub-folders or files).") # noqa print("Reason:") raise if not skip_geoserver: try: - print(("[Sanity Check] Full Write Access to '{}' ...".format(target_folder))) + print((f"[Sanity Check] Full Write Access to '{target_folder}' ...")) chmod_tree(target_folder) self.restore_geoserver_backup(config, settings, target_folder, - skip_geoserver_info, skip_geoserver_security, ignore_errors, soft_reset) + skip_geoserver_info, skip_geoserver_security, + ignore_errors, soft_reset) self.prepare_geoserver_gwc_config(config, settings) self.restore_geoserver_raster_data(config, settings, target_folder) self.restore_geoserver_vector_data(config, settings, target_folder, soft_reset) @@ -308,7 +309,8 @@ def execute_restore(self, **options): with tempfile.TemporaryDirectory(dir=temp_dir_path) as restore_folder: recovery_folder = extract_archive(recovery_file, restore_folder) self.restore_geoserver_backup(config, settings, recovery_folder, - skip_geoserver_info, skip_geoserver_security, ignore_errors, soft_reset) + skip_geoserver_info, skip_geoserver_security, + ignore_errors, soft_reset) self.restore_geoserver_raster_data(config, settings, recovery_folder) self.restore_geoserver_vector_data(config, settings, recovery_folder, soft_reset) self.restore_geoserver_externals(config, settings, recovery_folder) @@ -354,27 +356,26 @@ def execute_restore(self, **options): raise # Restore Fixtures + abortlater = False for app_name, dump_name in zip(config.app_names, config.dump_names): fixture_file = os.path.join(target_folder, dump_name+'.json') - print("Deserializing "+fixture_file) + print(f"Deserializing '{fixture_file}'") try: call_command('loaddata', fixture_file, app_label=app_name) - except IntegrityError as e: + except IntegrityError: traceback.print_exc() - logger.warning("WARNING: The fixture '"+dump_name+"' fails on integrity check and import is aborted after all fixtures have been checked.") + logger.warning(f"WARNING: The fixture '{dump_name}' fails on integrity check and import is aborted after all fixtures have been checked.") # noqa abortlater = True - except Exception: + except Exception as e: traceback.print_exc() - logger.warning("WARNING: No valid fixture data found for '"+dump_name+"'.") + logger.warning(f"WARNING: No valid fixture data found for '{dump_name}'.") # helpers.load_fixture(app_name, fixture_file) - raise - try: - if abortlater==True: raise e - except UnboundLocalError: - pass - + + if abortlater: + raise IntegrityError() + # Restore Media Root if config.gs_data_dt_filter[0] is None: shutil.rmtree(media_root, ignore_errors=True) @@ -384,7 +385,7 @@ def execute_restore(self, **options): copy_tree(media_folder, media_root) chmod_tree(media_root) - print("Media Files Restored into '"+media_root+"'.") + print(f"Media Files Restored into '{media_root}'.") # Restore Static Root if config.gs_data_dt_filter[0] is None: @@ -395,16 +396,19 @@ def execute_restore(self, **options): copy_tree(static_folder, static_root) chmod_tree(static_root) - print("Static Root Restored into '"+static_root+"'.") + print(f"Static Root Restored into '{static_root}'.") # Restore Static Folders for static_files_folder in static_folders: # skip restoration of static files of apps not located under LOCAL_ROOT path - # (check to prevent overriding files from site-packages in project-template based GeoNode projects) - if getattr(settings, 'LOCAL_ROOT', None) and not static_files_folder.startswith(settings.LOCAL_ROOT): + # (check to prevent overriding files from site-packages + # in project-template based GeoNode projects) + if getattr(settings, 'LOCAL_ROOT', None) and \ + not static_files_folder.startswith(settings.LOCAL_ROOT): print( - f"Skipping static directory: {static_files_folder}. It's not located under LOCAL_ROOT path: {settings.LOCAL_ROOT}.") + f"Skipping static directory: {static_files_folder}. " + f"It's not located under LOCAL_ROOT path: {settings.LOCAL_ROOT}.") continue if config.gs_data_dt_filter[0] is None: @@ -417,16 +421,19 @@ def execute_restore(self, **options): os.path.basename(os.path.normpath(static_files_folder))), static_files_folder) chmod_tree(static_files_folder) - print("Static Files Restored into '"+static_files_folder+"'.") + print(f"Static Files Restored into '{static_files_folder}'.") # Restore Template Folders for template_files_folder in template_folders: # skip restoration of template files of apps not located under LOCAL_ROOT path - # (check to prevent overriding files from site-packages in project-template based GeoNode projects) - if getattr(settings, 'LOCAL_ROOT', None) and not template_files_folder.startswith(settings.LOCAL_ROOT): + # (check to prevent overriding files from site-packages + # in project-template based GeoNode projects) + if getattr(settings, 'LOCAL_ROOT', None) and \ + not template_files_folder.startswith(settings.LOCAL_ROOT): print( - f"Skipping template directory: {template_files_folder}. It's not located under LOCAL_ROOT path: {settings.LOCAL_ROOT}.") + f"Skipping template directory: {template_files_folder}. " + f"It's not located under LOCAL_ROOT path: {settings.LOCAL_ROOT}.") continue if config.gs_data_dt_filter[0] is None: @@ -439,16 +446,19 @@ def execute_restore(self, **options): os.path.basename(os.path.normpath(template_files_folder))), template_files_folder) chmod_tree(template_files_folder) - print("Template Files Restored into '"+template_files_folder+"'.") + print(f"Template Files Restored into '{template_files_folder}'.") # Restore Locale Folders for locale_files_folder in locale_folders: # skip restoration of locale files of apps not located under LOCAL_ROOT path - # (check to prevent overriding files from site-packages in project-template based GeoNode projects) - if getattr(settings, 'LOCAL_ROOT', None) and not locale_files_folder.startswith(settings.LOCAL_ROOT): + # (check to prevent overriding files from site-packages + # in project-template based GeoNode projects) + if getattr(settings, 'LOCAL_ROOT', None) and \ + not locale_files_folder.startswith(settings.LOCAL_ROOT): print( - f"Skipping locale directory: {locale_files_folder}. It's not located under LOCAL_ROOT path: {settings.LOCAL_ROOT}.") + f"Skipping locale directory: {locale_files_folder}. " + f"It's not located under LOCAL_ROOT path: {settings.LOCAL_ROOT}.") continue if config.gs_data_dt_filter[0] is None: @@ -461,7 +471,7 @@ def execute_restore(self, **options): os.path.basename(os.path.normpath(locale_files_folder))), locale_files_folder) chmod_tree(locale_files_folder) - print("Locale Files Restored into '"+locale_files_folder+"'.") + print(f"Locale Files Restored into '{locale_files_folder}'.") call_command('collectstatic', interactive=False) @@ -499,7 +509,7 @@ def execute_restore(self, **options): restore_notification.apply_async( (admin_emails, backup_file, backup_md5)) - print("HINT: If you migrated from another site, do not forget to run the command 'migrate_baseurl' to fix Links") + print("HINT: If you migrated from another site, do not forget to run the command 'migrate_baseurl' to fix Links") # noqa print( " e.g.: DJANGO_SETTINGS_MODULE=my_geonode.settings python manage.py migrate_baseurl " "--source-address=my-host-dev.geonode.org --target-address=my-host-prod.geonode.org" @@ -547,11 +557,11 @@ def parse_backup_files_dir(self, backup_files_dir: str) -> Union[str, None]: for file_name in os.listdir(backup_files_dir): file = os.path.join(backup_files_dir, file_name) if zipfile.is_zipfile(file): - backup_file = file if backup_file is None or os.path.getmtime(file) > os.path.getmtime(backup_file) else backup_file + backup_file = file if backup_file is None or os.path.getmtime(file) > os.path.getmtime(backup_file) else backup_file # noqa if backup_file is None: warnings.warn( - f"Nothing to do. No backup archive found in provided '--backup-file-dir' directory", + "Nothing to do. No backup archive found in provided '--backup-file-dir' directory", RuntimeWarning ) return @@ -567,7 +577,7 @@ def parse_backup_files_dir(self, backup_files_dir: str) -> Union[str, None]: if last_restored_backup.creation_date.timestamp() > os.path.getmtime(backup_file): warnings.warn( f"Nothing to do. The newest backup file from --backup-files-dir: '{backup_file}' " - f"is older than the last restored backup.", + "is older than the last restored backup.", RuntimeWarning ) return @@ -586,7 +596,7 @@ def validate_backup_file_hash(self, backup_file: str) -> str: backup_hash = utils.md5_file_hash(backup_file) # check md5 hash for backup archive, if the md5 file is in place - archive_md5_file = backup_file.rsplit('.', 1)[0] + '.md5' + archive_md5_file = f"{backup_file.rsplit('.', 1)[0]}.md5" if os.path.exists(archive_md5_file): with open(archive_md5_file, 'r') as md5_file: @@ -594,7 +604,7 @@ def validate_backup_file_hash(self, backup_file: str) -> str: if original_backup_md5 != backup_hash: raise RuntimeError( - f'Backup archive integrity failure. MD5 hash of the archive ' + 'Backup archive integrity failure. MD5 hash of the archive ' f'is different from the one provided in {archive_md5_file}' ) else: @@ -615,14 +625,16 @@ def check_backup_ini_settings(self, backup_file: str) -> str: :return: backup_ini_file_path original settings used by the backup file """ # check if the ini file is in place - backup_ini_file_path = backup_file.rsplit('.', 1)[0] + '.ini' + backup_ini_file_path = f"{backup_file.rsplit('.', 1)[0]}.ini" if os.path.exists(backup_ini_file_path): return backup_ini_file_path return None - def restore_geoserver_backup(self, config, settings, target_folder, skip_geoserver_info, skip_geoserver_security, ignore_errors, soft_reset): + def restore_geoserver_backup(self, config, settings, target_folder, + skip_geoserver_info, skip_geoserver_security, + ignore_errors, soft_reset): """Restore GeoServer Catalog""" url = settings.OGC_SERVER['default']['LOCATION'] user = settings.OGC_SERVER['default']['USER'] @@ -630,19 +642,18 @@ def restore_geoserver_backup(self, config, settings, target_folder, skip_geoserv geoserver_bk_file = os.path.join(target_folder, 'geoserver_catalog.zip') if not os.path.exists(geoserver_bk_file) or not os.access(geoserver_bk_file, os.R_OK): - raise Exception(('ERROR: geoserver restore: ' + - 'file "{}" not found.'.format(geoserver_bk_file))) + raise Exception((f'ERROR: geoserver restore: file "{geoserver_bk_file}" not found.')) - print("Restoring 'GeoServer Catalog ["+url+"]' from '"+geoserver_bk_file+"'.") + print(f"Restoring 'GeoServer Catalog [{url}]' from '{geoserver_bk_file}'.") # Best Effort Restore: 'options': {'option': ['BK_BEST_EFFORT=true']} _options = [ - 'BK_PURGE_RESOURCES={}'.format('true' if not soft_reset else 'false'), + f"BK_PURGE_RESOURCES={'true' if not soft_reset else 'false'}", 'BK_CLEANUP_TEMP=true', - 'BK_SKIP_SETTINGS={}'.format('true' if skip_geoserver_info else 'false'), - 'BK_SKIP_SECURITY={}'.format('true' if skip_geoserver_security else 'false'), - 'BK_BEST_EFFORT={}'.format('true' if ignore_errors else 'false'), - 'exclude.file.path={}'.format(config.gs_exclude_file_path) + f'BK_SKIP_SETTINGS={("true" if skip_geoserver_info else "false")}', + f'BK_SKIP_SECURITY={("true" if skip_geoserver_security else "false")}', + f'BK_BEST_EFFORT={("true" if ignore_errors else "false")}', + f'exclude.file.path={config.gs_exclude_file_path}' ] data = {'restore': {'archiveFile': geoserver_bk_file, 'options': {'option': _options}}} @@ -650,14 +661,14 @@ def restore_geoserver_backup(self, config, settings, target_folder, skip_geoserv 'Accept': 'application/json', 'Content-type': 'application/json' } - r = requests.post(url + 'rest/br/restore/', data=json.dumps(data), + r = requests.post(f'{url}rest/br/restore/', data=json.dumps(data), headers=headers, auth=HTTPBasicAuth(user, passwd)) error_backup = 'Could not successfully restore GeoServer ' + \ 'catalog [{}rest/br/restore/]: {} - {}' if r.status_code in (200, 201, 406): try: - r = requests.get(url + 'rest/br/restore.json', + r = requests.get(f'{url}rest/br/restore.json', headers=headers, auth=HTTPBasicAuth(user, passwd), timeout=10) @@ -665,7 +676,7 @@ def restore_geoserver_backup(self, config, settings, target_folder, skip_geoserv if (r.status_code == 200): gs_backup = r.json() _url = urlparse(gs_backup['restores']['restore'][len(gs_backup['restores']['restore']) - 1]['href']) - _url = '{}?{}'.format(urljoin(url, _url.path), _url.query) + _url = f'{urljoin(url, _url.path)}?{_url.query}' r = requests.get(_url, headers=headers, auth=HTTPBasicAuth(user, passwd), @@ -680,7 +691,7 @@ def restore_geoserver_backup(self, config, settings, target_folder, skip_geoserv raise ValueError(error_backup.format(url, r.status_code, r.text)) gs_bk_exec_id = gs_backup['restore']['execution']['id'] - r = requests.get(url + 'rest/br/restore/' + str(gs_bk_exec_id) + '.json', + r = requests.get(f'{url}rest/br/restore/{gs_bk_exec_id}.json', headers=headers, auth=HTTPBasicAuth(user, passwd), timeout=10) @@ -692,7 +703,7 @@ def restore_geoserver_backup(self, config, settings, target_folder, skip_geoserv while (gs_bk_exec_status != 'COMPLETED' and gs_bk_exec_status != 'FAILED'): if (gs_bk_exec_progress != gs_bk_exec_progress_updated): gs_bk_exec_progress_updated = gs_bk_exec_progress - r = requests.get(url + 'rest/br/restore/' + str(gs_bk_exec_id) + '.json', + r = requests.get(f'{url}rest/br/restore/{gs_bk_exec_id}.json', headers=headers, auth=HTTPBasicAuth(user, passwd), timeout=10) @@ -706,7 +717,7 @@ def restore_geoserver_backup(self, config, settings, target_folder, skip_geoserv gs_bk_exec_status = gs_backup['restore']['execution']['status'] gs_bk_exec_progress = gs_backup['restore']['execution']['progress'] - print(str(gs_bk_exec_status) + ' - ' + gs_bk_exec_progress) + print(f'{gs_bk_exec_status} - {gs_bk_exec_progress}') time.sleep(3) else: raise ValueError(error_backup.format(url, r.status_code, r.text)) @@ -728,7 +739,7 @@ def prepare_geoserver_gwc_config(self, config, settings): gwc_layers_root = os.path.join(settings.PROJECT_ROOT, '..', gwc_layers_root) try: shutil.rmtree(gwc_layers_root) - print('Cleaned out old GeoServer GWC Layers Config: ' + gwc_layers_root) + print(f'Cleaned out old GeoServer GWC Layers Config: {gwc_layers_root}') except Exception: pass if not os.path.exists(gwc_layers_root): @@ -748,10 +759,10 @@ def restore_geoserver_raster_data(self, config, settings, target_folder): os.makedirs(gs_data_root) copy_tree(gs_data_folder, gs_data_root) - print("GeoServer Uploaded Raster Data Restored to '" + gs_data_root + "'.") + print(f"GeoServer Uploaded Raster Data Restored to '{gs_data_root}'.") else: print(('Skipping geoserver raster data restore: ' + - 'directory "{}" not found.'.format(gs_data_folder))) + f'directory "{gs_data_folder}" not found.')) # Restore '$config.gs_data_dir/data/geonode' gs_data_folder = os.path.join(target_folder, 'gs_data_dir', 'data', 'geonode') @@ -764,10 +775,10 @@ def restore_geoserver_raster_data(self, config, settings, target_folder): os.makedirs(gs_data_root) copy_tree(gs_data_folder, gs_data_root) - print("GeoServer Uploaded Data Restored to '" + gs_data_root + "'.") + print(f"GeoServer Uploaded Data Restored to '{gs_data_root}'.") else: print(('Skipping geoserver raster data restore: ' + - 'directory "{}" not found.'.format(gs_data_folder))) + f'directory "{gs_data_folder}" not found.')) def restore_geoserver_vector_data(self, config, settings, target_folder, soft_reset): """Restore Vectorial Data from DB""" @@ -776,7 +787,7 @@ def restore_geoserver_vector_data(self, config, settings, target_folder, soft_re gs_data_folder = os.path.join(target_folder, 'gs_data_dir', 'geonode') if not os.path.exists(gs_data_folder): print(('Skipping geoserver vector data restore: ' + - 'directory "{}" not found.'.format(gs_data_folder))) + f'directory "{gs_data_folder}" not found.')) return datastore = settings.OGC_SERVER['default']['DATASTORE'] diff --git a/geonode/br/management/commands/utils/utils.py b/geonode/br/management/commands/utils/utils.py index 537b8a7d7d8..8d0c793a9fc 100644 --- a/geonode/br/management/commands/utils/utils.py +++ b/geonode/br/management/commands/utils/utils.py @@ -174,7 +174,7 @@ def get_db_conn(db_name, db_user, db_port, db_host, db_passwd): db_host = db_host if db_host is not None else 'localhost' db_port = db_port if db_port is not None else 5432 conn = psycopg2.connect( - "dbname='%s' user='%s' port='%s' host='%s' password='%s'" % (db_name, db_user, db_port, db_host, db_passwd) + f"dbname='{db_name}' user='{db_user}' port='{db_port}' host='{db_host}' password='{db_passwd}'" ) return conn @@ -227,7 +227,7 @@ def flush_db(db_name, db_user, db_port, db_host, db_passwd): curs = conn.cursor() try: - sql_dump = """SELECT tablename from pg_tables where tableowner = '%s'""" % (db_user) + sql_dump = f"""SELECT tablename from pg_tables where tableowner = '{db_user}'""" curs.execute(sql_dump) pg_tables = curs.fetchall() for table in pg_tables: @@ -255,7 +255,7 @@ def dump_db(config, db_name, db_user, db_port, db_host, db_passwd, target_folder curs = conn.cursor() try: - sql_dump = """SELECT tablename from pg_tables where tableowner = '%s'""" % (db_user) + sql_dump = f"""SELECT tablename from pg_tables where tableowner = '{db_user}'""" curs.execute(sql_dump) pg_all_tables = [table[0] for table in curs.fetchall()] pg_tables = [] @@ -272,7 +272,7 @@ def dump_db(config, db_name, db_user, db_port, db_host, db_passwd, target_folder pg_tables = pg_all_tables for table in pg_tables: - logger.info("Dumping GeoServer Vectorial Data : {}:{}".format(db_name, table)) + logger.info(f"Dumping GeoServer Vectorial Data : {db_name}:{table}") os.system('PGPASSWORD="' + db_passwd + '" ' + config.pg_dump_cmd + ' -h ' + db_host + ' -p ' + str(db_port) + ' -U ' + db_user + ' -F c -b' + ' -t \'"' + str(table) + '"\' -f ' + @@ -301,7 +301,7 @@ def restore_db(config, db_name, db_user, db_port, db_host, db_passwd, source_fol file_names = [fn for fn in os.listdir(source_folder) if any(fn.endswith(ext) for ext in included_extenstions)] for table in file_names: - logger.info("Restoring GeoServer Vectorial Data : {}:{} ".format(db_name, os.path.splitext(table)[0])) + logger.info(f"Restoring GeoServer Vectorial Data : {db_name}:{os.path.splitext(table)[0]} ") pg_rstcmd = 'PGPASSWORD="' + db_passwd + '" ' + config.pg_restore_cmd + ' -h ' + db_host + \ ' -p ' + str(db_port) + ' -U ' + db_user + ' --role=' + db_user + \ ' -F c -t "' + os.path.splitext(table)[0] + '" ' +\ @@ -323,13 +323,13 @@ def restore_db(config, db_name, db_user, db_port, db_host, db_passwd, source_fol def remove_existing_tables(db_name, db_user, db_port, db_host, db_passwd): conn = get_db_conn(db_name, db_user, db_port, db_host, db_passwd) curs = conn.cursor() - table_list = """SELECT tablename from pg_tables where tableowner = '%s'""" % (db_user) + table_list = f"""SELECT tablename from pg_tables where tableowner = '{db_user}'""" try: curs.execute(table_list) pg_all_tables = [table[0] for table in curs.fetchall()] for pg_table in pg_all_tables: - logger.info("Dropping existing GeoServer Vectorial Data : {}:{} ".format(db_name, pg_table)) + logger.info(f"Dropping existing GeoServer Vectorial Data : {db_name}:{pg_table} ") curs.execute(f"DROP TABLE {pg_table} CASCADE") conn.commit() @@ -366,9 +366,9 @@ def confirm(prompt=None, resp=False): prompt = 'Confirm' if resp: - prompt = '%s [%s]|%s: ' % (prompt, 'y', 'n') + prompt = f'{prompt} [y]|n: ' else: - prompt = '%s [%s]|%s: ' % (prompt, 'n', 'y') + prompt = f'{prompt} [n]|y: ' while True: ans = input(prompt) @@ -427,10 +427,10 @@ def glob2re(pat): c = pat[i] i = i+1 if c == '*': - #res = res + '.*' + # res = res + '.*' res = res + '[^/]*' elif c == '?': - #res = res + '.' + # res = res + '.' res = res + '[^/]' elif c == '[': j = i @@ -449,7 +449,7 @@ def glob2re(pat): stuff = '^' + stuff[1:] elif stuff[0] == '^': stuff = '\\' + stuff - res = '%s[%s]' % (res, stuff) + res = f'{res}[{stuff}]' else: res = res + re.escape(c) return res + r'\Z(?ms)' diff --git a/geonode/catalogue/__init__.py b/geonode/catalogue/__init__.py index ae1bd7f379c..29a36475ccd 100644 --- a/geonode/catalogue/__init__.py +++ b/geonode/catalogue/__init__.py @@ -34,7 +34,7 @@ # If settings.CATALOGUE is defined, we expect it to be properly named if DEFAULT_CATALOGUE_ALIAS not in settings.CATALOGUE: - raise ImproperlyConfigured("You must define a '%s' CATALOGUE" % DEFAULT_CATALOGUE_ALIAS) + raise ImproperlyConfigured(f"You must define a '{DEFAULT_CATALOGUE_ALIAS}' CATALOGUE") def load_backend(backend_name): @@ -68,7 +68,7 @@ def load_backend(backend_name): def default_catalogue_backend(): """Get the default backend """ - msg = "There is no '%s' backend in CATALOGUE" % DEFAULT_CATALOGUE_ALIAS + msg = f"There is no '{DEFAULT_CATALOGUE_ALIAS}' backend in CATALOGUE" assert DEFAULT_CATALOGUE_ALIAS in settings.CATALOGUE, msg return settings.CATALOGUE[DEFAULT_CATALOGUE_ALIAS] diff --git a/geonode/catalogue/backends/generic.py b/geonode/catalogue/backends/generic.py index a983a9479db..5b2ad9a430d 100644 --- a/geonode/catalogue/backends/generic.py +++ b/geonode/catalogue/backends/generic.py @@ -68,7 +68,7 @@ def __init__(self, *args, **kwargs): upurl = urlparse(self.url) - self.base = '%s://%s/' % (upurl.scheme, upurl.netloc) + self.base = f'{upurl.scheme}://{upurl.netloc}/' # User and Password are optional if 'USER' in kwargs: @@ -112,14 +112,15 @@ def get_by_uuid(self, uuid): return None def url_for_uuid(self, uuid, outputschema): - return "%s?%s" % (self.url, urlencode({ + _query_string = urlencode({ "request": "GetRecordById", "service": "CSW", "version": "2.0.2", "id": uuid, "outputschema": outputschema, "elementsetname": "full" - })) + }) + return f"{self.url}?{_query_string}" def urls_for_uuid(self, uuid): """returns list of valid GetRecordById URLs for a given record""" diff --git a/geonode/catalogue/backends/pycsw_plugin.py b/geonode/catalogue/backends/pycsw_plugin.py index de97c2948a6..4e1c3c1255e 100644 --- a/geonode/catalogue/backends/pycsw_plugin.py +++ b/geonode/catalogue/backends/pycsw_plugin.py @@ -1,217 +1,217 @@ -# -*- coding: utf-8 -*- -######################################################################### -# -# Copyright (C) 2018 OSGeo -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -######################################################################### - -import logging - -from django.db import connection -from django.db.models import Max, Min, Count -from django.conf import settings - -from pycsw.core.repository import Repository, query_spatial, get_geometry_area - -from geonode.base.models import ResourceBase -from geonode.layers.models import Layer - -from pycsw.core import util - -LOGGER = logging.getLogger(__name__) - -GEONODE_SERVICE_TYPES = { - # 'GeoNode enum': 'CSW enum' - 'http://www.opengis.net/cat/csw/2.0.2': 'OGC:CSW', - 'http://www.opengis.net/wms': 'OGC:WMS', - 'http://www.opengis.net/wmts/1.0': 'OGC:WMTS', - 'https://wiki.osgeo.org/wiki/TMS': 'OSGeo:TMS', - 'urn:x-esri:serviceType:ArcGIS:MapServer': 'ESRI:ArcGIS:MapServer', - 'urn:x-esri:serviceType:ArcGIS:ImageServer': 'ESRI:ArcGIS:ImageServer' -} - - -class GeoNodeRepository(Repository): - """ - Class to interact with underlying repository - """ - - def __init__(self, context, repo_filter=None): - """ - Initialize repository - """ - - self.context = context - self.filter = repo_filter - self.fts = False - self.label = 'GeoNode' - self.local_ingest = True - - self.dbtype = settings.DATABASES['default']['ENGINE'].split('.')[-1] - - # GeoNode PostgreSQL installs are PostGIS enabled - if self.dbtype == 'postgis': - self.dbtype = 'postgresql+postgis+wkt' - - if self.dbtype in {'sqlite', 'sqlite3'}: # load SQLite query bindings - connection.connection.create_function( - 'query_spatial', 4, query_spatial) - connection.connection.create_function( - 'get_anytext', 1, util.get_anytext) - connection.connection.create_function( - 'get_geometry_area', 1, get_geometry_area) - - # generate core queryables db and obj bindings - self.queryables = {} - - for tname in self.context.model['typenames']: - for qname in self.context.model['typenames'][tname]['queryables']: - self.queryables[qname] = {} - items = list(self.context.model['typenames'][tname]['queryables'][qname].items()) - - for qkey, qvalue in items: - self.queryables[qname][qkey] = qvalue - - # flatten all queryables - # TODO smarter way of doing this - self.queryables['_all'] = {} - for qbl in self.queryables: - self.queryables['_all'].update(self.queryables[qbl]) - self.queryables['_all'].update(self.context.md_core_model['mappings']) - - if 'Harvest' in self.context.model['operations'] and 'Transaction' in self.context.model['operations']: - self.context.model['operations']['Harvest']['parameters']['ResourceType']['values'] = list(GEONODE_SERVICE_TYPES.keys()) # noqa - self.context.model['operations']['Transaction']['parameters']['TransactionSchemas']['values'] = list(GEONODE_SERVICE_TYPES.keys()) # noqa - - def dataset(self): - """ - Stub to mock a pycsw dataset object for Transactions - """ - return type('ResourceBase', (object,), {}) - - def query_ids(self, ids): - """ - Query by list of identifiers - """ - - results = self._get_repo_filter( - Layer.objects).filter( - uuid__in=ids).all() - - if len(results) == 0: # try services - results = self._get_repo_filter( - ResourceBase.objects).filter( - uuid__in=ids).all() - - return results - - def query_domain(self, domain, typenames, - domainquerytype='list', count=False): - """ - Query by property domain values - """ - - objects = self._get_repo_filter(Layer.objects) - - if domainquerytype == 'range': - return [tuple(objects.aggregate( - Min(domain), Max(domain)).values())] - else: - if count: - return [(d[domain], d['%s__count' % domain]) - for d in objects.values(domain).annotate(Count(domain))] - else: - return objects.values_list(domain).distinct() - - def query_insert(self, direction='max'): - """ - Query to get latest (default) or earliest update to repository - """ - if direction == 'min': - return Layer.objects.aggregate( - Min('last_updated'))['last_updated__min'].strftime('%Y-%m-%dT%H:%M:%SZ') - return self._get_repo_filter(Layer.objects).aggregate( - Max('last_updated'))['last_updated__max'].strftime('%Y-%m-%dT%H:%M:%SZ') - - def query_source(self, source): - """ - Query by source - """ - return self._get_repo_filter(Layer.objects).filter(url=source) - - def query(self, constraint, sortby=None, typenames=None, - maxrecords=10, startposition=0): - """ - Query records from underlying repository - """ - - # run the raw query and get total - # we want to exclude layers which are not valid, as it is done in the - # search engine - if 'where' in constraint: # GetRecords with constraint - query = self._get_repo_filter( - Layer.objects).filter(alternate__isnull=False).extra( - where=[ - constraint['where']], - params=constraint['values']) - else: # GetRecords sans constraint - query = self._get_repo_filter( - Layer.objects).filter(alternate__isnull=False) - - total = query.count() - - # apply sorting, limit and offset - if sortby is not None: - if 'spatial' in sortby and sortby['spatial']: # spatial sort - desc = False - if sortby['order'] == 'DESC': - desc = True - query = query.all() - return [str(total), - sorted(query, - key=lambda x: float(util.get_geometry_area( - getattr(x, sortby['propertyname']))), - reverse=desc, - )[startposition:startposition + int(maxrecords)]] - else: - if sortby['order'] == 'DESC': - pname = '-%s' % sortby['propertyname'] - else: - pname = sortby['propertyname'] - return [str(total), - query.order_by(pname)[startposition:startposition + int(maxrecords)]] - else: # no sort - return [str(total), query.all()[ - startposition:startposition + int(maxrecords)]] - - def delete(self, constraint): - """ - Delete a record from the repository - """ - - results = self._get_repo_filter(ResourceBase.objects).extra(where=[constraint['where']], - params=constraint['values']).all() - deleted = len(results) - results.delete() - return deleted - - def _get_repo_filter(self, query): - """ - Apply repository wide side filter / mask query - """ - if self.filter is not None: - return query.extra(where=[self.filter]) - return query +# -*- coding: utf-8 -*- +######################################################################### +# +# Copyright (C) 2018 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging + +from django.db import connection +from django.db.models import Max, Min, Count +from django.conf import settings + +from pycsw.core.repository import Repository, query_spatial, get_geometry_area + +from geonode.base.models import ResourceBase +from geonode.layers.models import Layer + +from pycsw.core import util + +LOGGER = logging.getLogger(__name__) + +GEONODE_SERVICE_TYPES = { + # 'GeoNode enum': 'CSW enum' + 'http://www.opengis.net/cat/csw/2.0.2': 'OGC:CSW', + 'http://www.opengis.net/wms': 'OGC:WMS', + 'http://www.opengis.net/wmts/1.0': 'OGC:WMTS', + 'https://wiki.osgeo.org/wiki/TMS': 'OSGeo:TMS', + 'urn:x-esri:serviceType:ArcGIS:MapServer': 'ESRI:ArcGIS:MapServer', + 'urn:x-esri:serviceType:ArcGIS:ImageServer': 'ESRI:ArcGIS:ImageServer' +} + + +class GeoNodeRepository(Repository): + """ + Class to interact with underlying repository + """ + + def __init__(self, context, repo_filter=None): + """ + Initialize repository + """ + + self.context = context + self.filter = repo_filter + self.fts = False + self.label = 'GeoNode' + self.local_ingest = True + + self.dbtype = settings.DATABASES['default']['ENGINE'].split('.')[-1] + + # GeoNode PostgreSQL installs are PostGIS enabled + if self.dbtype == 'postgis': + self.dbtype = 'postgresql+postgis+wkt' + + if self.dbtype in {'sqlite', 'sqlite3'}: # load SQLite query bindings + connection.connection.create_function( + 'query_spatial', 4, query_spatial) + connection.connection.create_function( + 'get_anytext', 1, util.get_anytext) + connection.connection.create_function( + 'get_geometry_area', 1, get_geometry_area) + + # generate core queryables db and obj bindings + self.queryables = {} + + for tname in self.context.model['typenames']: + for qname in self.context.model['typenames'][tname]['queryables']: + self.queryables[qname] = {} + items = list(self.context.model['typenames'][tname]['queryables'][qname].items()) + + for qkey, qvalue in items: + self.queryables[qname][qkey] = qvalue + + # flatten all queryables + # TODO smarter way of doing this + self.queryables['_all'] = {} + for qbl in self.queryables: + self.queryables['_all'].update(self.queryables[qbl]) + self.queryables['_all'].update(self.context.md_core_model['mappings']) + + if 'Harvest' in self.context.model['operations'] and 'Transaction' in self.context.model['operations']: + self.context.model['operations']['Harvest']['parameters']['ResourceType']['values'] = list(GEONODE_SERVICE_TYPES.keys()) # noqa + self.context.model['operations']['Transaction']['parameters']['TransactionSchemas']['values'] = list(GEONODE_SERVICE_TYPES.keys()) # noqa + + def dataset(self): + """ + Stub to mock a pycsw dataset object for Transactions + """ + return type('ResourceBase', (object,), {}) + + def query_ids(self, ids): + """ + Query by list of identifiers + """ + + results = self._get_repo_filter( + Layer.objects).filter( + uuid__in=ids).all() + + if len(results) == 0: # try services + results = self._get_repo_filter( + ResourceBase.objects).filter( + uuid__in=ids).all() + + return results + + def query_domain(self, domain, typenames, + domainquerytype='list', count=False): + """ + Query by property domain values + """ + + objects = self._get_repo_filter(Layer.objects) + + if domainquerytype == 'range': + return [tuple(objects.aggregate( + Min(domain), Max(domain)).values())] + else: + if count: + return [(d[domain], d[f'{domain}__count']) + for d in objects.values(domain).annotate(Count(domain))] + else: + return objects.values_list(domain).distinct() + + def query_insert(self, direction='max'): + """ + Query to get latest (default) or earliest update to repository + """ + if direction == 'min': + return Layer.objects.aggregate( + Min('last_updated'))['last_updated__min'].strftime('%Y-%m-%dT%H:%M:%SZ') + return self._get_repo_filter(Layer.objects).aggregate( + Max('last_updated'))['last_updated__max'].strftime('%Y-%m-%dT%H:%M:%SZ') + + def query_source(self, source): + """ + Query by source + """ + return self._get_repo_filter(Layer.objects).filter(url=source) + + def query(self, constraint, sortby=None, typenames=None, + maxrecords=10, startposition=0): + """ + Query records from underlying repository + """ + + # run the raw query and get total + # we want to exclude layers which are not valid, as it is done in the + # search engine + if 'where' in constraint: # GetRecords with constraint + query = self._get_repo_filter( + Layer.objects).filter(alternate__isnull=False).extra( + where=[ + constraint['where']], + params=constraint['values']) + else: # GetRecords sans constraint + query = self._get_repo_filter( + Layer.objects).filter(alternate__isnull=False) + + total = query.count() + + # apply sorting, limit and offset + if sortby is not None: + if 'spatial' in sortby and sortby['spatial']: # spatial sort + desc = False + if sortby['order'] == 'DESC': + desc = True + query = query.all() + return [str(total), + sorted(query, + key=lambda x: float(util.get_geometry_area( + getattr(x, sortby['propertyname']))), + reverse=desc, + )[startposition:startposition + int(maxrecords)]] + else: + if sortby['order'] == 'DESC': + pname = f"-{sortby['propertyname']}" + else: + pname = sortby['propertyname'] + return [str(total), + query.order_by(pname)[startposition:startposition + int(maxrecords)]] + else: # no sort + return [str(total), query.all()[ + startposition:startposition + int(maxrecords)]] + + def delete(self, constraint): + """ + Delete a record from the repository + """ + + results = self._get_repo_filter(ResourceBase.objects).extra(where=[constraint['where']], + params=constraint['values']).all() + deleted = len(results) + results.delete() + return deleted + + def _get_repo_filter(self, query): + """ + Apply repository wide side filter / mask query + """ + if self.filter is not None: + return query.extra(where=[self.filter]) + return query diff --git a/geonode/catalogue/metadataxsl/management/commands/addmissinglinks.py b/geonode/catalogue/metadataxsl/management/commands/addmissinglinks.py index 5ad64b981bf..999b178b9bd 100644 --- a/geonode/catalogue/metadataxsl/management/commands/addmissinglinks.py +++ b/geonode/catalogue/metadataxsl/management/commands/addmissinglinks.py @@ -30,15 +30,14 @@ class Command(BaseCommand): def handle(self, *args, **options): for resource in ResourceBase.objects.all(): - print('Checking resource with id {}'.format(resource.id)) + print(f'Checking resource with id {resource.id}') # check ISO link exists isolink = Link.objects.filter(resource_id=resource.id, link_type='metadata', name='ISO') - - if(isolink): - print(' ISO link found for resource {} "{}"'.format(resource.id, resource.title)) + if isolink: + print(f' ISO link found for resource {resource.id} "{resource.title}"') created = add_xsl_link(resource) - if (created): + if created: print(' XSL link created') else: - print(' ISO link NOT found for resource {} "{}"'.format(resource.id, resource.title)) + print(f' ISO link NOT found for resource {resource.id} "{resource.title}"') diff --git a/geonode/catalogue/metadataxsl/views.py b/geonode/catalogue/metadataxsl/views.py index bcd20725f6b..6ef2c47f6f1 100644 --- a/geonode/catalogue/metadataxsl/views.py +++ b/geonode/catalogue/metadataxsl/views.py @@ -48,7 +48,7 @@ def prefix_xsl_line(req, id): logger.debug(record.xml) except Exception: logger.debug(traceback.format_exc()) - msg = 'Could not connect to catalogue to save information for layer "%s"' % str(resource) + msg = f'Could not connect to catalogue to save information for layer "{str(resource)}"' return HttpResponse( msg ) @@ -66,8 +66,8 @@ def prefix_xsl_line(req, id): "Resource Metadata not available!" ) site_url = settings.SITEURL.rstrip('/') if settings.SITEURL.startswith('http') else settings.SITEURL - xsl_path = '{}/static/metadataxsl/metadata.xsl'.format(site_url) - xsl_line = ''.format(xsl_path) + xsl_path = f'{site_url}/static/metadataxsl/metadata.xsl' + xsl_line = f'' return HttpResponse( xsl_line + xml, diff --git a/geonode/catalogue/models.py b/geonode/catalogue/models.py index 63fd5d5dce6..dff26029e90 100644 --- a/geonode/catalogue/models.py +++ b/geonode/catalogue/models.py @@ -49,7 +49,7 @@ def catalogue_post_save(instance, sender, **kwargs): catalogue.create_record(instance) record = catalogue.get_record(instance.uuid) except EnvironmentError as err: - msg = 'Could not connect to catalogue to save information for layer "%s"' % instance.name + msg = f'Could not connect to catalogue to save information for layer "{instance.name}"' if err.errno == errno.ECONNREFUSED: LOGGER.warn(msg, err) return @@ -63,7 +63,7 @@ def catalogue_post_save(instance, sender, **kwargs): return if not hasattr(record, 'links'): - msg = ('Metadata record for %s should contain links.' % instance.title) + msg = f'Metadata record for {instance.title} should contain links.' raise Exception(msg) # Create the different metadata links with the available formats diff --git a/geonode/catalogue/views.py b/geonode/catalogue/views.py index 35372122884..e0d0f3284b9 100644 --- a/geonode/catalogue/views.py +++ b/geonode/catalogue/views.py @@ -55,15 +55,15 @@ def csw_global_dispatch(request, layer_filter=None): if access_token and access_token.is_expired(): access_token = None - absolute_uri = ('%s' % request.build_absolute_uri()) - query_string = ('%s' % request.META['QUERY_STRING']) + absolute_uri = f'{request.build_absolute_uri()}' + query_string = f"{request.META['QUERY_STRING']}" env = request.META.copy() if access_token and not access_token.is_expired(): env.update({'access_token': access_token.token}) if 'access_token' not in query_string: - absolute_uri = ('%s&access_token=%s' % (absolute_uri, access_token.token)) - query_string = ('%s&access_token=%s' % (query_string, access_token.token)) + absolute_uri = f'{absolute_uri}&access_token={access_token.token}' + query_string = f'{query_string}&access_token={access_token.token}' env.update({'local.app_root': os.path.dirname(__file__), 'REQUEST_URI': absolute_uri, @@ -100,8 +100,7 @@ def csw_global_dispatch(request, layer_filter=None): authorized_layers_filter = "id IN " + authorized_layers mdict['repository']['filter'] += " AND " + authorized_layers_filter if request.user and request.user.is_authenticated: - mdict['repository']['filter'] = "({}) OR ({})".format(mdict['repository']['filter'], - authorized_layers_filter) + mdict['repository']['filter'] = f"({mdict['repository']['filter']}) OR ({authorized_layers_filter})" else: authorized_layers_filter = "id = -9999" mdict['repository']['filter'] += " AND " + authorized_layers_filter @@ -188,7 +187,7 @@ def csw_global_dispatch(request, layer_filter=None): url.text += "?" else: url.text += "&" - url.text += ("access_token=%s" % (access_token.token)) + url.text += f"access_token={access_token.token}" url.set('updated', 'yes') except Exception: pass @@ -233,7 +232,7 @@ def get_CSV_spec_char(): # format value to unicode str without ';' char def fst(value): chrs = get_CSV_spec_char() - result = "{}".format(value) + result = str(value) result = result.replace(chrs["separator"], ',').replace('\\n', ' ').replace('\r\n', ' ') return result @@ -364,9 +363,7 @@ def csw_render_extra_format_html(request, layeruuid, resname): layer = Layer.objects.get(resourcebase_ptr_id=resource.id) extra_res_md['atrributes'] = '' for attr in layer.attribute_set.all(): - s = "{}{}{}".format( - attr.attribute, attr.attribute_label, attr.description - ) + s = f"{attr.attribute}{attr.attribute_label}{attr.description}" extra_res_md['atrributes'] += s pocr = ContactRole.objects.get( diff --git a/geonode/celery_app.py b/geonode/celery_app.py index af4cc35329a..a849bf10dac 100644 --- a/geonode/celery_app.py +++ b/geonode/celery_app.py @@ -58,5 +58,5 @@ def test(arg): name='{{project_name}}.debug_task', queue='default') def debug_task(self): - _log("Request: {!r}".format(self.request)) + _log(f"Request: {self.request}") """ diff --git a/geonode/client/conf.py b/geonode/client/conf.py index d17d9ec1a74..95bce262fa6 100644 --- a/geonode/client/conf.py +++ b/geonode/client/conf.py @@ -31,11 +31,11 @@ def load_path_attr(path): try: mod = importlib.import_module(module) except ImportError as e: - raise ImproperlyConfigured("Error importing {0}: '{1}'".format(module, e)) + raise ImproperlyConfigured(f"Error importing {module}: '{e}'") try: attr = getattr(mod, attr) except AttributeError: - raise ImproperlyConfigured("Module '{0}' does not define a '{1}'".format(module, attr)) + raise ImproperlyConfigured(f"Module '{module}' does not define a '{attr}'") return attr diff --git a/geonode/decorators.py b/geonode/decorators.py index ce3d18275e0..14b0ef7b127 100644 --- a/geonode/decorators.py +++ b/geonode/decorators.py @@ -94,7 +94,7 @@ def view_or_basicauth(view, request, test_func, realm="", *args, **kwargs): # response = HttpResponse() response.status_code = 401 - response['WWW-Authenticate'] = 'Basic realm="%s"' % realm + response['WWW-Authenticate'] = f'Basic realm="{realm}"' return response @@ -109,8 +109,7 @@ def decorator(cls): raise TypeError( "You should only decorate subclasses of View, not mixins.") if subclass: - cls = type("%sWithDecorator(%s)" % - (cls.__name__, fdec.__name__), (cls,), {}) + cls = type(f"{cls.__name__}WithDecorator({fdec.__name__})", (cls,), {}) original = cls.as_view.__func__ @wraps(original) @@ -316,6 +315,6 @@ def wrapper(request, *args, **kwargs): def dump_func_name(func): def echo_func(*func_args, **func_kwargs): - logger.debug('Start func: {}'.format(func.__name__)) + logger.debug(f'Start func: {func.__name__}') return func(*func_args, **func_kwargs) return echo_func diff --git a/geonode/documents/forms.py b/geonode/documents/forms.py index b2b87ff24c5..ac2fce80d7c 100644 --- a/geonode/documents/forms.py +++ b/geonode/documents/forms.py @@ -55,8 +55,8 @@ def generate_link_choices(self, resources=None): for obj in resources: type_id = ContentType.objects.get_for_model(obj.__class__).id choices.append([ - "type:%s-id:%s" % (type_id, obj.id), - '%s (%s)' % (obj.title, obj.polymorphic_ctype.model) + f"type:{type_id}-id:{obj.id}", + f"{obj.title} ({obj.polymorphic_ctype.model})" ]) return choices diff --git a/geonode/documents/models.py b/geonode/documents/models.py index 5c930695039..e29e1d7817e 100644 --- a/geonode/documents/models.py +++ b/geonode/documents/models.py @@ -70,7 +70,7 @@ class Document(ResourceBase): verbose_name=_('URL')) def __str__(self): - return "{0}".format(self.title) + return str(self.title) def get_absolute_url(self): return reverse('document_detail', args=(self.id,)) @@ -87,7 +87,7 @@ def name_long(self): if not self.title: return str(self.id) else: - return '%s (%s)' % (self.title, self.id) + return f'{self.title} ({self.id})' def find_placeholder(self): placeholder = 'documents/{0}-placeholder.png' @@ -236,9 +236,7 @@ def post_save_document(instance, *args, **kwargs): if instance.id and instance.doc_file: name = "Hosted Document" site_url = settings.SITEURL.rstrip('/') if settings.SITEURL.startswith('http') else settings.SITEURL - url = '%s%s' % ( - site_url, - reverse('document_download', args=(instance.id,))) + url = f"{site_url}{reverse('document_download', args=(instance.id,))}" create_document_thumbnail.apply_async((instance.id,)) elif instance.doc_url: name = "External Document" diff --git a/geonode/documents/renderers.py b/geonode/documents/renderers.py index 8de11944773..5f75fb9ed67 100644 --- a/geonode/documents/renderers.py +++ b/geonode/documents/renderers.py @@ -67,7 +67,7 @@ def render_document(document_path, extension="png"): output_path = None if settings.UNOCONV_ENABLE: timeout = None - _, output_path = tempfile.mkstemp(suffix=".{}".format(extension)) + _, output_path = tempfile.mkstemp(suffix=f".{extension}") try: unoconv = subprocess.Popen( [settings.UNOCONV_EXECUTABLE, "-v", "-e", "PageRange=1-2", diff --git a/geonode/documents/tasks.py b/geonode/documents/tasks.py index ba41e285d83..49afcec1026 100644 --- a/geonode/documents/tasks.py +++ b/geonode/documents/tasks.py @@ -48,12 +48,12 @@ def create_document_thumbnail(self, object_id): """ Create thumbnail for a document. """ - logger.debug("Generating thumbnail for document #{}.".format(object_id)) + logger.debug(f"Generating thumbnail for document #{object_id}.") try: document = Document.objects.get(id=object_id) except Document.DoesNotExist: - logger.error("Document #{} does not exist.".format(object_id)) + logger.error(f"Document #{object_id} does not exist.") raise image_path = None @@ -82,24 +82,23 @@ def create_document_thumbnail(self, object_id): try: image_file = open(image_path, 'rb') except Exception as e: - logger.debug(e) - logger.debug("Failed to render document #{}".format(object_id)) + logger.debug(f"Failed to render document #{object_id}: {e}") else: - logger.debug("Failed to render document #{}".format(object_id)) + logger.debug(f"Failed to render document #{object_id}") except ConversionError as e: - logger.debug("Could not convert document #{}: {}.".format(object_id, e)) + logger.debug(f"Could not convert document #{object_id}: {e}.") except NotImplementedError as e: - logger.debug("Failed to render document #{}: {}".format(object_id, e)) + logger.debug(f"Failed to render document #{object_id}: {e}") thumbnail_content = None try: try: thumbnail_content = generate_thumbnail_content(image_file) except Exception as e: - logger.error("Could not generate thumbnail, falling back to 'placeholder': {}".format(e)) + logger.error(f"Could not generate thumbnail, falling back to 'placeholder': {e}") thumbnail_content = generate_thumbnail_content(document.find_placeholder()) except Exception as e: - logger.error("Could not generate thumbnail: {}".format(e)) + logger.error(f"Could not generate thumbnail: {e}") return finally: if image_file is not None: @@ -109,10 +108,10 @@ def create_document_thumbnail(self, object_id): os.remove(image_path) if not thumbnail_content: - logger.warning("Thumbnail for document #{} empty.".format(object_id)) - filename = 'document-{}-thumb.png'.format(document.uuid) + logger.warning(f"Thumbnail for document #{object_id} empty.") + filename = f'document-{document.uuid}-thumb.png' document.save_thumbnail(filename, thumbnail_content) - logger.debug("Thumbnail for document #{} created.".format(object_id)) + logger.debug(f"Thumbnail for document #{object_id} created.") @app.task( diff --git a/geonode/documents/tests.py b/geonode/documents/tests.py index 085dbfc1d08..4616a30f37c 100644 --- a/geonode/documents/tests.py +++ b/geonode/documents/tests.py @@ -458,8 +458,7 @@ def setUp(self): self.passwd = 'admin' create_models(type=b'document') create_models(type=b'map') - self.document_upload_url = "{}?no__redirect=true".format( - reverse('document_upload')) + self.document_upload_url = f"{(reverse('document_upload'))}?no__redirect=true" self.u = get_user_model().objects.get(username=self.user) self.u.email = 'test@email.com' self.u.is_active = True @@ -473,7 +472,7 @@ def test_document_upload_redirect(self): with self.settings(ADMIN_MODERATE_UPLOADS=False): self.client.login(username=self.user, password=self.passwd) input_path = self._get_input_path() - document_upload_url = "{}".format(reverse('document_upload')) + document_upload_url = str(reverse('document_upload')) with open(input_path, 'rb') as f: data = {'title': 'document title', 'doc_file': f, diff --git a/geonode/documents/utils.py b/geonode/documents/utils.py index 291468ea1af..3082d3b9214 100644 --- a/geonode/documents/utils.py +++ b/geonode/documents/utils.py @@ -43,13 +43,13 @@ def delete_orphaned_document_files(): for filename in files: if Document.objects.filter(doc_file__contains=filename).count() == 0: - logger.debug("Deleting orphaned document " + filename) + logger.debug(f"Deleting orphaned document {filename}") try: storage.delete(os.path.join( os.path.join("documents", "document"), filename)) deleted.append(filename) except NotImplementedError as e: logger.error( - "Failed to delete orphaned document '{}': {}".format(filename, e)) + f"Failed to delete orphaned document '{filename}': {e}") return deleted diff --git a/geonode/favorite/models.py b/geonode/favorite/models.py index fd8573c75d9..88be3df5b94 100644 --- a/geonode/favorite/models.py +++ b/geonode/favorite/models.py @@ -105,7 +105,6 @@ class Meta: def __str__(self): if self.content_object: - return "Favorite: {}, {}, {}".format( - self.content_object.title, self.content_type, self.user) + return f"Favorite: {self.content_object.title}, {self.content_type}, {self.user}" else: return "Unknown" diff --git a/geonode/favorite/utils.py b/geonode/favorite/utils.py index 3d0946e827d..e09f952d778 100644 --- a/geonode/favorite/utils.py +++ b/geonode/favorite/utils.py @@ -32,7 +32,7 @@ def get_favorite_info(user, content_object): result = {} url_content_type = type(content_object).__name__.lower() - result["add_url"] = reverse("add_favorite_{}".format(url_content_type), args=[content_object.pk]) + result["add_url"] = reverse(f"add_favorite_{url_content_type}", args=[content_object.pk]) existing_favorite = models.Favorite.objects.favorite_for_user_and_content_object(user, content_object) diff --git a/geonode/geoapps/api/serializers.py b/geonode/geoapps/api/serializers.py index f8ab2be20c7..e9e1143134f 100644 --- a/geonode/geoapps/api/serializers.py +++ b/geonode/geoapps/api/serializers.py @@ -102,7 +102,7 @@ def extra_update_checks(self, validated_data): if _u: validated_data[_key] = _u else: - raise ValidationError("The specified '{}' does not exist!".format(_key)) + raise ValidationError(f"The specified '{_key}' does not exist!") def extra_create_checks(self, validated_data): if 'name' not in validated_data or \ diff --git a/geonode/geoapps/models.py b/geonode/geoapps/models.py index 248160c6e9e..19dea48ff72 100644 --- a/geonode/geoapps/models.py +++ b/geonode/geoapps/models.py @@ -81,8 +81,7 @@ class GeoApp(ResourceBase): on_delete=models.CASCADE) def __str__(self): - return '%s by %s' % ( - self.title, (self.owner.username if self.owner else "")) + return f'{self.title} by {(self.owner.username if self.owner else "")}' @property def class_name(self): diff --git a/geonode/geoserver/__init__.py b/geonode/geoserver/__init__.py index 8a8c4d0bc24..a5a4b3a59da 100644 --- a/geonode/geoserver/__init__.py +++ b/geonode/geoserver/__init__.py @@ -59,14 +59,14 @@ def set_resource_links(*args, **kwargs): _all_layers = Layer.objects.all() for index, layer in enumerate(_all_layers, start=1): _lyr_name = layer.name - message = "[%s / %s] Updating Layer [%s] ..." % (index, len(_all_layers), _lyr_name) + message = f"[{index} / {len(_all_layers)}] Updating Layer [{_lyr_name}] ..." logger.debug(message) try: set_resource_default_links(layer, layer) catalogue_post_save(instance=layer, sender=layer.__class__) except Exception: logger.exception( - "[ERROR] Layer [%s] couldn't be updated" % _lyr_name + f"[ERROR] Layer [{_lyr_name}] couldn't be updated" ) diff --git a/geonode/geoserver/createlayer/tests.py b/geonode/geoserver/createlayer/tests.py index 875e59164e7..281c49bf0a8 100644 --- a/geonode/geoserver/createlayer/tests.py +++ b/geonode/geoserver/createlayer/tests.py @@ -122,7 +122,7 @@ def test_layer_creation(self): self.assertEqual(resource.projection, layer.srid) # check if layer detail page is accessible with client - response = self.client.get(reverse('layer_detail', args=('geonode:%s' % layer_name,))) + response = self.client.get(reverse('layer_detail', args=(f'geonode:{layer_name}',))) self.assertEqual(response.status_code, 200) def test_layer_creation_with_wrong_geometry_type(self): diff --git a/geonode/geoserver/createlayer/utils.py b/geonode/geoserver/createlayer/utils.py index 4c764ba8e6d..70003e685fa 100644 --- a/geonode/geoserver/createlayer/utils.py +++ b/geonode/geoserver/createlayer/utils.py @@ -70,7 +70,7 @@ def create_gn_layer(workspace, datastore, name, title, owner_name): workspace=workspace.name, store=datastore.name, storeType='dataStore', - alternate='%s:%s' % (workspace.name, name), + alternate=f'{workspace.name}:{name}', title=title, owner=owner, uuid=str(uuid.uuid4()), @@ -116,7 +116,7 @@ def get_attributes(geometry_type, json_attrs=None): lattrs = [] gattr = [] gattr.append('the_geom') - gattr.append('com.vividsolutions.jts.geom.%s' % geometry_type) + gattr.append(f'com.vividsolutions.jts.geom.{geometry_type}') gattr.append({'nillable': False}) lattrs.append(gattr) if json_attrs: @@ -126,17 +126,17 @@ def get_attributes(geometry_type, json_attrs=None): attr_name = slugify(jattr[0]) attr_type = jattr[1].lower() if len(attr_name) == 0: - msg = 'You must provide an attribute name for attribute of type %s' % (attr_type) + msg = f'You must provide an attribute name for attribute of type {attr_type}' logger.error(msg) raise GeoNodeException(msg) if attr_type not in ('float', 'date', 'string', 'integer'): - msg = '%s is not a valid type for attribute %s' % (attr_type, attr_name) + msg = f'{attr_type} is not a valid type for attribute {attr_name}' logger.error(msg) raise GeoNodeException(msg) if attr_type == 'date': - attr_type = 'java.util.%s' % attr_type[:1].upper() + attr_type[1:] + attr_type = f'java.util.{(attr_type[:1].upper() + attr_type[1:])}' else: - attr_type = 'java.lang.%s' % attr_type[:1].upper() + attr_type[1:] + attr_type = f'java.lang.{(attr_type[:1].upper() + attr_type[1:])}' lattr.append(attr_name) lattr.append(attr_type) lattr.append({'nillable': True}) @@ -178,7 +178,7 @@ def create_gs_layer(name, title, geometry_type, attributes=None): resources = datastore.get_resources() for resource in resources: if resource.name == name: - msg = "There is already a layer named %s in %s" % (name, workspace) + msg = f"There is already a layer named {name} in {workspace}" logger.error(msg) raise GeoNodeException(msg) @@ -187,36 +187,35 @@ def create_gs_layer(name, title, geometry_type, attributes=None): for spec in attributes: att_name, binding, opts = spec nillable = opts.get("nillable", False) - attributes_block += ("" - "{name}" - "{binding}" - "{nillable}" - "").format(name=att_name, binding=binding, nillable=nillable) + attributes_block += ( + "" + f"{att_name}" + f"{binding}" + f"{nillable}" + "") attributes_block += "" # TODO implement others srs and not only EPSG:4326 - xml = ("" - "{name}" - "{native_name}" - "{title}" - "EPSG:4326" - "{minx}{maxx}{miny}{maxy}" - "EPSG:4326" - "{attributes}" - "").format( - name=name, native_name=native_name, - title=title, - minx=BBOX[0], maxx=BBOX[1], miny=BBOX[2], maxy=BBOX[3], - attributes=attributes_block) - - url = ('%s/workspaces/%s/datastores/%s/featuretypes' - % (ogc_server_settings.rest, workspace.name, datastore.name)) + xml = ( + "" + f"{name}" + f"{native_name}" + f"{title}" + "EPSG:4326" + f"{BBOX[0]}{BBOX[1]}{BBOX[2]}{BBOX[3]}" + f"EPSG:4326" + f"{attributes_block}" + "") + + url = ( + f'{ogc_server_settings.rest}/workspaces/{workspace.name}/datastores/{datastore.name}/featuretypes' + ) headers = {'Content-Type': 'application/xml'} _user, _password = ogc_server_settings.credentials req = requests.post(url, data=xml, headers=headers, auth=(_user, _password)) if req.status_code != 201: - logger.error('Request status code was: %s' % req.status_code) - logger.error('Response was: %s' % req.text) - raise Exception("Layer could not be created in GeoServer {}".format(req.text)) + logger.error(f'Request status code was: {req.status_code}') + logger.error(f'Response was: {req.text}') + raise Exception(f"Layer could not be created in GeoServer {req.text}") return workspace, datastore diff --git a/geonode/geoserver/createlayer/views.py b/geonode/geoserver/createlayer/views.py index b806bf0450e..622ebbdeee6 100644 --- a/geonode/geoserver/createlayer/views.py +++ b/geonode/geoserver/createlayer/views.py @@ -49,7 +49,7 @@ def layer_create(request, template='createlayer/layer_create.html'): layer.set_permissions(json.loads(permissions), created=True) return redirect(layer) except Exception as e: - error = '%s (%s)' % (e, type(e)) + error = f'{e} ({type(e)})' else: form = NewLayerForm() diff --git a/geonode/geoserver/helpers.py b/geonode/geoserver/helpers.py index ec71bd71b0f..61d81af45c1 100755 --- a/geonode/geoserver/helpers.py +++ b/geonode/geoserver/helpers.py @@ -83,7 +83,7 @@ def check_geoserver_is_up(): """Verifies all geoserver is running, this is needed to be able to upload. """ - url = "%s" % ogc_server_settings.LOCATION + url = f"{ogc_server_settings.LOCATION}" req, content = http_client.get(url, user=_user) msg = ('Cannot connect to the GeoServer at %s\nPlease make sure you ' 'have started it.' % url) @@ -422,10 +422,8 @@ def cascading_delete(layer_name=None, catalog=None): resource = cat.get_resource(name=layer_name) except EnvironmentError as e: if e.errno == errno.ECONNREFUSED: - msg = ('Could not connect to geoserver at "%s"' - 'to save information for layer "%s"' % ( - ogc_server_settings.LOCATION, layer_name) - ) + msg = (f'Could not connect to geoserver at "{ogc_server_settings.LOCATION}"' + f'to save information for layer "{layer_name}"') logger.error(msg) return None else: @@ -461,7 +459,7 @@ def cascading_delete(layer_name=None, catalog=None): for s in styles: if s is not None and s.name not in _default_style_names: try: - logger.debug("Trying to delete Style [%s]" % s.name) + logger.debug(f"Trying to delete Style [{s.name}]") cat.delete(s, purge='true') except Exception as e: # Trying to delete a shared style will fail @@ -519,7 +517,7 @@ def delete_from_postgis(layer_name, store): try: conn = psycopg2.connect(dbname=db_name, user=user, host=host, port=port, password=password) cur = conn.cursor() - cur.execute("SELECT DropGeometryTable ('%s')" % layer_name) + cur.execute(f"SELECT DropGeometryTable ('{layer_name}')") conn.commit() except Exception as e: logger.error( @@ -629,7 +627,7 @@ def gs_slurp( if skip_geonode_registered: try: resources = [k for k in resources - if not '%s:%s' % (k.workspace.name, k.name) in layer_names] + if f'{k.workspace.name}:{k.name}' not in layer_names] except Exception: if ignore_errors: pass @@ -668,9 +666,9 @@ def gs_slurp( workspace=workspace.name, store=the_store.name, storeType=the_store.resource_type, - alternate="%s:%s" % (workspace.name, resource.name), - title=resource.title or 'No title provided', - abstract=resource.abstract or "{}".format(_('No abstract provided')), + alternate=f"{workspace.name}:{resource.name}", + title=resource.title or _('No title provided'), + abstract=resource.abstract or _('No abstract provided'), owner=owner, uuid=str(uuid.uuid4()) ) @@ -709,8 +707,7 @@ def gs_slurp( if verbosity > 0: msg = "Stopping process because --ignore-errors was not set and an error was found." print(msg, file=sys.stderr) - - raise Exception("Failed to process {}".format(resource.name)) from e + raise Exception(f"Failed to process {resource.name}") from e else: if created: @@ -725,7 +722,7 @@ def gs_slurp( status = 'updated' output['stats']['updated'] += 1 - msg = "[%s] Layer %s (%d/%d)" % (status, name, i + 1, number) + msg = f"[{status}] Layer {name} ({(i + 1)}/{number})" info = {'name': name, 'status': status} if status == 'failed': output['stats']['failed'] += 1 @@ -817,10 +814,7 @@ def gs_slurp( from .signals import geoserver_pre_delete pre_delete.connect(geoserver_pre_delete, sender=Layer) - msg = "[%s] Layer %s (%d/%d)" % (status, - layer.name, - i + 1, - number_deleted) + msg = f"[{status}] Layer {layer.name} ({(i + 1)}/{number_deleted})" info = {'name': layer.name, 'status': status} if status == "delete_failed": exception_type, error, traceback = sys.exc_info() @@ -946,7 +940,7 @@ def set_attributes_from_geoserver(layer, overwrite=False): attribute_map = [] server_url = ogc_server_settings.LOCATION if layer.storeType != "remoteStore" else layer.remote_service.service_url if layer.storeType == "remoteStore" and layer.remote_service.ptype == "gxp_arcrestsource": - dft_url = server_url + ("%s?f=json" % (layer.alternate or layer.typename)) + dft_url = f"{server_url}{(layer.alternate or layer.typename)}?f=json" try: # The code below will fail if http_client cannot be imported req, body = http_client.get(dft_url, user=_user) @@ -970,8 +964,8 @@ def set_attributes_from_geoserver(layer, overwrite=False): # The code below will fail if http_client cannot be imported or WFS not supported req, body = http_client.get(dft_url, user=_user) doc = dlxml.fromstring(body.encode()) - path = ".//{xsd}extension/{xsd}sequence/{xsd}element".format( - xsd="{http://www.w3.org/2001/XMLSchema}") + xsd = "{http://www.w3.org/2001/XMLSchema}" + path = f".//{xsd}extension/{xsd}sequence/{xsd}element" attribute_map = [[n.attrib["name"], n.attrib["type"]] for n in doc.findall( path) if n.attrib.get("name") and n.attrib.get("type")] except Exception: @@ -1018,8 +1012,8 @@ def set_attributes_from_geoserver(layer, overwrite=False): try: req, body = http_client.get(dc_url, user=_user) doc = dlxml.fromstring(body.encode()) - path = ".//{wcs}Axis/{wcs}AvailableKeys/{wcs}Key".format( - wcs="{http://www.opengis.net/wcs/1.1.1}") + wcs = "{http://www.opengis.net/wcs/1.1.1}" + path = f".//{wcs}Axis/{wcs}AvailableKeys/{wcs}Key" attribute_map = [[n.text, "raster"] for n in doc.findall(path)] except Exception: tb = traceback.format_exc() @@ -1363,7 +1357,7 @@ def create_geoserver_db_featurestore( ds_exists = True except FailedRequestError: logger.debug( - 'Creating target datastore %s' % dsname) + f'Creating target datastore {dsname}') ds = cat.create_datastore(dsname, workspace=workspace) db = ogc_server_settings.datastore_db db_engine = 'postgis' if \ @@ -1447,12 +1441,12 @@ def _create_db_featurestore(name, data, overwrite=False, charset="UTF-8", worksp return ds, resource except Exception: msg = _("An exception occurred loading data to PostGIS") - msg += "- %s" % (sys.exc_info()[1]) + msg += f"- {sys.exc_info()[1]}" try: delete_from_postgis(name, ds) except Exception: msg += _(" Additionally an error occured during database cleanup") - msg += "- %s" % (sys.exc_info()[1]) + msg += f"- {sys.exc_info()[1]}" raise GeoNodeException(msg) @@ -1467,15 +1461,15 @@ def get_store(cat, name, workspace=None): if workspace: try: - store = cat.get_xml('%s/%s.xml' % (workspace.datastore_url[:-4], name)) + store = cat.get_xml(f'{workspace.datastore_url[:-4]}/{name}.xml') except FailedRequestError: try: - store = cat.get_xml('%s/%s.xml' % (workspace.coveragestore_url[:-4], name)) + store = cat.get_xml(f'{workspace.coveragestore_url[:-4]}/{name}.xml') except FailedRequestError: try: - store = cat.get_xml('%s/%s.xml' % (workspace.wmsstore_url[:-4], name)) + store = cat.get_xml(f'{workspace.wmsstore_url[:-4]}/{name}.xml') except FailedRequestError: - raise FailedRequestError("No store found named: " + name) + raise FailedRequestError(f"No store found named: {name}") if store: if store.tag == 'dataStore': store = datastore_from_index(cat, workspace, store) @@ -1485,9 +1479,9 @@ def get_store(cat, name, workspace=None): store = wmsstore_from_index(cat, workspace, store) return store else: - raise FailedRequestError("No store found named: " + name) + raise FailedRequestError(f"No store found named: {name}") else: - raise FailedRequestError("No store found named: " + name) + raise FailedRequestError(f"No store found named: {name}") class ServerDoesNotExist(Exception): @@ -1565,7 +1559,7 @@ def netloc(self): return urlsplit(self.LOCATION).netloc def __str__(self): - return "{0}".format(self.alias) + return str(self.alias) class OGC_Servers_Handler(object): @@ -1587,7 +1581,7 @@ def ensure_valid_configuration(self, alias): try: server = self.servers[alias] except KeyError: - raise ServerDoesNotExist("The server %s doesn't exist" % alias) + raise ServerDoesNotExist(f"The server {alias} doesn't exist") if 'PRINTNG_ENABLED' in server: raise ImproperlyConfigured("The PRINTNG_ENABLED setting has been removed, use 'PRINT_NG_ENABLED' instead.") @@ -1599,7 +1593,7 @@ def ensure_defaults(self, alias): try: server = self.servers[alias] except KeyError: - raise ServerDoesNotExist("The server %s doesn't exist" % alias) + raise ServerDoesNotExist(f"The server {alias} doesn't exist") server.setdefault('BACKEND', 'geonode.geoserver') server.setdefault('LOCATION', 'http://localhost:8080/geoserver/') @@ -1667,7 +1661,7 @@ def fetch_gs_resource(instance, values, tries): abstract=gs_resource.abstract or '', owner=instance.owner)) else: - msg = "There isn't a geoserver resource for this layer: %s" % instance.name + msg = f"There isn't a geoserver resource for this layer: {instance.name}" logger.exception(msg) if tries >= _max_tries: # raise GeoNodeException(msg) @@ -1741,7 +1735,7 @@ def _stylefilterparams_geowebcache_layer(layer_name): headers = { "Content-Type": "text/xml" } - url = '%sgwc/rest/layers/%s.xml' % (ogc_server_settings.LOCATION, layer_name) + url = f'{ogc_server_settings.LOCATION}gwc/rest/layers/{layer_name}.xml' # read GWC configuration req, content = http_client.get( @@ -1749,10 +1743,9 @@ def _stylefilterparams_geowebcache_layer(layer_name): headers=headers, user=_user) if req.status_code != 200: - line = "Error {0} reading Style Filter Params GeoWebCache at {1}".format( - req.status_code, url + logger.error( + f"Error {req.status_code} reading Style Filter Params GeoWebCache at {url}" ) - logger.error(line) return # check/write GWC filter parameters @@ -1773,10 +1766,9 @@ def _stylefilterparams_geowebcache_layer(layer_name): headers=headers, user=_user) if req.status_code != 200: - line = "Error {0} writing Style Filter Params GeoWebCache at {1}".format( - req.status_code, url + logger.error( + f"Error {req.status_code} writing Style Filter Params GeoWebCache at {url}" ) - logger.error(line) def _invalidate_geowebcache_layer(layer_name, url=None): @@ -1784,11 +1776,11 @@ def _invalidate_geowebcache_layer(layer_name, url=None): headers = { "Content-Type": "text/xml", } - body = """ - {0} - """.strip().format(layer_name) + body = f""" + {layer_name} + """.strip() if not url: - url = '%sgwc/rest/masstruncate' % ogc_server_settings.LOCATION + url = f'{ogc_server_settings.LOCATION}gwc/rest/masstruncate' req, content = http_client.post( url, data=body, @@ -1796,10 +1788,9 @@ def _invalidate_geowebcache_layer(layer_name, url=None): user=_user) if req.status_code != 200: - line = "Error {0} invalidating GeoWebCache at {1}".format( - req.status_code, url + logger.debug( + f"Error {req.status_code} invalidating GeoWebCache at {url}" ) - logger.debug(line) def style_update(request, url): @@ -1844,7 +1835,7 @@ def style_update(request, url): elm_user_style_title = elm_user_style_name.text layer_name = elm_namedlayer_name.text style_name = elm_user_style_name.text - sld_body = '%s' % request.body + sld_body = f'{request.body}' except Exception: logger.warn("Could not recognize Style and Layer name from Request!") # add style in GN and associate it to layer @@ -1910,7 +1901,7 @@ def set_time_info(layer, attribute, end_attribute, presentation, ''' layer = gs_catalog.get_layer(layer.name) if layer is None: - raise ValueError('no such layer: %s' % layer.name) + raise ValueError(f'no such layer: {layer.name}') resource = layer.resource if layer else None if not resource: resources = gs_catalog.get_resources(stores=[layer.name]) @@ -1919,7 +1910,7 @@ def set_time_info(layer, attribute, end_attribute, presentation, resolution = None if precision_value and precision_step: - resolution = '%s %s' % (precision_value, precision_step) + resolution = f'{precision_value} {precision_step}' info = DimensionInfo("time", enabled, presentation, resolution, "ISO8601", None, attribute=attribute, end_attribute=end_attribute) if resource and resource.metadata: @@ -1943,7 +1934,7 @@ def get_time_info(layer): ''' layer = gs_catalog.get_layer(layer.name) if layer is None: - raise ValueError('no such layer: %s' % layer.name) + raise ValueError(f'no such layer: {layer.name}') resource = layer.resource if layer else None if not resource: resources = gs_catalog.get_resources(stores=[layer.name]) @@ -2085,8 +2076,8 @@ def set_time_dimension(cat, name, workspace, time_presentation, time_presentatio resource = resources[0] if not resource: - logger.exception("No resource could be found on GeoServer with name %s" % name) - raise Exception("No resource could be found on GeoServer with name %s" % name) + logger.exception(f"No resource could be found on GeoServer with name {name}") + raise Exception(f"No resource could be found on GeoServer with name {name}") resource.metadata = {'time': timeInfo} cat.save(resource) diff --git a/geonode/geoserver/management/commands/find_geoserver_broken_layers.py b/geonode/geoserver/management/commands/find_geoserver_broken_layers.py index 3341ff1e92c..ab248278ed8 100644 --- a/geonode/geoserver/management/commands/find_geoserver_broken_layers.py +++ b/geonode/geoserver/management/commands/find_geoserver_broken_layers.py @@ -77,14 +77,11 @@ def handle(self, **options): for layer in layers: count += 1 try: - print("Checking layer {}/{}: {} owned by {}".format( - count, - layers_count, - layer.alternate, - layer.owner.username - )) + print( + f"Checking layer {count}/{layers_count}: {layer.alternate} owned by {layer.owner.username}" + ) if not is_gs_resource_valid(layer): - print("Layer {} is broken!".format(layer.alternate)) + print(f"Layer {layer.alternate} is broken!") layer_errors.append(layer) if options['remove']: print("Removing this layer...") @@ -92,9 +89,6 @@ def handle(self, **options): except Exception: print("Unexpected error:", sys.exc_info()[0]) - print("\n***** Layers with errors: {} in a total of {} *****".format( - len(layer_errors), - layers_count - )) + print(f"\n***** Layers with errors: {len(layer_errors)} in a total of {layers_count} *****") for layer_error in layer_errors: - print("{} by {}".format(layer_error.alternate, layer_error.owner.username)) + print(f"{layer_error.alternate} by {layer_error.owner.username}") diff --git a/geonode/geoserver/management/commands/sync_geonode_layers.py b/geonode/geoserver/management/commands/sync_geonode_layers.py index 00f40ef814a..fab977792af 100644 --- a/geonode/geoserver/management/commands/sync_geonode_layers.py +++ b/geonode/geoserver/management/commands/sync_geonode_layers.py @@ -17,8 +17,6 @@ # along with this program. If not, see . # ######################################################################### - -import ast import sys import json @@ -51,7 +49,7 @@ def sync_geonode_layers(ignore_errors, for layer in layers: try: count += 1 - print("Syncing layer {}/{}: {}".format(count, layers_count, layer.name)) + print(f"Syncing layer {count}/{layers_count}: {layer.name}") if updatepermissions: print("Syncing permissions...") # sync permissions in GeoFence @@ -79,7 +77,7 @@ def sync_geonode_layers(ignore_errors, traceback.print_exc() print("Stopping process because --ignore-errors was not set and an error was found.") return - print("There are {} layers which could not be updated because of errors".format(len(layer_errors))) + print(f"There are {len(layer_errors)} layers which could not be updated because of errors") for layer_error in layer_errors: print(layer_error) diff --git a/geonode/geoserver/management/commands/sync_geonode_maps.py b/geonode/geoserver/management/commands/sync_geonode_maps.py index deb71d67278..31fc5ca01c4 100644 --- a/geonode/geoserver/management/commands/sync_geonode_maps.py +++ b/geonode/geoserver/management/commands/sync_geonode_maps.py @@ -45,7 +45,7 @@ def sync_geonode_maps(ignore_errors, for map in maps: try: count += 1 - print("Syncing map %s/%s: %s" % (count, maps_count, map.title)) + print(f"Syncing map {count}/{maps_count}: {map.title}") if updatethumbnails: print("Regenerating thumbnails...") create_gs_thumbnail(map, overwrite=True, check_bbox=False) @@ -64,9 +64,9 @@ def sync_geonode_maps(ignore_errors, traceback.print_exc() print("Stopping process because --ignore-errors was not set and an error was found.") return - print("There are {} maps which could not be updated because of errors".format(len(map_errors))) + print(f"There are {len(map_errors)} maps which could not be updated because of errors") for map_error in map_errors: - print(map_error) + print(map_error) class Command(BaseCommand): diff --git a/geonode/geoserver/management/commands/updatelayers.py b/geonode/geoserver/management/commands/updatelayers.py index e1584197b7d..991f85cd7ff 100644 --- a/geonode/geoserver/management/commands/updatelayers.py +++ b/geonode/geoserver/management/commands/updatelayers.py @@ -140,17 +140,19 @@ def handle(self, **options): dict_['traceback']) if verbosity > 0: - print("\n\nFinished processing {} layers in {} seconds.\n".format( - len(output['layers']), round(output['stats']['duration_sec'], 2))) - print("{} Created layers".format(output['stats']['created'])) - print("{} Updated layers".format(output['stats']['updated'])) - print("{} Failed layers".format(output['stats']['failed'])) + print( + f"\n\nFinished processing {len(output['layers'])} " + f"layers in {output['stats']['duration_sec']:.2f} seconds.\n" + ) + print(f"{output['stats']['created']} Created layers") + print(f"{output['stats']['updated']} Updated layers") + print(f"{output['stats']['failed']} Failed layers") try: duration_layer = round( output['stats']['duration_sec'] * 1.0 / len(output['layers']), 2) except ZeroDivisionError: duration_layer = 0 if len(output) > 0: - print("{} seconds per layer".format(duration_layer)) + print(f"{duration_layer} seconds per layer") if remove_deleted: - print("\n{} Deleted layers".format(output['stats']['deleted'])) + print(f"\n{output['stats']['deleted']} Deleted layers") diff --git a/geonode/geoserver/signals.py b/geonode/geoserver/signals.py index c6ac1b0917b..844f7c588a9 100644 --- a/geonode/geoserver/signals.py +++ b/geonode/geoserver/signals.py @@ -119,7 +119,7 @@ def geoserver_pre_save_maplayer(instance, sender, **kwargs): GsLayer) except EnvironmentError as e: if e.errno == errno.ECONNREFUSED: - msg = 'Could not connect to catalog to verify if layer %s was local' % instance.name + msg = f'Could not connect to catalog to verify if layer {instance.name} was local' logger.warn(msg) else: raise e @@ -131,7 +131,7 @@ def geoserver_post_save_map(instance, sender, created, **kwargs): if not created: if not instance.thumbnail_url or \ instance.thumbnail_url == staticfiles.static(settings.MISSING_THUMBNAIL): - logger.debug("... Creating Thumbnail for Map [%s]" % (instance.title)) + logger.debug(f"... Creating Thumbnail for Map [{instance.title}]") # create_gs_thumbnail(instance, overwrite=False, check_bbox=True) geoserver_create_thumbnail.apply_async(((instance.id, False, True, ))) diff --git a/geonode/geoserver/tasks.py b/geonode/geoserver/tasks.py index 5e89e4e6adb..d7dbc92b5e9 100644 --- a/geonode/geoserver/tasks.py +++ b/geonode/geoserver/tasks.py @@ -365,7 +365,7 @@ def geoserver_finalize_upload( else: geoserver_create_style(instance.id, instance.name, sld_file, tempdir) - logger.debug('Finalizing (permissions and notifications) Layer {0}'.format(instance)) + logger.debug(f'Finalizing (permissions and notifications) Layer {instance}') instance.handle_moderated_uploads() if permissions is not None and not spec_perms_is_empty(permissions): @@ -478,7 +478,7 @@ def geoserver_post_save_layers( metadata_links.append((link.mime, link.name, link.url)) if gs_resource: - logger.debug("Found geoserver resource for this layer: %s" % instance.name) + logger.debug(f"Found geoserver resource for this layer: {instance.name}") gs_resource.metadata_links = metadata_links instance.gs_resource = gs_resource @@ -523,8 +523,8 @@ def geoserver_post_save_layers( if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True): gs_catalog.save(gs_resource) except Exception as e: - msg = ('Error while trying to save resource named %s in GeoServer, ' - 'try to use: "%s"' % (gs_resource, str(e))) + msg = (f'Error while trying to save resource named {gs_resource} in GeoServer, ' + f'try to use: "{e}"') e.args = (msg,) logger.exception(e) diff --git a/geonode/geoserver/tests/integration.py b/geonode/geoserver/tests/integration.py index d0f1efc7677..59d70db93f1 100644 --- a/geonode/geoserver/tests/integration.py +++ b/geonode/geoserver/tests/integration.py @@ -82,8 +82,8 @@ def test_set_attributes_from_geoserver(self): try: # set attributes for resource for attribute in layer.attribute_set.all(): - attribute.attribute_label = '%s_label' % attribute.attribute - attribute.description = '%s_description' % attribute.attribute + attribute.attribute_label = f'{attribute.attribute}_label' + attribute.description = f'{attribute.attribute}_description' attribute.save() # sync the attributes with GeoServer @@ -93,11 +93,11 @@ def test_set_attributes_from_geoserver(self): for attribute in layer.attribute_set.all(): self.assertEqual( attribute.attribute_label, - '%s_label' % attribute.attribute + f'{attribute.attribute}_label' ) self.assertEqual( attribute.description, - '%s_description' % attribute.attribute + f'{attribute.attribute}_description' ) links = Link.objects.filter(resource=layer.resourcebase_ptr) diff --git a/geonode/geoserver/tests/test_server.py b/geonode/geoserver/tests/test_server.py index a6ae8e5dfd9..07b67f49d48 100644 --- a/geonode/geoserver/tests/test_server.py +++ b/geonode/geoserver/tests/test_server.py @@ -1081,9 +1081,9 @@ def test_ogc_server_defaults(self): wms_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'wms') identifier = urlencode({'layers': instance.alternate}) for _link in wms_links: - logger.debug('%s --> %s' % (wms_url, _link[3])) + logger.debug(f'{wms_url} --> {_link[3]}') self.assertTrue(wms_url in _link[3]) - logger.debug('%s --> %s' % (identifier, _link[3])) + logger.debug(f'{identifier} --> {_link[3]}') self.assertTrue(identifier in _link[3]) # WFS Links @@ -1096,9 +1096,9 @@ def test_ogc_server_defaults(self): wfs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'wfs') identifier = urlencode({'typename': instance.alternate}) for _link in wfs_links: - logger.debug('%s --> %s' % (wfs_url, _link[3])) + logger.debug(f'{wfs_url} --> {_link[3]}') self.assertTrue(wfs_url in _link[3]) - logger.debug('%s --> %s' % (identifier, _link[3])) + logger.debug(f'{identifier} --> {_link[3]}') self.assertTrue(identifier in _link[3]) # WCS Links @@ -1111,9 +1111,9 @@ def test_ogc_server_defaults(self): wcs_url = urljoin(ogc_settings.PUBLIC_LOCATION, 'wcs') identifier = urlencode({'coverageid': instance.alternate.replace(':', '__', 1)}) for _link in wcs_links: - logger.debug('%s --> %s' % (wcs_url, _link[3])) + logger.debug(f'{wcs_url} --> {_link[3]}') self.assertTrue(wcs_url in _link[3]) - logger.debug('%s --> %s' % (identifier, _link[3])) + logger.debug(f'{identifier} --> {_link[3]}') self.assertTrue(identifier in _link[3]) @on_ogc_backend(geoserver.BACKEND_PACKAGE) @@ -1217,18 +1217,14 @@ def test_set_resources_links(self): ) self.assertTrue( _post_migrate_links_orig.count() > 0, - "No 'original' links has been found for the layer '{}'".format( - _lyr.alternate - ) + f"No 'original' links has been found for the layer '{_lyr.alternate}'" ) for _link_orig in _post_migrate_links_orig: self.assertIn( _link_orig.url, _lyr.csw_anytext, - "The link URL {0} is not present in the 'csw_anytext' attribute of the layer '{1}'".format( - _link_orig.url, - _lyr.alternate - ) + f"The link URL {_link_orig.url} is not present in the 'csw_anytext' \ +attribute of the layer '{_lyr.alternate}'" ) # Check catalogue catalogue = get_catalogue() @@ -1236,9 +1232,7 @@ def test_set_resources_links(self): self.assertIsNotNone(record) self.assertTrue( hasattr(record, 'links'), - "No records have been found in the catalogue for the resource '{}'".format( - _lyr.alternate - ) + f"No records have been found in the catalogue for the resource '{_lyr.alternate}'" ) # Check 'metadata' links for each record for mime, name, metadata_url in record.links['metadata']: @@ -1255,8 +1249,5 @@ def test_set_resources_links(self): _post_migrate_link_meta = None self.assertIsNotNone( _post_migrate_link_meta, - "No '{}' links have been found in the catalogue for the resource '{}'".format( - name, - _lyr.alternate - ) + f"No '{name}' links have been found in the catalogue for the resource '{_lyr.alternate}'" ) diff --git a/geonode/geoserver/upload.py b/geonode/geoserver/upload.py index 2ca96b0b860..76624a44e5f 100644 --- a/geonode/geoserver/upload.py +++ b/geonode/geoserver/upload.py @@ -87,10 +87,9 @@ def geoserver_upload( assert overwrite, msg existing_type = resource.resource_type if existing_type != the_layer_type: - msg = ('Type of uploaded file %s (%s) ' + msg = (f'Type of uploaded file {name} ({the_layer_type}) ' 'does not match type of existing ' - 'resource type ' - '%s' % (name, the_layer_type, existing_type)) + f'resource type {existing_type}') logger.debug(msg) raise GeoNodeException(msg) @@ -108,11 +107,8 @@ def geoserver_upload( logger.debug("Uploading raster layer: [%s]", base_file) create_store_and_resource = _create_coveragestore else: - msg = ('The layer type for name %s is %s. It should be ' - '%s or %s,' % (name, - the_layer_type, - FeatureType.resource_type, - Coverage.resource_type)) + msg = (f'The layer type for name {name} is {the_layer_type}. It should be ' + f'{FeatureType.resource_type} or {Coverage.resource_type},') logger.warn(msg) raise GeoNodeException(msg) @@ -132,18 +128,17 @@ def geoserver_upload( overwrite=overwrite, workspace=workspace) except UploadError as e: - msg = ('Could not save the layer %s, there was an upload ' - 'error: %s' % (name, str(e))) + msg = (f'Could not save the layer {name}, there was an upload ' + f'error: {e}') logger.warn(msg) e.args = (msg,) raise except ConflictingDataError as e: # A datastore of this name already exists - msg = ('GeoServer reported a conflict creating a store with name %s: ' - '"%s". This should never happen because a brand new name ' + msg = (f'GeoServer reported a conflict creating a store with name {name}: ' + f'"{e}". This should never happen because a brand new name ' 'should have been generated. But since it happened, ' - 'try renaming the file or deleting the store in ' - 'GeoServer.' % (name, str(e))) + 'try renaming the file or deleting the store in GeoServer.') logger.warn(msg) e.args = (msg,) raise @@ -171,7 +166,7 @@ def geoserver_upload( assert gs_resource.name == name # Step 6. Make sure our data always has a valid projection - logger.debug('>>> Step 6. Making sure [%s] has a valid projection' % name) + logger.debug(f'>>> Step 6. Making sure [{name}] has a valid projection') _native_bbox = None try: _native_bbox = gs_resource.native_bbox @@ -194,7 +189,7 @@ def geoserver_upload( logger.debug('BBOX coordinates forced to [-180, -90, 180, 90] for layer [%s].', name) # Step 7. Create the style and assign it to the created resource - logger.debug('>>> Step 7. Creating style for [%s]' % name) + logger.debug(f'>>> Step 7. Creating style for [{name}]') cat.save(gs_resource) publishing = cat.get_layer(name) or gs_resource sld = None @@ -219,13 +214,13 @@ def geoserver_upload( cat.create_style(name, sld, overwrite=overwrite, raw=True, workspace=workspace) cat.reset() except geoserver.catalog.ConflictingDataError as e: - msg = ('There was already a style named %s in GeoServer, ' - 'try to use: "%s"' % (name + "_layer", str(e))) + msg = (f'There was already a style named {name}_layer in GeoServer, ' + f'try to use: "{e}"') logger.warn(msg) e.args = (msg,) except geoserver.catalog.UploadError as e: - msg = ('Error while trying to upload style named %s in GeoServer, ' - 'try to use: "%s"' % (name + "_layer", str(e))) + msg = (f'Error while trying to upload style named {name}_layer in GeoServer, ' + f'try to use: "{e}"') e.args = (msg,) logger.exception(e) @@ -245,8 +240,8 @@ def geoserver_upload( try: cat.save(publishing) except geoserver.catalog.FailedRequestError as e: - msg = ('Error while trying to save resource named %s in GeoServer, ' - 'try to use: "%s"' % (publishing, str(e))) + msg = (f'Error while trying to save resource named {publishing} in GeoServer, ' + f'try to use: "{e}"') e.args = (msg,) logger.exception(e) diff --git a/geonode/geoserver/urls.py b/geonode/geoserver/urls.py index 30aa59400bf..45a52342cbf 100644 --- a/geonode/geoserver/urls.py +++ b/geonode/geoserver/urls.py @@ -55,16 +55,16 @@ downstream_path='pdf'), name='pdf_endpoint'), url(r'^(?P[^/]*)/(?P[^/]*)/ows', views.geoserver_proxy, - dict(proxy_path='/gs/%s' % settings.DEFAULT_WORKSPACE, downstream_path='ows')), + dict(proxy_path=f'/gs/{settings.DEFAULT_WORKSPACE}', downstream_path='ows')), url(r'^(?P[^/]*)/(?P[^/]*)/wms', views.geoserver_proxy, - dict(proxy_path='/gs/%s' % settings.DEFAULT_WORKSPACE, downstream_path='wms')), + dict(proxy_path=f'/gs/{settings.DEFAULT_WORKSPACE}', downstream_path='wms')), url(r'^(?P[^/]*)/(?P[^/]*)/wfs', views.geoserver_proxy, - dict(proxy_path='/gs/%s' % settings.DEFAULT_WORKSPACE, downstream_path='wfs')), + dict(proxy_path=f'/gs/{settings.DEFAULT_WORKSPACE}', downstream_path='wfs')), url(r'^(?P[^/]*)/(?P[^/]*)/wcs', views.geoserver_proxy, - dict(proxy_path='/gs/%s' % settings.DEFAULT_WORKSPACE, downstream_path='wcs')), + dict(proxy_path=f'/gs/{settings.DEFAULT_WORKSPACE}', downstream_path='wcs')), url(r'^updatelayers/$', views.updatelayers, name="updatelayers"), diff --git a/geonode/geoserver/views.py b/geonode/geoserver/views.py index 49f6a5a4903..6597e42ffcf 100644 --- a/geonode/geoserver/views.py +++ b/geonode/geoserver/views.py @@ -128,8 +128,7 @@ def layer_style(request, layername): old_default = layer.default_style if old_default.name == style_name: return HttpResponse( - "Default style for %s remains %s" % - (layer.name, style_name), status=200) + f"Default style for {layer.name} remains {style_name}", status=200) # This code assumes without checking # that the new default style name is included @@ -151,8 +150,7 @@ def layer_style(request, layername): pass return HttpResponse( - "Default style for %s changed to %s" % - (layer.name, style_name), status=200) + f"Default style for {layer.name} changed to {style_name}", status=200) @login_required @@ -189,7 +187,7 @@ def respond(*args, **kw): sld_name = extract_name_from_sld( gs_catalog, sld, sld_file=request.FILES['sld']) except Exception as e: - respond(errors="The uploaded SLD file is not valid XML: {}".format(e)) + respond(errors=f"The uploaded SLD file is not valid XML: {e}") name = data.get('name') or sld_name @@ -267,10 +265,8 @@ def layer_style_manage(request, layername): except (FailedRequestError, EnvironmentError): tb = traceback.format_exc() logger.debug(tb) - msg = ('Could not connect to geoserver at "%s"' - 'to manage style information for layer "%s"' % ( - ogc_server_settings.LOCATION, layer.name) - ) + msg = (f'Could not connect to geoserver at "{ogc_server_settings.LOCATION}"' + f'to manage style information for layer "{layer.name}"') logger.debug(msg) # If geoserver is not online, return an error return render( @@ -335,8 +331,7 @@ def layer_style_manage(request, layername): except (FailedRequestError, EnvironmentError, MultiValueDictKeyError): tb = traceback.format_exc() logger.debug(tb) - msg = ('Error Saving Styles for Layer "%s"' % (layer.name) - ) + msg = (f'Error Saving Styles for Layer "{layer.name}"') logger.warn(msg) return render( request, @@ -436,7 +431,7 @@ def style_change_check(request, path): except Exception: authorized = (request.method == 'POST') # The user is probably trying to create a new style logger.warn( - 'There is not a style with such a name: %s.' % style_name) + f'There is not a style with such a name: {style_name}.') return authorized @@ -485,8 +480,7 @@ def strip_prefix(path, prefix): assert _prefix in path prefix_idx = path.index(_prefix) _prefix = path[:prefix_idx] + _prefix - full_prefix = "%s/%s/%s" % ( - _prefix, layername, downstream_path) if layername else _prefix + full_prefix = f"{_prefix}/{layername}/{downstream_path}" if layername else _prefix return path[len(full_prefix):] path = strip_prefix(request.get_full_path(), proxy_path) @@ -499,11 +493,11 @@ def strip_prefix(path, prefix): if ws and ws in path: # Strip out WS from PATH try: - path = "/%s" % strip_prefix(path, "/%s:" % (ws)) + path = f'/{strip_prefix(path, f"/{ws}:")}' except Exception: pass - if proxy_path == '/gs/%s' % settings.DEFAULT_WORKSPACE and layername: + if proxy_path == f'/gs/{settings.DEFAULT_WORKSPACE}' and layername: import posixpath raw_url = urljoin(ogc_server_settings.LOCATION, posixpath.join(workspace, layername, downstream_path, path)) @@ -530,9 +524,9 @@ def strip_prefix(path, prefix): url = urlsplit(raw_url) affected_layers = None - if '%s/layers' % ws in path: + if f'{ws}/layers' in path: downstream_path = 'rest/layers' - elif '%s/styles' % ws in path: + elif f'{ws}/styles' in path: downstream_path = 'rest/styles' if request.method in ("POST", "PUT", "DELETE"): @@ -546,8 +540,7 @@ def strip_prefix(path, prefix): status=401) elif downstream_path == 'rest/styles': logger.debug( - "[geoserver_proxy] Updating Style ---> url %s" % - url.geturl()) + f"[geoserver_proxy] Updating Style ---> url {url.geturl()}") _style_name, _style_ext = os.path.splitext(os.path.basename(urlsplit(url.geturl()).path)) if _style_name == 'styles.json' and request.method == "PUT": _parsed_get_args = dict(parse_qsl(urlsplit(url.geturl()).query)) @@ -560,14 +553,13 @@ def strip_prefix(path, prefix): affected_layers = style_update(request, raw_url) elif downstream_path == 'rest/layers': logger.debug( - "[geoserver_proxy] Updating Layer ---> url %s" % - url.geturl()) + f"[geoserver_proxy] Updating Layer ---> url {url.geturl()}") try: _layer_name = os.path.splitext(os.path.basename(request.path))[0] _layer = Layer.objects.get(name__icontains=_layer_name) affected_layers = [_layer] except Exception: - logger.warn("Could not find any Layer %s on DB" % os.path.basename(request.path)) + logger.warn(f"Could not find any Layer {os.path.basename(request.path)} on DB") kwargs = {'affected_layers': affected_layers} raw_url = unquote(raw_url) @@ -736,13 +728,11 @@ def get_layer_capabilities(layer, version='1.3.0', access_token=None, tolerant=F """ workspace, layername = layer.alternate.split(":") if ":" in layer.alternate else (None, layer.alternate) if not layer.remote_service: - wms_url = '%s%s/%s/wms?service=wms&version=%s&request=GetCapabilities'\ - % (ogc_server_settings.LOCATION, workspace, layername, version) + wms_url = f'{ogc_server_settings.LOCATION}{workspace}/{layername}/wms?service=wms&version={version}&request=GetCapabilities' # noqa if access_token: - wms_url += ('&access_token=%s' % access_token) + wms_url += f'&access_token={access_token}' else: - wms_url = '%s?service=wms&version=%s&request=GetCapabilities'\ - % (layer.remote_service.service_url, version) + wms_url = f'{layer.remote_service.service_url}?service=wms&version={version}&request=GetCapabilities' _user, _password = ogc_server_settings.credentials req, content = http_client.get(wms_url, user=_user) @@ -751,10 +741,9 @@ def get_layer_capabilities(layer, version='1.3.0', access_token=None, tolerant=F if tolerant and ('ServiceException' in getcap or req.status_code == 404): # WARNING Please make sure to have enabled DJANGO CACHE as per # https://docs.djangoproject.com/en/2.0/topics/cache/#filesystem-caching - wms_url = '%s%s/ows?service=wms&version=%s&request=GetCapabilities&layers=%s'\ - % (ogc_server_settings.public_url, workspace, version, layer) + wms_url = f'{ogc_server_settings.public_url}{workspace}/ows?service=wms&version={version}&request=GetCapabilities&layers={layer}' # noqa if access_token: - wms_url += ('&access_token=%s' % access_token) + wms_url += f'&access_token={access_token}' req, content = http_client.get(wms_url, user=_user) getcap = ensure_string(content) diff --git a/geonode/groups/__init__.py b/geonode/groups/__init__.py index ad24e55f272..65e6fea3d23 100644 --- a/geonode/groups/__init__.py +++ b/geonode/groups/__init__.py @@ -31,7 +31,7 @@ def init_registered_members_groupprofile(): group_name = settings.REGISTERED_MEMBERS_GROUP_NAME group_title = settings.REGISTERED_MEMBERS_GROUP_TITLE - logger.debug("Creating %s default Group Profile" % group_name) + logger.debug(f"Creating {group_name} default Group Profile") groupprofile, created = GroupProfile.objects.get_or_create( slug=group_name) if created: diff --git a/geonode/groups/models.py b/geonode/groups/models.py index 6646a672346..7a082314e92 100644 --- a/geonode/groups/models.py +++ b/geonode/groups/models.py @@ -54,7 +54,7 @@ class Meta: verbose_name_plural = _('Group Categories') def __str__(self): - return "{0}".format(self.name) + return str(self.name) def get_absolute_url(self): return reverse('group_category_detail', args=(self.slug,)) @@ -130,7 +130,7 @@ def groups_for_user(cls, user): return [] def __str__(self): - return "{0}".format(self.title) + return str(self.title) def keyword_list(self): """ @@ -203,7 +203,7 @@ def join(self, user, **kwargs): if not _members.count(): GroupMember.objects.get_or_create(group=self, user=user, defaults=kwargs) else: - logger.warning("The invited user \"{0}\" is already a member".format(user.username)) + logger.warning(f"The invited user \"{user.username}\" is already a member") def leave(self, user, **kwargs): if not user or user.is_anonymous or user == user.get_anonymous(): @@ -215,7 +215,7 @@ def leave(self, user, **kwargs): user.groups.remove(self.group) member.delete() else: - logger.warning("The invited user \"{0}\" is not a member".format(user.username)) + logger.warning(f"The invited user \"{user.username}\" is not a member") def get_absolute_url(self): return reverse('group_detail', args=[self.slug, ]) diff --git a/geonode/groups/templatetags/groups_tags.py b/geonode/groups/templatetags/groups_tags.py index 8397fc56f1a..74cf835f653 100644 --- a/geonode/groups/templatetags/groups_tags.py +++ b/geonode/groups/templatetags/groups_tags.py @@ -33,16 +33,14 @@ def group_profile_image(group_profile, css_classes="", size=None): """ if isinstance(css_classes, str): - class_attr = 'class="{}" '.format(css_classes) + class_attr = f'class="{css_classes}" ' else: try: - class_attr = 'class="{}" '.format( - " ".join(str(i) for i in css_classes)) + class_attr = f'class="{(" ".join(str(i) for i in css_classes))}" ' except Exception: class_attr = "" if size is not None: - style_attr = 'style="width: {size}px; height: {size}px" '.format( - size=size) + style_attr = f'style="width: {size}px; height: {size}px" ' else: style_attr = "" @@ -50,10 +48,5 @@ def group_profile_image(group_profile, css_classes="", size=None): url = group_profile.logo_url else: url = staticfiles_storage.url("geonode/img/default-avatar.jpg") - img_tag = '{alt}'.format( - css=class_attr, - style=style_attr, - url=url, - alt=group_profile.title, - ) + img_tag = f'{group_profile.title}' return img_tag diff --git a/geonode/layers/forms.py b/geonode/layers/forms.py index 89e1a8b223a..dcea12659fa 100644 --- a/geonode/layers/forms.py +++ b/geonode/layers/forms.py @@ -153,12 +153,10 @@ def clean(self): "uploaded a %s file" % base_ext) elif cleaned["metadata_upload_form"] and base_ext.lower() not in (".xml"): raise forms.ValidationError( - "Only XML files are supported. You uploaded a %s file" % - base_ext) + f"Only XML files are supported. You uploaded a {base_ext} file") elif cleaned["style_upload_form"] and base_ext.lower() not in (".sld"): raise forms.ValidationError( - "Only SLD files are supported. You uploaded a %s file" % - base_ext) + f"Only SLD files are supported. You uploaded a {base_ext} file") if base_ext.lower() == ".shp": if dbf_file is None or shx_file is None: @@ -183,14 +181,14 @@ def clean(self): # force rename of file so that file.shp.xml doesn't # overwrite as file.shp if cleaned.get("xml_file"): - cleaned["xml_file"].name = '%s.xml' % base_name + cleaned["xml_file"].name = f'{base_name}.xml' if sld_file is not None: if os.path.splitext(sld_file)[0] != base_name: if sld_file.find('.shp') != -1: # force rename of file so that file.shp.xml doesn't # overwrite as file.shp if cleaned.get("sld_file"): - cleaned["sld_file"].name = '%s.sld' % base_name + cleaned["sld_file"].name = f'{base_name}.sld' return cleaned def write_files(self): diff --git a/geonode/layers/management/commands/importlayers.py b/geonode/layers/management/commands/importlayers.py index 8965b237030..c4bbdf2c609 100644 --- a/geonode/layers/management/commands/importlayers.py +++ b/geonode/layers/management/commands/importlayers.py @@ -236,12 +236,10 @@ def handle(self, *args, **options): dict_['traceback']) if verbosity > 0: - print("\n\nFinished processing {} layers in {} seconds.\n".format( - len(output), duration_rounded)) - print("{} Created layers".format(len(created))) - print("{} Updated layers".format(len(updated))) - print("{} Skipped layers".format(len(skipped))) - print("{} Failed layers".format(len(failed))) - + print(f"\n\nFinished processing {len(output)} layers in {duration_rounded} seconds.\n") + print(f"{len(created)} Created layers") + print(f"{len(updated)} Updated layers") + print(f"{len(skipped)} Skipped layers") + print(f"{len(failed)} Failed layers") if len(output) > 0: - print("{} seconds per layer".format(duration * 1.0 / len(output))) + print(f"{(duration * 1.0 / len(output))} seconds per layer") diff --git a/geonode/layers/metadata.py b/geonode/layers/metadata.py index 411f0316e3e..f017da29161 100644 --- a/geonode/layers/metadata.py +++ b/geonode/layers/metadata.py @@ -45,7 +45,7 @@ def set_metadata(xml): exml = dlxml.fromstring(xml.encode()) except Exception as err: raise GeoNodeException( - 'Uploaded XML document is not XML: %s' % str(err)) + f'Uploaded XML document is not XML: {str(err)}') # check if document is an accepted XML metadata format tagname = get_tagname(exml) diff --git a/geonode/layers/models.py b/geonode/layers/models.py index 61c36ce9d5d..e3897c3d887 100644 --- a/geonode/layers/models.py +++ b/geonode/layers/models.py @@ -87,7 +87,7 @@ class Style(models.Model, PermissionLevelMixin): workspace = models.CharField(max_length=255, null=True, blank=True) def __str__(self): - return "{0}".format(self.name) + return str(self.name) def absolute_url(self): if self.sld_url: @@ -102,8 +102,7 @@ def absolute_url(self): return self.sld_url else: logger.error( - "SLD URL is empty for Style %s" % - self.name) + f"SLD URL is empty for Style {self.name}") return None def get_self_resource(self): @@ -139,15 +138,15 @@ def successful(self): return self.processed and self.errors is None def __str__(self): - _s = "[Upload session-id: {}]".format(self.id) + _s = f"[Upload session-id: {self.id}]" try: - _s += " - {}".format(self.resource.title) + _s += f" - {self.resource.title}" except Exception: pass - return "{0}".format(_s) + return str(_s) def __unicode__(self): - return "{0}".format(self.__str__()) + return str(self.__str__()) class Layer(ResourceBase): @@ -251,9 +250,7 @@ def ows_url(self): if self.remote_service is not None and self.remote_service.method == INDEXED: result = self.remote_service.service_url else: - result = "{base}ows".format( - base=settings.OGC_SERVER['default']['PUBLIC_LOCATION'], - ) + result = f"{(settings.OGC_SERVER['default']['PUBLIC_LOCATION'])}ows" return result @property @@ -263,7 +260,7 @@ def ptype(self): @property def service_typename(self): if self.remote_service is not None: - return "%s:%s" % (self.remote_service.name, self.alternate) + return f"{self.remote_service.name}:{self.alternate}" else: return self.alternate @@ -309,7 +306,7 @@ def get_base_file(self): if base_files_count == 0: return None, None - msg = 'There should only be one main file (.shp or .geotiff or .asc), found %s' % base_files_count + msg = f'There should only be one main file (.shp or .geotiff or .asc), found {base_files_count}' assert base_files_count == 1, msg # we need to check, for shapefile, if column names are valid @@ -318,7 +315,7 @@ def get_base_file(self): valid_shp, wrong_column_name, list_col = check_shp_columnnames( self) if wrong_column_name: - msg = 'Shapefile has an invalid column name: %s' % wrong_column_name + msg = f'Shapefile has an invalid column name: {wrong_column_name}' else: msg = _('File cannot be opened, maybe check the encoding') # AF: Removing assertion since if the original file does not exists anymore @@ -331,7 +328,7 @@ def get_base_file(self): def get_absolute_url(self): return reverse( 'layer_detail', - args=("%s:%s" % (self.store, self.alternate),) + args=(f"{self.store}:{self.alternate}",) ) @property @@ -355,7 +352,7 @@ def attribute_config(self): return cfg def __str__(self): - return "{0}".format(self.alternate) + return str(self.alternate) class Meta: # custom permissions, @@ -577,7 +574,7 @@ class Attribute(models.Model): objects = AttributeManager() def __str__(self): - return "{0}".format( + return str( self.attribute_label if self.attribute_label else self.attribute) def unique_values_as_list(self): @@ -588,15 +585,11 @@ def _get_alternate_name(instance): if instance.remote_service is not None and instance.remote_service.method == INDEXED: result = instance.name elif instance.remote_service is not None and instance.remote_service.method == CASCADED: - result = "{}:{}".format( - getattr(settings, "CASCADE_WORKSPACE", _DEFAULT_CASCADE_WORKSPACE), - instance.name - ) + _ws = getattr(settings, "CASCADE_WORKSPACE", _DEFAULT_CASCADE_WORKSPACE) + result = f"{_ws}:{instance.name}" else: # we are not dealing with a service-related instance - result = "{}:{}".format( - getattr(settings, "DEFAULT_WORKSPACE", _DEFAULT_WORKSPACE), - instance.name - ) + _ws = getattr(settings, "DEFAULT_WORKSPACE", _DEFAULT_WORKSPACE) + result = f"{_ws}:{instance.name}" return result @@ -632,7 +625,7 @@ def pre_save_layer(instance, sender, **kwargs): logger.debug("In pre_save_layer") if instance.alternate is None: instance.alternate = _get_alternate_name(instance) - logger.debug("instance.alternate is: {}".format(instance.alternate)) + logger.debug(f"instance.alternate is: {instance.alternate}") base_file, info = instance.get_base_file() @@ -640,7 +633,7 @@ def pre_save_layer(instance, sender, **kwargs): instance.info = info if base_file is not None: - extension = '.%s' % base_file.name + extension = f'.{base_file.name}' if extension in vec_exts: instance.storeType = 'dataStore' elif extension in cov_exts: @@ -655,7 +648,7 @@ def pre_save_layer(instance, sender, **kwargs): # Send a notification when a layer is created if instance.pk is None and instance.title: # Resource Created - notice_type_label = '%s_created' % instance.class_name.lower() + notice_type_label = f'{instance.class_name.lower()}_created' recipients = get_notification_recipients(notice_type_label, resource=instance) send_notification(recipients, notice_type_label, {'resource': instance}) diff --git a/geonode/layers/populate_layers_data.py b/geonode/layers/populate_layers_data.py index 71ee88dcd30..1aa2b551162 100644 --- a/geonode/layers/populate_layers_data.py +++ b/geonode/layers/populate_layers_data.py @@ -24,7 +24,7 @@ styles = [{"name": "test_style_1", - "sld_url": "{ogc_location}rest/styles/test_style.sld".format(ogc_location=ogc_location), + "sld_url": f"{ogc_location}rest/styles/test_style.sld", "sld_body": "", }, {"name": "test_style_2", - "sld_url": "{ogc_location}rest/styles/test_style.sld".format(ogc_location=ogc_location), + "sld_url": f"{ogc_location}rest/styles/test_style.sld", "sld_body": "", }, {"name": "test_style_3", - "sld_url": "{ogc_location}rest/styles/test_style.sld".format(ogc_location=ogc_location), + "sld_url": f"{ogc_location}rest/styles/test_style.sld", "sld_body": "", }, {"name": "Evaluación", - "sld_url": "{ogc_location}rest/styles/test_style.sld".format(ogc_location=ogc_location), + "sld_url": f"{ogc_location}rest/styles/test_style.sld", "sld_body": "\ diff --git a/geonode/layers/tasks.py b/geonode/layers/tasks.py index 318caaa2fb4..b0d561c0c68 100644 --- a/geonode/layers/tasks.py +++ b/geonode/layers/tasks.py @@ -48,7 +48,7 @@ def delete_layer(self, layer_id): except Layer.DoesNotExist: logger.warning(f"Layers {layer_id} does not exist!") return - logger.debug('Deleting Layer {0}'.format(layer)) + logger.debug(f'Deleting Layer {layer}') try: with transaction.atomic(): layer.delete() diff --git a/geonode/layers/tests.py b/geonode/layers/tests.py index 699852088b6..cf363730626 100644 --- a/geonode/layers/tests.py +++ b/geonode/layers/tests.py @@ -319,7 +319,7 @@ def test_layer_save(self): response = self.client.get(reverse('layer_detail', args=(lyr.alternate,))) self.assertEqual(response.status_code, 200) - response = self.client.get(reverse('layer_detail', args=(":%s" % lyr.alternate,))) + response = self.client.get(reverse('layer_detail', args=(f":{lyr.alternate}",))) self.assertEqual(response.status_code, 200) response = self.client.get(reverse('layer_metadata', args=(lyr.alternate,))) @@ -389,24 +389,22 @@ def test_get_valid_user(self): # Verify it accepts an admin user adminuser = get_user_model().objects.get(is_superuser=True) valid_user = get_valid_user(adminuser) - msg = ('Passed in a valid admin user "%s" but got "%s" in return' - % (adminuser, valid_user)) + msg = (f'Passed in a valid admin user "{adminuser}" but got "{valid_user}" in return') assert valid_user.id == adminuser.id, msg # Verify it returns a valid user after receiving None valid_user = get_valid_user(None) - msg = ('Expected valid user after passing None, got "%s"' % valid_user) + msg = f'Expected valid user after passing None, got "{valid_user}"' assert isinstance(valid_user, get_user_model()), msg newuser = get_user_model().objects.create(username='arieluser') valid_user = get_valid_user(newuser) - msg = ('Passed in a valid user "%s" but got "%s" in return' - % (newuser, valid_user)) + msg = (f'Passed in a valid user "{newuser}" but got "{valid_user}" in return') assert valid_user.id == newuser.id, msg valid_user = get_valid_user('arieluser') - msg = ('Passed in a valid user by username "%s" but got' - ' "%s" in return' % ('arieluser', valid_user)) + msg = ('Passed in a valid user by username "arieluser" but got' + f' "{valid_user}" in return') assert valid_user.username == 'arieluser', msg nn = get_anonymous_user() @@ -1122,7 +1120,7 @@ def _get_input_paths(self): base_name = 'single_point' suffixes = 'shp shx dbf prj'.split(' ') base_path = gisdata.GOOD_DATA - paths = [os.path.join(base_path, 'vector', '{}.{}'.format(base_name, suffix)) for suffix in suffixes] + paths = [os.path.join(base_path, 'vector', f'{base_name}.{suffix}') for suffix in suffixes] return paths, suffixes, @on_ogc_backend(geoserver.BACKEND_PACKAGE) @@ -1144,7 +1142,7 @@ def test_moderated_upload(self, thumbnail_mock): with contextlib.ExitStack() as stack: input_files = [ stack.enter_context(_fp) for _fp in input_files] - files = dict(zip(['{}_file'.format(s) for s in suffixes], input_files)) + files = dict(zip([f'{s}_file' for s in suffixes], input_files)) files['base_file'] = files.pop('shp_file') files['permissions'] = '{}' files['charset'] = 'utf-8' @@ -1176,7 +1174,7 @@ def test_moderated_upload(self, thumbnail_mock): with contextlib.ExitStack() as stack: input_files = [ stack.enter_context(_fp) for _fp in input_files] - files = dict(zip(['{}_file'.format(s) for s in suffixes], input_files)) + files = dict(zip([f'{s}_file' for s in suffixes], input_files)) files['base_file'] = files.pop('shp_file') files['permissions'] = '{}' files['charset'] = 'utf-8' @@ -1352,28 +1350,28 @@ def test_geonode_rest_layer_uploader(self): thelayer_name = 'ming_female_1' thelayer_path = os.path.join( PROJECT_ROOT, - '../tests/data/%s' % thelayer_name) + f'../tests/data/{thelayer_name}') files = dict( base_file=SimpleUploadedFile( - '%s.shp' % thelayer_name, + f'{thelayer_name}.shp', open(os.path.join(thelayer_path, - '%s.shp' % thelayer_name), mode='rb').read()), + f'{thelayer_name}.shp'), mode='rb').read()), shx_file=SimpleUploadedFile( - '%s.shx' % thelayer_name, + f'{thelayer_name}.shx', open(os.path.join(thelayer_path, - '%s.shx' % thelayer_name), mode='rb').read()), + f'{thelayer_name}.shx'), mode='rb').read()), dbf_file=SimpleUploadedFile( - '%s.dbf' % thelayer_name, + f'{thelayer_name}.dbf', open(os.path.join(thelayer_path, - '%s.dbf' % thelayer_name), mode='rb').read()), + f'{thelayer_name}.dbf'), mode='rb').read()), prj_file=SimpleUploadedFile( - '%s.prj' % thelayer_name, + f'{thelayer_name}.prj', open(os.path.join(thelayer_path, - '%s.prj' % thelayer_name), mode='rb').read()) + f'{thelayer_name}.prj'), mode='rb').read()) ) files['permissions'] = '{}' files['charset'] = 'windows-1258' - files['layer_title'] = 'test layer_{}'.format('windows-1258') + files['layer_title'] = 'test layer_windows-1258' resp = self.client.post(layer_upload_url, data=files) # Check response status code if resp.status_code == 200: @@ -1422,34 +1420,34 @@ def test_geonode_same_UUID_error(self, thumbnail_mock): thelayer_name = 'hydrodata' thelayer_path = os.path.join( PROJECT_ROOT, - '../tests/data/%s' % thelayer_name) + f'../tests/data/{thelayer_name}') # Uploading the first one should be OK same_uuid_root_file = 'same_uuid_a' files = dict( base_file=SimpleUploadedFile( - '%s.shp' % same_uuid_root_file, + f'{same_uuid_root_file}.shp', open(os.path.join(thelayer_path, - '%s.shp' % same_uuid_root_file), mode='rb').read()), + f'{same_uuid_root_file}.shp'), mode='rb').read()), shx_file=SimpleUploadedFile( - '%s.shx' % same_uuid_root_file, + f'{same_uuid_root_file}.shx', open(os.path.join(thelayer_path, - '%s.shx' % same_uuid_root_file), mode='rb').read()), + f'{same_uuid_root_file}.shx'), mode='rb').read()), dbf_file=SimpleUploadedFile( - '%s.dbf' % same_uuid_root_file, + f'{same_uuid_root_file}.dbf', open(os.path.join(thelayer_path, - '%s.dbf' % same_uuid_root_file), mode='rb').read()), + f'{same_uuid_root_file}.dbf'), mode='rb').read()), prj_file=SimpleUploadedFile( - '%s.prj' % same_uuid_root_file, + f'{same_uuid_root_file}.prj', open(os.path.join(thelayer_path, - '%s.prj' % same_uuid_root_file), mode='rb').read()), + f'{same_uuid_root_file}.prj'), mode='rb').read()), xml_file=SimpleUploadedFile( - '%s.xml' % same_uuid_root_file, + f'{same_uuid_root_file}.xml', open(os.path.join(thelayer_path, - '%s.xml' % same_uuid_root_file), mode='rb').read()) + f'{same_uuid_root_file}.xml'), mode='rb').read()) ) files['permissions'] = '{}' files['charset'] = 'utf-8' - files['layer_title'] = 'test layer_{}'.format(same_uuid_root_file) + files['layer_title'] = f'test layer_{same_uuid_root_file}' resp = self.client.post(layer_upload_url, data=files) # Check response status code self.assertEqual(resp.status_code, 200) @@ -1458,29 +1456,29 @@ def test_geonode_same_UUID_error(self, thumbnail_mock): same_uuid_root_file = 'same_uuid_b' files = dict( base_file=SimpleUploadedFile( - '%s.shp' % same_uuid_root_file, + f'{same_uuid_root_file}.shp', open(os.path.join(thelayer_path, - '%s.shp' % same_uuid_root_file), mode='rb').read()), + f'{same_uuid_root_file}.shp'), mode='rb').read()), shx_file=SimpleUploadedFile( - '%s.shx' % same_uuid_root_file, + f'{same_uuid_root_file}.shx', open(os.path.join(thelayer_path, - '%s.shx' % same_uuid_root_file), mode='rb').read()), + f'{same_uuid_root_file}.shx'), mode='rb').read()), dbf_file=SimpleUploadedFile( - '%s.dbf' % same_uuid_root_file, + f'{same_uuid_root_file}.dbf', open(os.path.join(thelayer_path, - '%s.dbf' % same_uuid_root_file), mode='rb').read()), + f'{same_uuid_root_file}.dbf'), mode='rb').read()), prj_file=SimpleUploadedFile( - '%s.prj' % same_uuid_root_file, + f'{same_uuid_root_file}.prj', open(os.path.join(thelayer_path, - '%s.prj' % same_uuid_root_file), mode='rb').read()), + f'{same_uuid_root_file}.prj'), mode='rb').read()), xml_file=SimpleUploadedFile( - '%s.xml' % same_uuid_root_file, + f'{same_uuid_root_file}.xml', open(os.path.join(thelayer_path, - '%s.xml' % same_uuid_root_file), mode='rb').read()) + f'{same_uuid_root_file}.xml'), mode='rb').read()) ) files['permissions'] = '{}' files['charset'] = 'utf-8' - files['layer_title'] = 'test layer_{}'.format(same_uuid_root_file) + files['layer_title'] = f'test layer_{same_uuid_root_file}' resp = self.client.post(layer_upload_url, data=files) # Check response status code self.assertEqual(resp.status_code, 400) diff --git a/geonode/layers/utils.py b/geonode/layers/utils.py index 7a6e69964e1..0591fdfa657 100644 --- a/geonode/layers/utils.py +++ b/geonode/layers/utils.py @@ -154,8 +154,7 @@ def get_files(filename): # Make sure the file exists. if not os.path.exists(filename): - msg = ('Could not open %s. Make sure you are using a ' - 'valid file' % filename) + msg = f'Could not open {filename}. Make sure you are using a valid file' logger.debug(msg) raise GeoNodeException(msg) @@ -169,10 +168,9 @@ def get_files(filename): for ext, pattern in required_extensions.items(): matches = glob.glob(glob_name + pattern) if len(matches) == 0: - msg = ('Expected helper file %s does not exist; a Shapefile ' + msg = (f'Expected helper file {base_name}.{ext} does not exist; a Shapefile ' 'requires helper files with the following extensions: ' - '%s') % (base_name + "." + ext, - list(required_extensions.keys())) + f'{list(required_extensions.keys())}') raise GeoNodeException(msg) elif len(matches) > 1: msg = ('Multiple helper files for %s exist; they need to be ' @@ -253,7 +251,7 @@ def layer_type(filename): elif extension.lower() in cov_exts: return 'raster' else: - msg = ('Saving of extension [%s] is not implemented' % extension) + msg = f'Saving of extension [{extension}] is not implemented' raise GeoNodeException(msg) @@ -266,9 +264,9 @@ def get_valid_name(layer_name): while Layer.objects.filter(name=proposed_name).exists(): possible_chars = string.ascii_lowercase + string.digits suffix = "".join([choice(possible_chars) for i in range(4)]) - proposed_name = '%s_%s' % (name, suffix) + proposed_name = f'{name}_{suffix}' logger.debug('Requested name already used; adjusting name ' - '[%s] => [%s]', layer_name, proposed_name) + f'[{layer_name}] => [{proposed_name}]') return proposed_name @@ -328,7 +326,7 @@ def get_resolution(filename): gtif = gdal.Open(filename) gt = gtif.GetGeoTransform() __, resx, __, __, __, resy = gt - resolution = '%s %s' % (resx, resy) + resolution = f'{resx} {resy}' return resolution except Exception: return None @@ -403,7 +401,7 @@ def get_bbox(filename): except Exception: pass - return [bbox_x0, bbox_x1, bbox_y0, bbox_y1, "EPSG:%s" % str(srid)] + return [bbox_x0, bbox_x1, bbox_y0, bbox_y1, f"EPSG:{str(srid)}"] @transaction.atomic @@ -506,7 +504,7 @@ def file_upload(filename, that is consistent with the file you are trying to replace.")) except Exception as e: raise Exception( - _("Some error occurred while trying to access the uploaded schema: %s" % str(e))) + _(f"Some error occurred while trying to access the uploaded schema: {str(e)}")) # Set a default title that looks nice ... if title is None: @@ -551,8 +549,7 @@ def file_upload(filename, with open(fn, 'rb') as f: upload_session.layerfile_set.create( name=type_name, file=File( - f, name='%s.%s' % - (assigned_name or valid_name, type_name))) + f, name=f'{assigned_name or valid_name}.{type_name}')) # save the system assigned name for the remaining files if not assigned_name: the_file = upload_session.layerfile_set.all()[0].file.name @@ -797,7 +794,7 @@ def upload(incoming, user=None, overwrite=False, elif not os.path.isdir(incoming): msg = ('Please pass a filename or a directory name as the "incoming" ' - 'parameter, instead of %s: %s' % (incoming, type(incoming))) + f'parameter, instead of {incoming}: {type(incoming)}') logger.exception(msg) raise GeoNodeException(msg) else: @@ -893,7 +890,7 @@ def upload(incoming, user=None, overwrite=False, print(msg, file=sys.stderr) raise Exception from e - msg = "[%s] Layer for '%s' (%d/%d)" % (status, filename, i + 1, number) + msg = f"[{status}] Layer for '{filename}' ({i + 1}/{number})" info = {'file': filename, 'status': status} if status == 'failed': info['traceback'] = traceback @@ -921,7 +918,7 @@ def delete_orphaned_layers(): deleted.append(filename) except NotImplementedError as e: logger.error( - "Failed to delete orphaned layer file '{}': {}".format(filename, e)) + f"Failed to delete orphaned layer file '{filename}': {e}") return deleted @@ -947,9 +944,7 @@ def set_layers_permissions(permissions_name, resources_names=None, resources = Layer.objects.filter(Q(title__in=resources_names) | Q(name__in=resources_names)) except Layer.DoesNotExist: logger.warning( - 'No resources have been found with these names: %s.' % ( - ", ".join(resources_names) - ) + f'No resources have been found with these names: {", ".join(resources_names)}.' ) if not resources: logger.warning("No resources have been found. No update operations have been executed.") @@ -1001,8 +996,8 @@ def set_layers_permissions(permissions_name, resources_names=None, users.append(user) except User.DoesNotExist: logger.warning( - 'The user {} does not exists. ' - 'It has been skipped.'.format(username) + f'The user {username} does not exists. ' + 'It has been skipped.' ) # GROUPS groups = [] @@ -1013,8 +1008,8 @@ def set_layers_permissions(permissions_name, resources_names=None, groups.append(group) except Group.DoesNotExist: logger.warning( - 'The group {} does not exists. ' - 'It has been skipped.'.format(group_name) + f'The group {group_name} does not exists. ' + 'It has been skipped.' ) if not users and not groups: logger.error( @@ -1066,15 +1061,16 @@ def set_layers_permissions(permissions_name, resources_names=None, perm_spec["users"][_user] = list(u_perms_set) else: logger.warning( - "The user %s does not have " - "any permission on the layer %s. " - "It has been skipped." % (_user.username, resource.title) + f"The user {_user.username} does not have " + f"any permission on the layer {resource.title}. " + "It has been skipped." ) else: logger.warning( - "Warning! - The user %s is the layer %s owner, " + f"Warning! - The user {_user.username} is the " + f"layer {resource.title} owner, " "so its permissions can't be changed. " - "It has been skipped." % (_user.username, resource.title) + "It has been skipped." ) for g in groups: _group = g @@ -1107,8 +1103,9 @@ def set_layers_permissions(permissions_name, resources_names=None, perm_spec["groups"][g] = list(g_perms_set) else: logger.warning( - "The group %s does not have any permission on the layer %s. " - "It has been skipped." % (g.name, resource.title) + f"The group {g.name} does not have any permission " + f"on the layer {resource.title}. " + "It has been skipped." ) # Set final permissions resource.set_permissions(perm_spec) diff --git a/geonode/layers/views.py b/geonode/layers/views.py index 685d5bc4e96..4bf0fa23f27 100644 --- a/geonode/layers/views.py +++ b/geonode/layers/views.py @@ -141,7 +141,7 @@ def log_snippet(log_file): if not log_file or not os.path.isfile(log_file): - return "No log file at %s" % log_file + return f"No log file at {log_file}" with open(log_file, "r") as f: f.seek(0, 2) # Seek @ EOF @@ -468,12 +468,11 @@ def sld_definition(style): ] # Add required parameters for GXP lazy-loading - attribution = "%s %s" % (layer.owner.first_name, - layer.owner.last_name) if layer.owner.first_name or layer.owner.last_name else str( - layer.owner) + attribution = f"{layer.owner.first_name} {layer.owner.last_name}" if layer.owner.first_name or \ + layer.owner.last_name else str(layer.owner) srs = getattr(settings, 'DEFAULT_MAP_CRS', 'EPSG:3857') srs_srid = int(srs.split(":")[1]) if srs != "EPSG:900913" else 3857 - config["attribution"] = "%s" % attribution + config["attribution"] = f"{attribution}" config["format"] = getattr( settings, 'DEFAULT_LAYER_FORMAT', 'image/png') config["title"] = layer.title @@ -560,7 +559,7 @@ def sld_definition(style): e = dlxml.fromstring(wms_capabilities) for atype in e.findall( - "./[wms:Name='%s']/wms:Dimension[@name='time']" % (layer.alternate), namespaces): + f"./[wms:Name='{layer.alternate}']/wms:Dimension[@name='time']", namespaces): dim_name = atype.get('name') if dim_name: dim_name = str(dim_name).lower() @@ -634,7 +633,7 @@ def sld_definition(style): "remote": True, "url": service.service_url, "name": service.name, - "title": "[R] %s" % service.title} + "title": f"[R] {service.title}"} maplayer = GXPLayer( name=layer.alternate, ows_url=layer.ows_url, @@ -773,10 +772,7 @@ def load_layer_data(request, template='layers/layer_detail.html'): if not isinstance(data_dict['filtered_attributes'], str): filtered_attributes = [x for x in data_dict['filtered_attributes'] if '/load_layer_data' not in x] name = layername if ':' not in layername else layername.split(':')[1] - location = "{location}{service}".format(** { - 'location': settings.OGC_SERVER['default']['LOCATION'], - 'service': 'wms', - }) + location = f"{(settings.OGC_SERVER['default']['LOCATION'])}wms" headers = {} if request and 'access_token' in request.session: access_token = request.session['access_token'] @@ -929,7 +925,7 @@ def layer_metadata( "remote": True, "url": service.service_url, "name": service.name, - "title": "[R] %s" % service.title} + "title": f"[R] {service.title}"} maplayer = GXPLayer( name=layer.alternate, ows_url=layer.ows_url, @@ -1399,16 +1395,14 @@ def layer_remove(request, layername, template='layers/layer_remove.html'): }) if (request.method == 'POST'): try: - logger.debug('Deleting Layer {0}'.format(layer)) + logger.debug(f'Deleting Layer {layer}') with transaction.atomic(): Layer.objects.filter(id=layer.id).delete() except IntegrityError: raise except Exception as e: traceback.print_exc() - message = '{0}: {1}.'.format( - _('Unable to delete layer'), layer.alternate) - + message = f'{_("Unable to delete layer")}: {layer.alternate}.' if getattr(e, 'message', None) and 'referenced by layer group' in getattr(e, 'message', ''): message = _( 'This layer is a member of a layer group, you must remove the layer from the group ' @@ -1458,9 +1452,7 @@ def layer_granule_remove( coverages['coverages']['coverage'][0]['name'], store, granule_id) except Exception as e: traceback.print_exc() - message = '{0}: {1}.'.format( - _('Unable to delete layer'), layer.alternate) - + message = f'{_("Unable to delete layer")}: {layer.alternate}.' if 'referenced by layer group' in getattr(e, 'message', ''): message = _( 'This layer is a member of a layer group, you must remove the layer from the group ' @@ -1671,7 +1663,7 @@ def batch_permissions(request, model): if "cancel" in request.POST or not ids: return HttpResponseRedirect( - '/admin/{model}s/{model}/'.format(model=model.lower()) + f'/admin/{model.lower()}s/{model.lower()}/' ) if request.method == 'POST': @@ -1706,7 +1698,7 @@ def batch_permissions(request, model): except set_permissions.OperationalError as exc: celery_logger.exception('Sending task raised: %r', exc) return HttpResponseRedirect( - '/admin/{model}s/{model}/'.format(model=model.lower()) + f'/admin/{model.lower()}s/{model.lower()}/' ) return render( request, diff --git a/geonode/maps/management/commands/remove_broken_layers.py b/geonode/maps/management/commands/remove_broken_layers.py index 272ef0ab10d..381273022c5 100644 --- a/geonode/maps/management/commands/remove_broken_layers.py +++ b/geonode/maps/management/commands/remove_broken_layers.py @@ -31,5 +31,5 @@ def handle(self, *args, **options): map_layers = MapLayer.objects.filter(local=True) for maplayer in map_layers: if not Layer.objects.filter(alternate=maplayer.name).exists(): - print('Removing broken map layer {}'.format(maplayer.name)) + print(f'Removing broken map layer {maplayer.name}') maplayer.delete() diff --git a/geonode/maps/models.py b/geonode/maps/models.py index cefacc94013..f10bc554be3 100644 --- a/geonode/maps/models.py +++ b/geonode/maps/models.py @@ -90,8 +90,7 @@ class Map(ResourceBase, GXPMapBase): # Full URL for featured map view, ie http://domain/someview def __str__(self): - return '%s by %s' % ( - self.title, (self.owner.username if self.owner else "")) + return f'{self.title} by {(self.owner.username if self.owner else "")}' @property def center(self): @@ -131,17 +130,17 @@ def json(self, layer_filter): # the readme text will appear in a README file in the zip readme = ( - "Title: %s\n" + - "Author: %s\n" + - "Abstract: %s\n" - ) % (self.title, self.poc, self.abstract) + f"Title: {self.title}\n" + + f"Author: {self.poc}\n" + + f"Abstract: {self.abstract}\n" + ) if self.license: - readme += "License: %s" % self.license + readme += f"License: {self.license}" if self.license.url: - readme += " (%s)" % self.license.url + readme += f" ({self.license.url})" readme += "\n" if self.constraints_other: - readme += "Additional constraints: %s\n" % self.constraints_other + readme += f"Additional constraints: {self.constraints_other}\n" def layer_json(lyr): return { @@ -294,8 +293,7 @@ def create_from_layer_list(self, user, layers, title, abstract): layer = Layer.objects.get(alternate=layer) except ObjectDoesNotExist: raise Exception( - 'Could not find layer with name %s' % - layer) + f'Could not find layer with name {layer}') if not user.has_perm( 'base.view_resourcebase', @@ -359,7 +357,7 @@ def layer_group(self): """ if check_ogc_backend(geoserver.BACKEND_PACKAGE): from geonode.geoserver.helpers import gs_catalog, ogc_server_settings - lg_name = '%s_%d' % (slugify(self.title), self.id) + lg_name = f'{slugify(self.title)}_{self.id}' try: return { 'catalog': gs_catalog.get_layergroup(lg_name), @@ -405,7 +403,7 @@ def publish_layer_group(self): # Group layer bounds and name lg_bounds = [str(coord) for coord in self.bbox] - lg_name = '%s_%d' % (slugify(self.title), self.id) + lg_name = f'{slugify(self.title)}_{self.id}' # Update existing or add new group layer lg = self.layer_group @@ -573,19 +571,18 @@ def local_link(self): store=self.store, alternate=self.name) else: layer = Layer.objects.get(alternate=self.name) - link = "%s" % ( - layer.get_absolute_url(), layer.title) + link = f"{layer.title}" except Exception: link = None if link is None: - link = "%s " % self.name + link = f"{self.name} " return link class Meta: ordering = ["stack_order"] def __str__(self): - return '%s?layers=%s' % (self.ows_url, self.name) + return f'{self.ows_url}?layers={self.name}' def pre_delete_map(instance, sender, **kwrargs): diff --git a/geonode/maps/tests.py b/geonode/maps/tests.py index ee5c1121d31..bff902fe68f 100644 --- a/geonode/maps/tests.py +++ b/geonode/maps/tests.py @@ -274,9 +274,9 @@ def test_map_to_wmc(self): # check specific XPaths wmc = dlxml.fromstring(response.content) - namespace = '{http://www.opengis.net/context}' - title = '{ns}General/{ns}Title'.format(ns=namespace) - abstract = '{ns}General/{ns}Abstract'.format(ns=namespace) + ns = '{http://www.opengis.net/context}' + title = f'{ns}General/{ns}Title' + abstract = f'{ns}General/{ns}Abstract' self.assertIsNotNone(wmc.attrib.get('id')) self.assertEqual(wmc.find(title).text, 'GeoNode Default Map') @@ -328,16 +328,16 @@ def test_new_map_without_layers(self): def test_new_map_with_layer(self): layer = Layer.objects.all().first() - self.client.get(reverse('new_map') + '?layer=' + layer.alternate) + self.client.get(f"{reverse('new_map')}?layer={layer.alternate}") def test_new_map_with_empty_bbox_layer(self): layer = Layer.objects.all().first() - self.client.get(reverse('new_map') + '?layer=' + layer.alternate) + self.client.get(f"{reverse('new_map')}?layer={layer.alternate}") def test_add_layer_to_existing_map(self): layer = Layer.objects.all().first() map_obj = Map.objects.all().first() - self.client.get(reverse('add_layer') + '?layer_name=%s&map_id=%s' % (layer.alternate, map_obj.id)) + self.client.get(f"{reverse('add_layer')}?layer_name={layer.alternate}&map_id={map_obj.id}") map_obj = Map.objects.get(id=map_obj.id) for map_layer in map_obj.layers: diff --git a/geonode/maps/utils.py b/geonode/maps/utils.py index 9b7858391e3..cfc8a80dc26 100644 --- a/geonode/maps/utils.py +++ b/geonode/maps/utils.py @@ -86,7 +86,7 @@ def fix_baselayers(map_id): return if not Map.objects.filter(pk=id).exists(): - logger.error('There is not a map with id %s' % id) + logger.error(f'There is not a map with id {id}') return map = Map.objects.get(pk=id) diff --git a/geonode/maps/views.py b/geonode/maps/views.py index 36c6ccb7c8b..8a9632cff67 100644 --- a/geonode/maps/views.py +++ b/geonode/maps/views.py @@ -888,12 +888,10 @@ def sld_definition(style): 'properties': layer.srid } # Add required parameters for GXP lazy-loading - attribution = "%s %s" % (layer.owner.first_name, - layer.owner.last_name) if layer.owner.first_name or layer.owner.last_name else str( - layer.owner) + attribution = f"{layer.owner.first_name} {layer.owner.last_name}" if layer.owner.first_name or layer.owner.last_name else str(layer.owner) # noqa srs = getattr(settings, 'DEFAULT_MAP_CRS', 'EPSG:3857') srs_srid = int(srs.split(":")[1]) if srs != "EPSG:900913" else 3857 - config["attribution"] = "%s" % attribution + config["attribution"] = f"{attribution}" config["format"] = getattr( settings, 'DEFAULT_LAYER_FORMAT', 'image/png') config["title"] = layer.title @@ -977,7 +975,7 @@ def sld_definition(style): 'xsi': 'http://www.w3.org/2001/XMLSchema-instance'} e = dlxml.fromstring(wms_capabilities) for atype in e.findall( - "./[wms:Name='%s']/wms:Dimension[@name='time']" % (layer.alternate), namespaces): + f"./[wms:Name='{layer.alternate}']/wms:Dimension[@name='time']", namespaces): dim_name = atype.get('name') if dim_name: dim_name = str(dim_name).lower() @@ -1009,7 +1007,7 @@ def sld_definition(style): "remote": True, "url": service.service_url, "name": service.name, - "title": "[R] %s" % service.title} + "title": f"[R] {service.title}"} maplayer = MapLayer(map=map_obj, name=layer.alternate, ows_url=layer.ows_url, @@ -1024,7 +1022,7 @@ def sld_definition(style): access_token = request.session['access_token'] if request and 'access_token' in request.session else None if access_token and ogc_server_url == layer_url and 'access_token' not in layer.ows_url: - url = '%s?access_token=%s' % (layer.ows_url, access_token) + url = f'{layer.ows_url}?access_token={access_token}' else: url = layer.ows_url maplayer = MapLayer( diff --git a/geonode/messaging/consumer.py b/geonode/messaging/consumer.py index b750b0f79d5..4b77cb591f9 100644 --- a/geonode/messaging/consumer.py +++ b/geonode/messaging/consumer.py @@ -84,12 +84,12 @@ def on_consume_end(self, connection, channel): logger.debug("finished.") def on_message(self, body, message): - logger.debug("broadcast: RECEIVED MSG - body: %r" % (body,)) + logger.debug(f"broadcast: RECEIVED MSG - body: {body}") message.ack() self._check_message_limit() def on_email_messages(self, body, message): - logger.debug("on_email_messages: RECEIVED MSG - body: %r" % (body,)) + logger.debug(f"on_email_messages: RECEIVED MSG - body: {body}") layer_uuid = body.get("layer_uuid") user_id = body.get("user_id") send_email_consumer(layer_uuid, user_id) @@ -99,7 +99,7 @@ def on_email_messages(self, body, message): self._check_message_limit() def on_geoserver_messages(self, body, message): - logger.debug("on_geoserver_messages: RECEIVED MSG - body: %r" % (body,)) + logger.debug(f"on_geoserver_messages: RECEIVED MSG - body: {body}") layer_id = body.get("id") try: layer = _wait_for_layer(layer_id) @@ -115,7 +115,7 @@ def on_geoserver_messages(self, body, message): self._check_message_limit() def on_notifications_messages(self, body, message): - logger.debug("on_notifications_message: RECEIVED MSG - body: %r" % (body,)) + logger.debug(f"on_notifications_message: RECEIVED MSG - body: {body}") body.get("id") body.get("app_label") body.get("model") @@ -126,30 +126,30 @@ def on_notifications_messages(self, body, message): self._check_message_limit() def on_geoserver_all(self, body, message): - logger.debug("on_geoserver_all: RECEIVED MSG - body: %r" % (body,)) + logger.debug(f"on_geoserver_all: RECEIVED MSG - body: {body}") message.ack() logger.debug("on_geoserver_all: finished") # TODO:Adding consurmer's producers. self._check_message_limit() def on_geoserver_catalog(self, body, message): - logger.debug("on_geoserver_catalog: RECEIVED MSG - body: %r" % (body,)) + logger.debug(f"on_geoserver_catalog: RECEIVED MSG - body: {body}") try: _update_layer_data(body, self.last_message) self.last_message = json.loads(body) except Exception: - logger.debug("Could not encode message {!r}".format(body)) + logger.debug(f"Could not encode message {body}") message.ack() logger.debug("on_geoserver_catalog: finished") self._check_message_limit() def on_geoserver_data(self, body, message): - logger.debug("on_geoserver_data: RECEIVED MSG - body: %r" % (body,)) + logger.debug(f"on_geoserver_data: RECEIVED MSG - body: {body}") try: _update_layer_data(body, self.last_message) self.last_message = json.loads(body) except Exception: - logger.debug("Could not encode message {!r}".format(body)) + logger.debug(f"Could not encode message {body}") message.ack() logger.debug("on_geoserver_data: finished") self._check_message_limit() @@ -157,14 +157,14 @@ def on_geoserver_data(self, body, message): def on_consume_ready(self, connection, channel, consumers, **kwargs): logger.debug(">>> Ready:") logger.debug(connection) - logger.debug("{} consumers:".format(len(consumers))) + logger.debug(f"{len(consumers)} consumers:") for i, consumer in enumerate(consumers, start=1): - logger.debug("{0} {1}".format(i, consumer)) - super(Consumer, self).on_consume_ready(connection, channel, consumers, - **kwargs) + logger.debug(f"{i} {consumer}") + super(Consumer, self).on_consume_ready( + connection, channel, consumers, **kwargs) def on_layer_viewer(self, body, message): - logger.debug("on_layer_viewer: RECEIVED MSG - body: %r" % (body,)) + logger.debug(f"on_layer_viewer: RECEIVED MSG - body: {body}") viewer = body.get("viewer") # owner_layer = body.get("owner_layer") layer_id = body.get("layer_id") @@ -217,15 +217,18 @@ def _wait_for_layer(layer_id, num_attempts=5, wait_seconds=1): for current in range(1, num_attempts + 1): try: instance = Layer.objects.get(id=layer_id) - logger.debug("Attempt {}/{} - Found layer in the " - "database".format(current, num_attempts)) + logger.debug( + f"Attempt {current}/{num_attempts} - Found layer in the " + "database") break except Layer.DoesNotExist: time.sleep(wait_seconds) - logger.debug("Attempt {}/{} - Could not find layer " - "instance".format(current, num_attempts)) + logger.debug( + f"Attempt {current}/{num_attempts} - Could not find layer " + "instance") else: - logger.debug("Reached maximum attempts and layer {!r} is still not " - "saved. Exiting...".format(layer_id)) + logger.debug( + f"Reached maximum attempts and layer {layer_id} is still not " + "saved. Exiting...") raise Layer.DoesNotExist return instance diff --git a/geonode/messaging/management/commands/runmessaging.py b/geonode/messaging/management/commands/runmessaging.py index ed3bceed619..b725553d285 100644 --- a/geonode/messaging/management/commands/runmessaging.py +++ b/geonode/messaging/management/commands/runmessaging.py @@ -19,10 +19,8 @@ ######################################################################### import logging -import sys from django.core.management.base import BaseCommand -from django.conf import settings from geonode.messaging import connection from geonode.messaging.consumer import Consumer diff --git a/geonode/messaging/producer.py b/geonode/messaging/producer.py index 3014b2c2131..cc6ad654f9f 100644 --- a/geonode/messaging/producer.py +++ b/geonode/messaging/producer.py @@ -73,11 +73,11 @@ def sync_if_local_memory(func, *args, **kwargs): worker.run(timeout=broker_socket_timeout) except Exception: tb = traceback.format_exc() - msg = "Exception while publishing message: {}".format(tb) + msg = f"Exception while publishing message: {tb}" logger.error(msg) raise Exception(msg) elif not getattr(connection.connection, 'driver_name', None): - msg = "Exception while getting connection to {}".format(url) + msg = f"Exception while getting connection to {url}" logger.error(msg) raise Exception(msg) diff --git a/geonode/monitoring/__init__.py b/geonode/monitoring/__init__.py index c5b4d88bd49..2488b0748be 100644 --- a/geonode/monitoring/__init__.py +++ b/geonode/monitoring/__init__.py @@ -97,7 +97,7 @@ def register_event(request, event_type, resource): resource_name = getattr(resource, 'alternate', None) or resource.title resource_id = resource.id else: - raise ValueError("Invalid resource: {}".format(resource)) + raise ValueError(f"Invalid resource: {resource}") if request and hasattr(request, 'register_event'): request.register_event(event_type, resource_type, resource_name, resource_id) diff --git a/geonode/monitoring/aggregation.py b/geonode/monitoring/aggregation.py index d62e9084419..935350df2fb 100644 --- a/geonode/monitoring/aggregation.py +++ b/geonode/monitoring/aggregation.py @@ -64,7 +64,7 @@ def get_metric_names(): def get_labels_for_metric(metric_name, resource=None): mt = ServiceTypeMetric.objects.filter(metric__name=metric_name) if not mt: - raise ValueError("No metric for {}".format(metric_name)) + raise ValueError(f"No metric for {metric_name}") qparams = {'metric_values__service_metric__in': mt} if resource: @@ -76,7 +76,7 @@ def get_labels_for_metric(metric_name, resource=None): def get_resources_for_metric(metric_name): mt = ServiceTypeMetric.objects.filter(metric__name=metric_name) if not mt: - raise ValueError("No metric for {}".format(metric_name)) + raise ValueError(f"No metric for {metric_name}") return list(MonitoredResource.objects.filter(metric_values__service_metric__in=mt) .exclude(name='', type='') .distinct() diff --git a/geonode/monitoring/collector.py b/geonode/monitoring/collector.py index e9203ee578f..5107c46bcd9 100644 --- a/geonode/monitoring/collector.py +++ b/geonode/monitoring/collector.py @@ -106,7 +106,7 @@ def get_network_rate(row, value, metric_defaults, 'value_raw': rate, 'value_num': rate, 'label': iface_label, - 'metric': '{}.rate'.format(metric_name)} + 'metric': f'{metric_name}.rate'} mdata.update(metric_defaults) log.debug(MetricValue.add(**mdata)) @@ -207,13 +207,13 @@ def process_host_geonode(self, service, data, valid_from, valid_to): 'value_raw': tx_value, 'value_num': tx_value, 'label': ifname, - 'metric': 'network.{}'.format(tx_label)} + 'metric': f'network.{tx_label}'} mdata.update(mdefaults) rate = self._calculate_rate( mdata['metric'], ifname, tx_value, valid_to) log.debug(MetricValue.add(**mdata)) if rate: - mdata['metric'] = '{}.rate'.format(mdata['metric']) + mdata['metric'] = f"{mdata['metric']}.rate" mdata['value'] = rate mdata['value_num'] = rate mdata['value_raw'] = rate @@ -232,7 +232,7 @@ def process_host_geonode(self, service, data, valid_from, valid_to): mdata = {'value': mdata, 'value_raw': mdata, 'value_num': mdata, - 'metric': 'mem.{}'.format(mkey), + 'metric': f'mem.{mkey}', 'label': 'B', } mdata.update(mdefaults) @@ -275,7 +275,7 @@ def process_host_geonode(self, service, data, valid_from, valid_to): mdata = {'value': l, 'value_raw': l, 'value_num': l, - 'metric': 'load.{}m'.format(llabel[lidx]), + 'metric': f'load.{llabel[lidx]}m', 'label': 'Value', } @@ -329,7 +329,7 @@ def process_host_geonode(self, service, data, valid_from, valid_to): mdata['valid_to']) if rate: rate_data = mdata.copy() - rate_data['metric'] = '{}.rate'.format(mdata['metric']) + rate_data['metric'] = f"{mdata['metric']}.rate" rate_data['value'] = rate rate_data['value_num'] = rate rate_data['value_raw'] = rate @@ -342,7 +342,7 @@ def process_host_geonode(self, service, data, valid_from, valid_to): mdata['valid_to']) if percent: percent_data = mdata.copy() - percent_data['metric'] = '{}.percent'.format(mdata['metric']) + percent_data['metric'] = f"{mdata['metric']}.percent" percent_data['value'] = percent percent_data['value_num'] = percent percent_data['value_raw'] = percent @@ -440,7 +440,7 @@ def _key(v): row['label'] = Metric.TYPE_VALUE_NUMERIC q.append(row) else: - raise ValueError("Unsupported metric type: {}".format(metric.type)) + raise ValueError(f"Unsupported metric type: {metric.type}") rows = q[:100] metric_values.update({'metric': metric_name, 'service': service}) for row in rows: @@ -660,7 +660,7 @@ def get_aggregate_function(self, column_name, metric_name, service=None): """ metric = Metric.get_for(metric_name, service=service) if not metric: - raise ValueError("Invalid metric {}".format(metric_name)) + raise ValueError(f"Invalid metric {metric_name}") f = metric.get_aggregate_name() return f or column_name @@ -831,9 +831,9 @@ def get_metrics_data(self, metric_name, q_order_by = ['val desc'] - q_select = [('select ml.name as label, {} as val, ' + q_select = [(f'select ml.name as label, {agg_f} as val, ' 'count(1) as metric_count, sum(samples_count) as samples_count, ' - 'sum(mv.value_num), min(mv.value_num), max(mv.value_num)').format(agg_f)] + 'sum(mv.value_num), min(mv.value_num), max(mv.value_num)')] if service and service_type: raise ValueError( "Cannot use service and service type in the same query") @@ -876,11 +876,11 @@ def get_metrics_data(self, metric_name, group_by_cfg = group_by_map[group_by] g_sel = group_by_cfg.get('select') if g_sel: - q_select.append(', {}'.format(', '.join(g_sel))) + q_select.append(f", {(', '.join(g_sel))}") g_sel = group_by_cfg.get('select_only') if g_sel: - q_select = ['select {}'.format(', '.join(g_sel))] + q_select = [f"select {(', '.join(g_sel))}"] q_from.extend(group_by_cfg['from']) q_where.extend(group_by_cfg['where']) @@ -916,7 +916,7 @@ def get_metrics_data(self, metric_name, if q_group: q_group = [' group by ', ','.join(q_group)] if q_order_by: - q_order_by = 'order by {}'.format(','.join(q_order_by)) + q_order_by = f"order by {(','.join(q_order_by))}" q = ' '.join(chain(q_select, q_from, q_where, q_group, [q_order_by])) @@ -968,7 +968,7 @@ def clear_old_data(self): threshold = settings.MONITORING_DATA_TTL if not isinstance(threshold, timedelta): raise TypeError("MONITORING_DATA_TTL should be an instance of " - "datatime.timedelta, not {}".format(threshold.__class__)) + f"datatime.timedelta, not {threshold.__class__}") cutoff = datetime.utcnow().replace(tzinfo=utc) - threshold ExceptionEvent.objects.filter(created__lte=cutoff).delete() RequestEvent.objects.filter(created__lte=cutoff).delete() @@ -999,8 +999,8 @@ def emit_notifications(self, for_timestamp=None): def send_mails(self, notification, emails, ndata, when=None): base_ctx = self.compose_notifications(ndata, when=when) - subject = _("GeoNode Monitoring on {} reports errors: {}").format(base_ctx['host'], - notification.notification_subject) + subject = _(f"GeoNode Monitoring on {base_ctx['host']} " + f"reports errors: {notification.notification_subject}") for email in emails: ctx = {'recipient': {'username': email}} ctx.update(base_ctx) diff --git a/geonode/monitoring/management/commands/render_metrics.py b/geonode/monitoring/management/commands/render_metrics.py index 51e0902ed1f..198a4cddd0e 100644 --- a/geonode/monitoring/management/commands/render_metrics.py +++ b/geonode/monitoring/management/commands/render_metrics.py @@ -47,15 +47,17 @@ def add_arguments(self, parser): parser.add_argument('-m', '--list-metrics', dest='list_metrics', action='store_true', default=False, help=_("Show list of metrics")) parser.add_argument('-l', '--list-labels', dest='list_labels', action='store_true', default=False, - help=_("Show list of labels for metric")) + help=_("Show list of labels for metric")) parser.add_argument('-r', '--list-resources', dest='list_resources', action='store_true', default=False, - help=_("Show list of resources for metric")) + help=_("Show list of resources for metric")) parser.add_argument('-s', '--since', dest='since', default=None, type=parse_datetime, - help=_("Process data since specific timestamp (YYYY-MM-DD HH:MM:SS format). If not provided, last sync will be used.")) + help=_("Process data since specific timestamp (YYYY-MM-DD HH:MM:SS format). " + "If not provided, last sync will be used.")) parser.add_argument('-u', '--until', dest='until', default=None, type=parse_datetime, - help=_("Process data until specific timestamp (YYYY-MM-DD HH:MM:SS format). If not provided, now will be used.")) + help=_("Process data until specific timestamp (YYYY-MM-DD HH:MM:SS format). " + "If not provided, now will be used.")) parser.add_argument('-i', '--interval', dest='interval', default=60, type=int, help=_("Data aggregation interval in seconds (default: 60)")) @@ -70,7 +72,6 @@ def add_arguments(self, parser): parser.add_argument('-ll', '--label', dest='label', type=TypeChecks.label_type, help=_("Show data for specific label")) - @timeout_decorator.timeout(LOCAL_TIMEOUT) def handle(self, *args, **options): self.collector = CollectorAPI() @@ -88,7 +89,6 @@ def handle(self, *args, **options): if isinstance(metric_names, str): metric_names = [metric_names] for m in metric_names: - #def get_metrics_for(self, metric_name, valid_from=None, valid_to=None, interval=None, service=None, label=None, resource=None): if options['list_labels']: self.list_labels(m) elif options['list_resources']: @@ -99,40 +99,41 @@ def handle(self, *args, **options): @timeout_decorator.timeout(LOCAL_TIMEOUT) def list_labels(self, metric, resource=None): labels = self.collector.get_labels_for_metric(metric, resource=resource) - print('Labels for metric {}'.format(metric)) + print(f'Labels for metric {metric}') for label in labels: print(' ', *label) @timeout_decorator.timeout(LOCAL_TIMEOUT) def list_resources(self, metric): resources = self.collector.get_resources_for_metric(metric) - print('Resources for metric {}'.format(metric)) + print(f'Resources for metric {metric}') for res in resources: print(' ', '='.join(res)) @timeout_decorator.timeout(LOCAL_TIMEOUT) def show_metrics(self, metric, since, until, interval, resource=None, label=None, service=None): - print('Monitoring Metric values for {}'.format(metric)) + print(f'Monitoring Metric values for {metric}') if service: - print(' for service: {} '.format(service)) + print(f' for service: {service} ') if resource: - print(' for resource: {}={} '.format(resource.type, resource.name)) + print(f' for resource: {resource.type}={resource.name} ') if label: - print(' for label: {} label'.format(label.name)) + print(f' for label: {label.name} label') utc = pytz.utc since = since.replace(tzinfo=utc) if since else None until = until.replace(tzinfo=utc) if until else None - data = self.collector.get_metrics_for(metric, - valid_from=since, - valid_to=until, - interval=interval, - resource=resource, - label=label, - service=service) - print(' since {} until {}\n'.format(data['input_valid_from'].strftime(TIMESTAMP_OUTPUT), - data['input_valid_to'].strftime(TIMESTAMP_OUTPUT))) + data = self.collector.get_metrics_for( + metric, + valid_from=since, + valid_to=until, + interval=interval, + resource=resource, + label=label, + service=service) + print(f" since {data['input_valid_from'].strftime(TIMESTAMP_OUTPUT)} " + f"until {data['input_valid_to'].strftime(TIMESTAMP_OUTPUT)}\n") for row in data['data']: val = None @@ -140,11 +141,10 @@ def show_metrics(self, metric, since, until, interval, resource=None, label=None val = row['data'][0]['val'] print(' ', row['valid_to'].strftime(TIMESTAMP_OUTPUT), '->', '' if not val else val) - @timeout_decorator.timeout(LOCAL_TIMEOUT) def list_metrics(self): _metrics = self.collector.get_metric_names() for stype, metrics in _metrics: print('service type', stype.name) for m in metrics: - print(' {}[{}]'.format(m.name, m.type)) + print(f' {m.name}[{m.type}]') diff --git a/geonode/monitoring/management/commands/updategeoip.py b/geonode/monitoring/management/commands/updategeoip.py index c0cc5dc742f..55dc15682cf 100644 --- a/geonode/monitoring/management/commands/updategeoip.py +++ b/geonode/monitoring/management/commands/updategeoip.py @@ -41,7 +41,7 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument('-f', '--file', dest='file', default=settings.GEOIP_PATH, - help=_("Write result to file, default GEOIP_PATH: {}".format(settings.GEOIP_PATH))) + help=_(f"Write result to file, default GEOIP_PATH: {settings.GEOIP_PATH}")) parser.add_argument('-u', '--url', dest='url', default=URL, help=_("Fetch database from specific url. If nothing provided, default {} will be used")) parser.add_argument('-o', '--overwrite', dest='overwrite', action='store_true', default=False, @@ -67,10 +67,10 @@ def handle(self, *args, **options): block_size = 1024 wrote = 0 with open('output.bin', 'wb') as f: - for data in tqdm(r.iter_content(block_size), total=math.ceil(total_size//block_size) , unit='KB', unit_scale=False): - wrote = wrote + len(data) + for data in tqdm(r.iter_content(block_size), total=math.ceil(total_size//block_size), unit='KB', unit_scale=False): # noqa + wrote = wrote + len(data) f.write(data) - logger.info(" total_size [%d] / wrote [%d] " % (total_size, wrote)) + logger.info(f" total_size [{total_size}] / wrote [{wrote}] ") if total_size != 0 and wrote != total_size: logger.info("ERROR, something went wrong") else: @@ -84,7 +84,6 @@ def handle(self, *args, **options): except OSError: pass - def handle_new_format(self, f, fname): try: with tarfile.open(fileobj=f) as zfile: @@ -106,7 +105,6 @@ def handle_new_format(self, f, fname): except Exception as err: logger.error("Cannot process %s: %s", f, err, exc_info=err) - def handle_old_format(self, f, fname): try: with gzip.GzipFile(fileobj=f) as zfile: @@ -119,6 +117,6 @@ def handle_old_format(self, f, fname): try: os.remove(fname) except OSError: - logger.debug('Could not delete file %s' % fname) + logger.debug(f'Could not delete file {fname}') except Exception as err: logger.error("Cannot process %s: %s", f, err, exc_info=err) diff --git a/geonode/monitoring/middleware.py b/geonode/monitoring/middleware.py index b8110c932bc..e71b2b91da1 100644 --- a/geonode/monitoring/middleware.py +++ b/geonode/monitoring/middleware.py @@ -47,7 +47,7 @@ def __init__(self, get_response): self.setup_logging() def setup_logging(self): - self.log = logging.getLogger('{}.catcher'.format(__name__)) + self.log = logging.getLogger(f'{__name__}.catcher') self.log.propagate = False self.log.setLevel(logging.DEBUG) self.log.handlers = [] diff --git a/geonode/monitoring/models.py b/geonode/monitoring/models.py index 9c3c3c45149..d179bc3a41a 100644 --- a/geonode/monitoring/models.py +++ b/geonode/monitoring/models.py @@ -83,7 +83,7 @@ class Host(models.Model): active = models.BooleanField(null=False, blank=False, default=True) def __str__(self): - return 'Host: {} ({})'.format(self.name, self.ip) + return f'Host: {self.name} ({self.ip})' class ServiceType(models.Model): @@ -109,7 +109,7 @@ class ServiceType(models.Model): choices=TYPES) def __str__(self): - return 'Service Type: {}'.format(self.name) + return f'Service Type: {self.name}' @property def is_system_monitor(self): @@ -136,7 +136,7 @@ class Service(models.Model): url = models.URLField(null=True, blank=True, default='') def __str__(self): - return 'Service: {}@{}'.format(self.name, self.host.name) + return f'Service: {self.name}@{self.host.name}' def get_metrics(self): return [m.metric for m in self.service_type.metric.all()] @@ -191,7 +191,7 @@ class Meta: unique_together = (('name', 'type',),) def __str__(self): - return 'Monitored Resource: {} {}'.format(self.name, self.type) + return f'Monitored Resource: {self.name} {self.type}' @classmethod def get(cls, resource_type, resource_name, or_create=False): @@ -282,7 +282,7 @@ def get_aggregate_name(self): return self.AGGREGATE_MAP[self.type] def __unicode__(self): - return "Metric: {}".format(self.name) + return f"Metric: {self.name}" @property def is_rate(self): @@ -320,7 +320,7 @@ class ServiceTypeMetric(models.Model): metric = models.ForeignKey(Metric, related_name='service_type', on_delete=models.CASCADE) def __str__(self): - return '{} - {}'.format(self.service_type, self.metric) + return f'{self.service_type} - {self.metric}' class EventType(models.Model): @@ -346,7 +346,7 @@ class EventType(models.Model): EVENT_OTHER = 'other' # non-ows event EVENT_ALL = 'all' # all events - baseline: ows + non-ows - EVENT_TYPES = list(zip(['OWS:{}'.format(ows) for ows in _ows_types], _ows_types)) + \ + EVENT_TYPES = list(zip([f'OWS:{ows}' for ows in _ows_types], _ows_types)) + \ [(EVENT_OTHER, _("Not OWS"))] +\ [(EVENT_OWS, _("Any OWS"))] +\ [(EVENT_ALL, _("All"))] +\ @@ -367,7 +367,7 @@ class EventType(models.Model): blank=False) def __str__(self): - return 'Event Type: {}'.format(self.name) + return f'Event Type: {self.name}' @classmethod def get(cls, service_name=None): @@ -766,18 +766,20 @@ def from_geoserver(cls, service, request_data, received=None): rl = rd['responseLength'] event_type_name = rd.get('service') if event_type_name: - event_type = EventType.get('OWS:{}'.format(event_type_name.upper())) + event_type = EventType.get(f'OWS:{event_type_name.upper()}') else: event_type = EventType.get(EventType.EVENT_GEOSERVER) + if rd.get('queryString'): + request_path = f"{rd['path']}?{rd['queryString']}" + else: + request_path = rd['path'] data = {'created': start_time, 'received': received, 'host': rd['host'], 'event_type': event_type, 'service': service, - 'request_path': - '{}?{}'.format(rd['path'], rd['queryString']) if rd.get( - 'queryString') else rd['path'], + 'request_path': request_path, 'request_method': rd['httpMethod'], 'response_status': rd['responseStatus'], 'response_size': rl[0] if isinstance(rl, list) else rl, @@ -829,7 +831,7 @@ def add_error(cls, from_service, error_type, stack_trace, received = datetime.utcnow().replace(tzinfo=pytz.utc) if not isinstance(error_type, str): _cls = error_type.__class__ - error_type = '{}.{}'.format(_cls.__module__, _cls.__name__) + error_type = f'{_cls.__module__}.{_cls.__name__}' if not message: message = str(error_type) if isinstance(stack_trace, (list, tuple)): @@ -896,7 +898,7 @@ class MetricLabel(models.Model): blank=True) def __unicode__(self): - return 'Metric Label: {}'.format(self.name.encode('ascii', 'ignore')) + return f"Metric Label: {self.name.encode('ascii', 'ignore')}" class MetricValue(models.Model): @@ -944,13 +946,11 @@ def __str__(self): metric = self.service_metric.metric.name if self.label: _l = self.label.name - metric = '{} [{}]'.format(metric, _l) + metric = f'{metric} [{_l}]' if self.resource and self.resource.type: - metric = '{} for {}'.format( - metric, '{}={}'.format( - self.resource.type, self.resource.name)) - return 'Metric Value: {}: [{}] (since {} until {})'.format( - metric, self.value, self.valid_from, self.valid_to) + _s = f'{self.resource.type}={self.resource.name}' + metric = f'{metric} for {_s}' + return f'Metric Value: {metric}: [{self.value}] (since {self.valid_from} until {self.valid_to})' @classmethod def add(cls, metric, valid_from, valid_to, service, label, @@ -1119,11 +1119,11 @@ class NotificationCheck(models.Model): help_text=_("Is it active")) def __str__(self): - return "Notification Check #{}: {}".format(self.id, self.name) + return f"Notification Check #{self.id}: {self.name}" @property def notification_subject(self): - return _("{}: {}").format(self.severity, self.name) + return _(f"{self.severity}: {self.name}") @property def is_warning(self): @@ -1213,8 +1213,7 @@ def get_steps(cls, min_, max_, thresholds): pass else: raise TypeError( - "Unsupported threshold type: {} ({})".format( - thresholds, type(thresholds))) + f"Unsupported threshold type: {thresholds} ({type(thresholds)})") return thresholds @classmethod @@ -1455,9 +1454,7 @@ def current_value(self): val_ = val return { 'class': - '{}.{}'.format( - val.__class__.__module__, - val.__class__.__name__), + f'{val.__class__.__module__}.{val.__class__.__name__}', 'value': val_} except MetricNotificationCheck.DoesNotExist: return @@ -1486,7 +1483,7 @@ def get_fields(self): @property def field_name(self): - return '{}.{}'.format(self.metric.name, self.field_option) + return f'{self.metric.name}.{self.field_option}' def populate_min_max(self): notification = self.notification_check @@ -1502,7 +1499,7 @@ def populate_min_max(self): .filter( notification_check=self.notification_check, metric=self.metric, - **{'{}__isnull'.format(self.field_option): False})\ + **{f'{self.field_option}__isnull': False})\ .get() if mcheck: self.metric_check = mcheck @@ -1552,16 +1549,14 @@ class MetricNotificationCheck(models.Model): def __str__(self): indicator = [] if self.min_value is not None: - indicator.append("value above {}".format(self.min_value)) + indicator.append(f"value above {self.min_value}") if self.max_value is not None: - indicator.append("value below {}".format(self.max_value)) + indicator.append(f"value below {self.max_value}") if self.max_timeout is not None: indicator.append( - "value must be collected within {}".format( - self.max_timeout)) + f"value must be collected within {self.max_timeout}") indicator = ' and '.join(indicator) - return "MetricCheck({}@{}: {})".format( - self.metric.name, self.service.name if self.service else '', indicator) + return f"MetricCheck({self.metric.name}@{(self.service.name if self.service else '')}: {indicator})" @property def field_option(self): @@ -1596,10 +1591,7 @@ def __init__(self, metric, check, message, self.valid_to = metric.valid_to if hasattr(metric, 'valid_to') else None def __str__(self): - return "MetricValueError({}: metric {} misses {} check: {})".format(self.severity, - self.metric, - self.check, - self.message) + return f"MetricValueError({self.severity}: metric {self.metric} misses {self.check} check: {self.message})" def check_value(self, metric, valid_on): """ @@ -1608,7 +1600,7 @@ def check_value(self, metric, valid_on): v = metric.value_num m = metric.service_metric.metric metric_name = m.description or m.name - unit_name = ' {}'.format(m.unit) if not m.is_count else '' + unit_name = f' {m.unit}' if not m.is_count else '' had_check = False if self.definition: @@ -1617,30 +1609,23 @@ def check_value(self, metric, valid_on): if self.event_type: os = self.event_type if os.is_all or os.is_other: - msg_prefix.append("for {} OWS".format(os.name)) + msg_prefix.append(f"for {os.name} OWS") else: - msg_prefix.append("for {} OWS".format(os.name)) + msg_prefix.append(f"for {os.name} OWS") if self.service: - msg_prefix.append("for {} service".format(self.service.name)) + msg_prefix.append(f"for {self.service.name} service") if self.resource: msg_prefix.append( - "for {}[{}] resource".format( - self.resource.name, - self.resource.type)) + f"for {self.resource.name}[{self.resource.type}] resource") msg_prefix = ' '.join(msg_prefix) - description_tmpl = ("{} {} should be {{}} " - "{{:0.0f}}{}, got {{:0.0f}}{} instead").format(msg_prefix, - metric_name, - unit_name, - unit_name)\ - .strip() + description_tmpl = (f"{msg_prefix} {metric_name} should be {{}} " + f"{{:0.0f}}{unit_name}, got {{:0.0f}}{unit_name} instead").strip() if v is not None and self.min_value is not None: had_check = True if float(v) < float(self.min_value): - msg = "{} {} {}".format( - def_msg, int(self.min_value), unit_name) + msg = f"{def_msg} {int(self.min_value)} {unit_name}" description = description_tmpl.format( 'at least', float(self.min_value), float(v)) raise self.MetricValueError( @@ -1648,8 +1633,7 @@ def check_value(self, metric, valid_on): if v is not None and self.max_value is not None: had_check = True if float(v) > float(self.max_value): - msg = "{} {} {}".format( - def_msg, int(self.max_value), unit_name) + msg = f"{def_msg} {int(self.max_value)} {unit_name}" description = description_tmpl.format( 'at most', float(self.max_value), float(v)) raise self.MetricValueError( @@ -1665,11 +1649,11 @@ def check_value(self, metric, valid_on): if (valid_on - metric.valid_to) > self.max_timeout: total_seconds = self.max_timeout.total_seconds() actual_seconds = (valid_on - metric.valid_to).total_seconds() - msg = "{} {} seconds".format(def_msg, int(total_seconds)) - description = description_tmpl.format('recored at most ', - '{} seconds ago'.format( - total_seconds), - '{} seconds'.format(actual_seconds)) + msg = f"{def_msg} {int(total_seconds)} seconds" + description = description_tmpl.format( + 'recored at most ', + f'{total_seconds} seconds ago', + f'{actual_seconds} seconds') raise self.MetricValueError(metric, self, msg, @@ -1683,7 +1667,7 @@ def check_value(self, metric, valid_on): "", None, None, - "Metric check {} is not checking anything".format(self)) + f"Metric check {self} is not checking anything") def check_metric(self, for_timestamp=None): """ @@ -1710,7 +1694,7 @@ def check_metric(self, for_timestamp=None): "", None, None, - "Cannot find metric values for {} on {}".format(self.metric, for_timestamp)) + f"Cannot find metric values for {self.metric} on {for_timestamp}") for m in metrics: self.check_value(m, for_timestamp) return True @@ -1848,14 +1832,13 @@ def do_autoconfigure(): _host_by_name = '127.0.0.1' hosts = [(wsite.hostname, _host_by_name,)] # default geonode - geonode_name = settings.MONITORING_SERVICE_NAME or '{}-geonode'.format( - wsite.hostname) + geonode_name = settings.MONITORING_SERVICE_NAME or f'{wsite.hostname}-geonode' geonodes = [(geonode_name, settings.SITEURL, hosts[0])] geoservers = [] for k, val in settings.OGC_SERVER.items(): if val.get('BACKEND') == 'geonode.geoserver': - gname = '{}-geoserver'.format(k) + gname = f'{k}-geoserver' gsite = urlparse(val['LOCATION']) try: _host_by_name = gethostbyname(gsite.hostname) @@ -1899,7 +1882,7 @@ def do_autoconfigure(): service_type=geonode_type) service.save() - shost_name = '{}-hostgeonode'.format(host.name) + shost_name = f'{host.name}-hostgeonode' try: service = Service.objects.get(name=shost_name) except Service.DoesNotExist: @@ -1927,7 +1910,7 @@ def do_autoconfigure(): service_type=geoserver_type) service.save() - shost_name = '{}-hostgeoserver'.format(host.name) + shost_name = f'{host.name}-hostgeoserver' try: service = Service.objects.get(name=shost_name) except Service.DoesNotExist: diff --git a/geonode/monitoring/service_handlers.py b/geonode/monitoring/service_handlers.py index 099e7e33bd1..2e3c83cd7c5 100644 --- a/geonode/monitoring/service_handlers.py +++ b/geonode/monitoring/service_handlers.py @@ -171,7 +171,7 @@ def __init__(self, service, force_check=False): def setup(self): if not self.service.url: - raise ValueError("Monitoring is not configured to fetch from %s" % self.service.name) + raise ValueError(f"Monitoring is not configured to fetch from {self.service.name}") self.gs_monitor = GeoServerMonitorClient(self.service.url) def _collect(self, since, until, format=None, **kwargs): @@ -197,11 +197,11 @@ def __init__(self, service, force_check=False): def _collect(self, *args, **kwargs): base_url = self.service.url if not base_url: - raise ValueError("Service {} should have url provided".format(self.service.name)) - url = '{}{}'.format(base_url.rstrip('/'), self.PATH) + raise ValueError(f"Service {self.service.name} should have url provided") + url = f"{base_url.rstrip('/')}{self.PATH}" rdata = requests.get(url, timeout=10, verify=False) if rdata.status_code != 200: - raise ValueError("Error response from api: ({}) {}".format(url, rdata)) + raise ValueError(f"Error response from api: ({url}) {rdata}") data = rdata.json()['metrics']['metric'] return data @@ -217,11 +217,11 @@ def __init__(self, service, force_check=False): def _collect(self, since, until, *args, **kwargs): base_url = self.service.url if not base_url: - raise ValueError("Service {} should have url provided".format(self.service.name)) - url = '{}/monitoring/api/beacon/{}/'.format(base_url.rstrip('/'), self.service.service_type.name) + raise ValueError(f"Service {self.service.name} should have url provided") + url = f"{base_url.rstrip('/')}/monitoring/api/beacon/{self.service.service_type.name}/" rdata = requests.get(url, timeout=10, verify=False) if rdata.status_code != 200: - raise ValueError("Error response from api: ({}) {}".format(url, rdata)) + raise ValueError(f"Error response from api: ({url}) {rdata}") data = rdata.json() return data diff --git a/geonode/monitoring/tests/integration.py b/geonode/monitoring/tests/integration.py index 08277d29958..0c4b9b1687f 100644 --- a/geonode/monitoring/tests/integration.py +++ b/geonode/monitoring/tests/integration.py @@ -78,13 +78,7 @@ DB_NAME = settings.DATABASES['default']['NAME'] DB_USER = settings.DATABASES['default']['USER'] DB_PASSWORD = settings.DATABASES['default']['PASSWORD'] -DATASTORE_URL = 'postgis://{}:{}@{}:{}/{}'.format( - DB_USER, - DB_PASSWORD, - DB_HOST, - DB_PORT, - DB_NAME -) +DATASTORE_URL = f'postgis://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}' postgis_db = dj_database_url.parse(DATASTORE_URL, conn_max_age=0) logging.getLogger('south').setLevel(logging.WARNING) @@ -161,7 +155,7 @@ def login_user(self, user): """ if 'django.contrib.sessions' not in settings.INSTALLED_APPS: raise AssertionError("Unable to login without django.contrib.sessions in INSTALLED_APPS") - user.backend = "%s.%s" % ("django.contrib.auth.backends", "ModelBackend") + user.backend = f"{'django.contrib.auth.backends'}.{'ModelBackend'}" # Login self.force_login(user, backend=user.backend) @@ -1923,10 +1917,7 @@ def test_ows_service_enpoints(self): {'name': 'OWS:ALL', 'type_label': 'Any OWS'} ] # url - url = "%s?%s" % ( - reverse('monitoring:api_event_types'), - 'ows_service=true' - ) + url = f"{reverse('monitoring:api_event_types')}?{'ows_service=true'}" # Unauthorized response = self.client.get(url) out = json.loads(ensure_string(response.content)) @@ -1964,10 +1955,7 @@ def test_non_ows_events_enpoints(self): {'name': 'geoserver', 'type_label': 'Geoserver event'} ] # url - url = "%s?%s" % ( - reverse('monitoring:api_event_types'), - 'ows_service=false' - ) + url = f"{reverse('monitoring:api_event_types')}?{'ows_service=false'}" # Unauthorized response = self.client.get(url) out = json.loads(ensure_string(response.content)) diff --git a/geonode/monitoring/utils.py b/geonode/monitoring/utils.py index 41bf9b37204..b49cbfe84c3 100644 --- a/geonode/monitoring/utils.py +++ b/geonode/monitoring/utils.py @@ -110,7 +110,7 @@ def get_href(self, link, format=None): return href.geturl() if format in self.REPORT_FORMATS: href, ext = os.path.splitext(href.geturl()) - return '{}.{}'.format(href, format) + return f'{href}.{format}' return format def get_requests(self, format=None, since=None, until=None): @@ -119,7 +119,7 @@ def get_requests(self, format=None, since=None, until=None): """ from requests.auth import HTTPBasicAuth - rest_url = '{}rest/monitor/requests.html'.format(self.base_url) + rest_url = f'{self.base_url}rest/monitor/requests.html' qargs = {} if since: # since = since.astimezone(utc) @@ -128,7 +128,7 @@ def get_requests(self, format=None, since=None, until=None): # until = until.astimezone(utc) qargs['to'] = until.strftime(GS_FORMAT) if qargs: - rest_url = '{}?{}'.format(rest_url, urlencode(qargs)) + rest_url = f'{rest_url}?{urlencode(qargs)}' log.debug('checking', rest_url) username = settings.OGC_SERVER['default']['USER'] @@ -146,14 +146,14 @@ def get_requests(self, format=None, since=None, until=None): if data: yield data else: - log.warning("Skipping payload for {}".format(href)) + log.warning(f"Skipping payload for {href}") def get_request(self, href, format=format): from requests.auth import HTTPBasicAuth username = settings.OGC_SERVER['default']['USER'] password = settings.OGC_SERVER['default']['PASSWORD'] - log.debug(" href: %s " % href) + log.debug(f" href: {href} ") r = requests.get( href, auth=HTTPBasicAuth(username, password), @@ -187,10 +187,10 @@ def _from_html(self, val): raise ValueError("Cannot convert from html") def to_json(self, data, from_format): - h = getattr(self, '_from_{}'.format(from_format), None) + h = getattr(self, f'_from_{from_format}', None) if not h or not len(data): raise ValueError( - "Cannot convert from {} - no handler".format(from_format)) + f"Cannot convert from {from_format} - no handler") return h(data) @@ -259,7 +259,7 @@ class TypeChecks(object): @classmethod def audit_format(cls, val): if val not in cls.AUDIT_FORMATS: - raise ValueError("Invalid value for audit format: {}".format(val)) + raise ValueError(f"Invalid value for audit format: {val}") return val @staticmethod @@ -268,7 +268,7 @@ def host_type(val): try: return Host.objects.get(name=val) except Host.DoesNotExist: - raise ValueError("Host {} does not exist".format(val)) + raise ValueError(f"Host {val} does not exist") @staticmethod def resource_type(val): @@ -281,7 +281,7 @@ def resource_type(val): rtype, rname = val.split('=') except (ValueError, IndexError,): raise ValueError( - "{} is not valid resource description".format(val)) + f"{val} is not valid resource description") return MonitoredResource.objects.get(type=rtype, name=rname) @staticmethod @@ -289,7 +289,7 @@ def resource_type_type(val): from geonode.monitoring.models import MonitoredResource if val in MonitoredResource._TYPES: return val - raise ValueError("Invalid monitored resource type: {}".format(val)) + raise ValueError(f"Invalid monitored resource type: {val}") @staticmethod def metric_name_type(val): @@ -297,7 +297,7 @@ def metric_name_type(val): try: return Metric.objects.get(name=val) except Metric.DoesNotExist: - raise ValueError("Metric {} doesn't exist".format(val)) + raise ValueError(f"Metric {val} doesn't exist") @staticmethod def service_type(val): @@ -305,7 +305,7 @@ def service_type(val): try: return Service.objects.get(name=val) except Service.DoesNotExist: - raise ValueError("Service {} does not exist".format(val)) + raise ValueError(f"Service {val} does not exist") @staticmethod def service_type_type(val): @@ -313,7 +313,7 @@ def service_type_type(val): try: return ServiceType.objects.get(name=val) except ServiceType.DoesNotExist: - raise ValueError("Service Type {} does not exist".format(val)) + raise ValueError(f"Service Type {val} does not exist") @staticmethod def label_type(val): @@ -325,7 +325,7 @@ def label_type(val): return MetricLabel.objects.get(name=val) except MetricLabel.DoesNotExist: pass - raise ValueError("Invalid label value: {}".format(val)) + raise ValueError(f"Invalid label value: {val}") @staticmethod def user_type(val): @@ -334,7 +334,7 @@ def user_type(val): if MetricLabel.objects.filter(user=val).count(): return val except MetricLabel.DoesNotExist: - raise ValueError("Invalid user value: {}".format(val)) + raise ValueError(f"Invalid user value: {val}") @staticmethod def event_type_type(val): @@ -342,7 +342,7 @@ def event_type_type(val): try: return EventType.objects.get(name=val) except EventType.DoesNotExist: - raise ValueError("Event Type {} doesn't exist".format(val)) + raise ValueError(f"Event Type {val} doesn't exist") @staticmethod def ows_service_type(val): @@ -351,7 +351,7 @@ def ows_service_type(val): elif str(val).lower() in ("false", "0"): return False else: - raise ValueError("Invalid ows_service value {}".format(val)) + raise ValueError(f"Invalid ows_service value {val}") def dump(obj, additional_fields=tuple()): @@ -366,7 +366,7 @@ def dump(obj, additional_fields=tuple()): if isinstance(field, RelatedField): if val is not None: v = val - val = {'class': '{}.{}'.format(val.__class__.__module__, val.__class__.__name__), + val = {'class': f'{val.__class__.__module__}.{val.__class__.__name__}', 'id': val.pk} if hasattr(v, 'name'): val['name'] = v.name @@ -412,14 +412,10 @@ def collect_metric(**options): hexdigest = md5(name).hexdigest() lock_id = f'{name.decode()}-lock-{hexdigest}' _start_time = _end_time = datetime.utcnow().isoformat() - log.info('[{}] Collecting Metrics - started @ {}'.format( - lock_id, - _start_time)) + log.info(f'[{lock_id}] Collecting Metrics - started @ {_start_time}') with AcquireLock(lock_id) as lock: if lock.acquire() is True: - log.info('[{}] Collecting Metrics - [...acquired lock] @ {}'.format( - lock_id, - _start_time)) + log.info(f'[{lock_id}] Collecting Metrics - [...acquired lock] @ {_start_time}') try: oservice = options['service'] if not oservice: @@ -462,17 +458,11 @@ def collect_metric(**options): # notifications_check = now - interval c.emit_notifications() # notifications_check)) _end_time = datetime.utcnow().isoformat() - log.info('[{}] Collecting Metrics - finished @ {}'.format( - lock_id, - _end_time)) + log.info(f'[{lock_id}] Collecting Metrics - finished @ {_end_time}') except Exception as e: - log.info('[{}] Collecting Metrics - errored @ {}'.format( - lock_id, - _end_time)) + log.info(f'[{lock_id}] Collecting Metrics - errored @ {_end_time}') log.exception(e) - log.info('[{}] Collecting Metrics - exit @ {}'.format( - lock_id, - _end_time)) + log.info(f'[{lock_id}] Collecting Metrics - exit @ {_end_time}') return (_start_time, _end_time) diff --git a/geonode/monitoring/views.py b/geonode/monitoring/views.py index e2a3e3947fa..444a79d6fa8 100644 --- a/geonode/monitoring/views.py +++ b/geonode/monitoring/views.py @@ -145,9 +145,9 @@ def _check_type(self, tname): val = d[tname] if not val: return - tcheck = getattr(TypeChecks, '{}_type'.format(tname), None) + tcheck = getattr(TypeChecks, f'{tname}_type', None) if not tcheck: - raise forms.ValidationError("No type check for {}".format(tname)) + raise forms.ValidationError(f"No type check for {tname}") try: return tcheck(val) except (Exception,) as err: @@ -522,7 +522,7 @@ def get(self, request, *args, **kwargs): ex = exposes[service]() except KeyError: return json_response( - errors={'exposed': 'No service for {}'.format(service)}, status=404) + errors={'exposed': f'No service for {service}'}, status=404) out = {'data': ex.expose(), 'timestamp': datetime.utcnow().replace(tzinfo=pytz.utc)} return json_response(out) @@ -565,7 +565,7 @@ def _get_clean_model(self, cls, name): try: return cls.objects.get(name=val) except cls.DoesNotExist: - raise forms.ValidationError("Invalid {}: {}".format(name, val)) + raise forms.ValidationError(f"Invalid {name}: {val}") def clean_metric(self): return self._get_clean_model(Metric, 'metric') @@ -587,11 +587,11 @@ def clean_resource(self): vtype, vname = val.split('=') except IndexError: raise forms.ValidationError( - "Invalid resource name: {}".format(val)) + f"Invalid resource name: {val}") try: return MonitoredResource.objects.get(name=vname, type=vtype) except MonitoredResource.DoesNotExist: - raise forms.ValidationError("Invalid resource: {}".format(val)) + raise forms.ValidationError(f"Invalid resource: {val}") class UserNotificationConfigView(View): diff --git a/geonode/notifications_helper.py b/geonode/notifications_helper.py index e056bc01114..0320ed46a41 100644 --- a/geonode/notifications_helper.py +++ b/geonode/notifications_helper.py @@ -17,9 +17,7 @@ # along with this program. If not, see . # ######################################################################### - import logging -import traceback from importlib import import_module from django.apps import AppConfig @@ -130,11 +128,9 @@ def get_notification_recipients(notice_type_label, exclude_user=None, resource=N if user.pk == resource.owner.pk and \ not notice_type_label.split("_")[-1] in ("updated", "rated", "comment", "approved", "published"): exclude_users_ids.append(user.id) - except Exception: + except Exception as e: # fallback which wont send mails - tb = traceback.format_exc() - logger.error(tb) - logger.exception("Could not send notifications.") + logger.exception(f"Could not send notifications: {e}") return [] return profiles.exclude(id__in=exclude_users_ids) diff --git a/geonode/people/adapters.py b/geonode/people/adapters.py index 676b2a65aa5..4bb8cf0428b 100644 --- a/geonode/people/adapters.py +++ b/geonode/people/adapters.py @@ -88,7 +88,7 @@ def update_profile(sociallogin): for field in profile_fields: try: extractor_method = getattr( - extractor, "extract_{}".format(field)) + extractor, f"extract_{field}") value = extractor_method(sociallogin.account.extra_data) if not user_field(user, field): user_field(user, field, value) diff --git a/geonode/people/management/commands/setupsociallogins.py b/geonode/people/management/commands/setupsociallogins.py index e46774705da..519e5116b22 100644 --- a/geonode/people/management/commands/setupsociallogins.py +++ b/geonode/people/management/commands/setupsociallogins.py @@ -18,19 +18,19 @@ class Command(BaseCommand): @staticmethod def _get_client_id_arg(provider): - return "{}-client-id".format(provider) + return f"{provider}-client-id" @staticmethod def _get_client_secret_arg(provider): - return "{}-secret-key".format(provider) + return f"{provider}-secret-key" @staticmethod def _get_client_id_env(provider): - return "{}_OAUTH2_CLIENT_ID".format(provider.upper()) + return f"{provider.upper()}_OAUTH2_CLIENT_ID" @staticmethod def _get_client_secret_env(provider): - return "{}_OAUTH2_SECRET_KEY".format(provider.upper()) + return f"{provider.upper()}_OAUTH2_SECRET_KEY" @staticmethod def get_social_providers(): @@ -47,24 +47,18 @@ def add_arguments(self, parser): for provider_name, provider_id in self.get_social_providers(): client_id_arg = self._get_client_id_arg(provider_name) parser.add_argument( - "--{}".format(client_id_arg), + f"--{client_id_arg}", help=( - "Specify the client id for {}. You may also specify " - "the {} environment variable instead".format( - provider_name, - self._get_client_id_env(provider_name) - ) + f"Specify the client id for {provider_name}. You may also specify " + f"the {self._get_client_id_env(provider_name)} environment variable instead" ) ) client_secret_arg = self._get_client_secret_arg(provider_name) parser.add_argument( - "--{}".format(client_secret_arg), + f"--{client_secret_arg}", help=( - "Specify the secret key for {}. You may also specify " - "the {} environment variable instead".format( - provider_name, - self._get_client_secret_env(provider_name) - ) + f"Specify the secret key for {provider_name}. You may also specify " + f"the {self._get_client_secret_env(provider_name)} environment variable instead" ) ) @@ -86,13 +80,13 @@ def handle(self, *args, **options): ) if all((client_id, client_secret)): self.stdout.write( - "Configuring provider {}...".format(provider_name)) + f"Configuring provider {provider_name}...") self._handle_provider( provider_name, provider_id, client_id, client_secret) else: self.stdout.write( - "Provider {} not all params were specified, " - "skipping...".format(provider_name) + f"Provider {provider_name} not all params were specified, " + "skipping..." ) else: self.stdout.write( @@ -115,4 +109,4 @@ def _handle_provider(self, name, id_, client_id, secret_key): provider.save() else: self.stdout.write( - "Provider {} already exists, skipping...".format(name)) + f"Provider {name} already exists, skipping...") diff --git a/geonode/people/models.py b/geonode/people/models.py index 6da32596f8d..db5698d5bc3 100644 --- a/geonode/people/models.py +++ b/geonode/people/models.py @@ -139,7 +139,7 @@ def get_absolute_url(self): return reverse('profile_detail', args=[self.username, ]) def __str__(self): - return "{0}".format(self.username) + return str(self.username) @staticmethod def class_name(value): @@ -170,20 +170,18 @@ def keyword_list(self): @property def name_long(self): if self.first_name and self.last_name: - return '%s %s (%s)' % (self.first_name, - self.last_name, self.username) + return f'{self.first_name} {self.last_name} ({self.username})' elif (not self.first_name) and self.last_name: - return '%s (%s)' % (self.last_name, self.username) + return f'{self.last_name} ({self.username})' elif self.first_name and (not self.last_name): - return '%s (%s)' % (self.first_name, self.username) + return f'{self.first_name} ({self.username})' else: return self.username @property def full_name_or_nick(self): if self.first_name and self.last_name: - return '%s %s' % (self.first_name, - self.last_name) + return f'{self.first_name} {self.last_name}' else: return self.username diff --git a/geonode/people/signals.py b/geonode/people/signals.py index 325d9f4c83e..4c90401bb91 100644 --- a/geonode/people/signals.py +++ b/geonode/people/signals.py @@ -108,7 +108,7 @@ def update_user_email_addresses(sender, **kwargs): EmailAddress.objects.add_email( request=None, user=user, email=sociallogin_email, confirm=False) except IntegrityError: - logging.exception(msg="Could not add email address {} to user {}".format(sociallogin_email, user)) + logging.exception(msg=f"Could not add email address {sociallogin_email} to user {user}") def notify_admins_new_signup(sender, **kwargs): diff --git a/geonode/people/views.py b/geonode/people/views.py index bcadd605c4d..6b198d5904e 100644 --- a/geonode/people/views.py +++ b/geonode/people/views.py @@ -71,7 +71,7 @@ def profile_edit(request, username=None): form = ProfileForm(request.POST, request.FILES, instance=profile) if form.is_valid(): form.save() - messages.success(request, ("Profile %s updated." % username)) + messages.success(request, (f"Profile {username} updated.")) return redirect( reverse( 'profile_detail', diff --git a/geonode/proxy/tests.py b/geonode/proxy/tests.py index 4ddee32fd9a..4e62fdf27b1 100644 --- a/geonode/proxy/tests.py +++ b/geonode/proxy/tests.py @@ -43,7 +43,7 @@ from geonode.base.populate_test_data import create_models TEST_DOMAIN = '.github.com' -TEST_URL = 'https://help%s/' % TEST_DOMAIN +TEST_URL = f'https://help{TEST_DOMAIN}/' class ProxyTest(GeoNodeBaseTestSupport): @@ -59,8 +59,7 @@ def setUp(self): @override_settings(DEBUG=True, PROXY_ALLOWED_HOSTS=()) def test_validate_host_disabled_in_debug(self): """If PROXY_ALLOWED_HOSTS is empty and DEBUG is True, all hosts pass the proxy.""" - response = self.client.get('%s?url=%s' % - (self.proxy_url, self.url)) + response = self.client.get(f'{self.proxy_url}?url={self.url}') # 404 - NOT FOUND if response.status_code != 404: self.assertTrue(response.status_code in (200, 301)) @@ -68,8 +67,7 @@ def test_validate_host_disabled_in_debug(self): @override_settings(DEBUG=False, PROXY_ALLOWED_HOSTS=()) def test_validate_host_disabled_not_in_debug(self): """If PROXY_ALLOWED_HOSTS is empty and DEBUG is False requests should return 403.""" - response = self.client.get('%s?url=%s' % - (self.proxy_url, self.url)) + response = self.client.get(f'{self.proxy_url}?url={self.url}') # 404 - NOT FOUND if response.status_code != 404: self.assertEqual(response.status_code, 403) @@ -78,8 +76,7 @@ def test_validate_host_disabled_not_in_debug(self): @override_settings(DEBUG=False, PROXY_ALLOWED_HOSTS=(TEST_DOMAIN,)) def test_proxy_allowed_host(self): """If PROXY_ALLOWED_HOSTS is empty and DEBUG is False requests should return 403.""" - response = self.client.get('%s?url=%s' % - (self.proxy_url, self.url)) + response = self.client.get(f'{self.proxy_url}?url={self.url}') # 404 - NOT FOUND if response.status_code != 404: self.assertTrue(response.status_code in (200, 301)) @@ -98,7 +95,7 @@ def test_validate_remote_services_hosts(self): method=INDEXED, base_url='http://bogus.pocus.com/ows') response = self.client.get( - '%s?url=%s' % (self.proxy_url, 'http://bogus.pocus.com/ows/wms?request=GetCapabilities')) + f'{self.proxy_url}?url=http://bogus.pocus.com/ows/wms?request=GetCapabilities') # 200 - FOUND self.assertTrue(response.status_code in (200, 301)) @@ -119,7 +116,7 @@ class Response(object): geonode.proxy.views.http_client.request = request_mock url = "http://example.org/test/test/../../index.html" - self.client.get('%s?url=%s' % (self.proxy_url, url)) + self.client.get(f'{self.proxy_url}?url={url}') assert request_mock.call_args[0][0] == 'http://example.org/index.html' diff --git a/geonode/proxy/views.py b/geonode/proxy/views.py index 65a035d576f..a21af3466ab 100644 --- a/geonode/proxy/views.py +++ b/geonode/proxy/views.py @@ -167,8 +167,7 @@ def proxy(request, url=None, response_callback=None, if request.method == "GET" and access_token and 'access_token' not in _url: query_separator = '&' if '?' in _url else '?' - _url = ('%s%saccess_token=%s' % - (_url, query_separator, access_token)) + _url = f'{_url}{query_separator}access_token={access_token}' _data = request.body.decode('utf-8') @@ -176,10 +175,10 @@ def proxy(request, url=None, response_callback=None, if check_ogc_backend(geoserver.BACKEND_PACKAGE): from geonode.geoserver.helpers import ogc_server_settings _url = _url.replace( - '%s%s' % (settings.SITEURL, 'geoserver'), + f'{settings.SITEURL}geoserver', ogc_server_settings.LOCATION.rstrip('/')) _data = _data.replace( - '%s%s' % (settings.SITEURL, 'geoserver'), + f'{settings.SITEURL}geoserver', ogc_server_settings.LOCATION.rstrip('/')) response, content = http_client.request( @@ -236,8 +235,8 @@ def proxy(request, url=None, response_callback=None, else: # If we get a redirect, let's add a useful message. if status and status in (301, 302, 303, 307): - _response = HttpResponse(('This proxy does not support redirects. The server in "%s" ' - 'asked for a redirect to "%s"' % (url, response.getheader('Location'))), + _response = HttpResponse((f"This proxy does not support redirects. The server in '{url}' " + f"asked for a redirect to '{response.getheader('Location')}'"), status=status, content_type=content_type ) @@ -397,7 +396,7 @@ def download(request, resourceid, sender=Layer): content=open(target_file, mode='rb'), status=200, content_type="application/zip") - response['Content-Disposition'] = 'attachment; filename="%s"' % target_file_name + response['Content-Disposition'] = f'attachment; filename="{target_file_name}"' return response except NotImplementedError: traceback.print_exc() @@ -434,7 +433,7 @@ def get(self, request): headers, access_token = get_headers(request, _url, _raw_url) if access_token: _j = '&' if _url.query else '?' - _raw_url = _j.join([_raw_url, 'access_token={}'.format(access_token)]) + _raw_url = _j.join([_raw_url, f'access_token={access_token}']) data.append({'url': _raw_url, 'type': 'OGC:WMS'}) # WCS @@ -443,7 +442,7 @@ def get(self, request): headers, access_token = get_headers(request, _url, _raw_url) if access_token: _j = '&' if _url.query else '?' - _raw_url = _j.join([_raw_url, 'access_token={}'.format(access_token)]) + _raw_url = _j.join([_raw_url, f'access_token={access_token}']) data.append({'url': _raw_url, 'type': 'OGC:WCS'}) # WFS @@ -452,7 +451,7 @@ def get(self, request): headers, access_token = get_headers(request, _url, _raw_url) if access_token: _j = '&' if _url.query else '?' - _raw_url = _j.join([_raw_url, 'access_token={}'.format(access_token)]) + _raw_url = _j.join([_raw_url, f'access_token={access_token}']) data.append({'url': _raw_url, 'type': 'OGC:WFS'}) # catalogue from configuration @@ -463,7 +462,7 @@ def get(self, request): headers, access_token = get_headers(request, _url, _raw_url) if access_token: _j = '&' if _url.query else '?' - _raw_url = _j.join([_raw_url, 'access_token={}'.format(access_token)]) + _raw_url = _j.join([_raw_url, f'access_token={access_token}']) data.append({'url': _raw_url, 'type': 'OGC:CSW'}) # main site url diff --git a/geonode/security/management/__init__.py b/geonode/security/management/__init__.py index 906e5b3f967..edbbc7b7c84 100644 --- a/geonode/security/management/__init__.py +++ b/geonode/security/management/__init__.py @@ -16,4 +16,4 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -######################################################################### \ No newline at end of file +######################################################################### diff --git a/geonode/security/management/commands/__init__.py b/geonode/security/management/commands/__init__.py index 906e5b3f967..edbbc7b7c84 100644 --- a/geonode/security/management/commands/__init__.py +++ b/geonode/security/management/commands/__init__.py @@ -16,4 +16,4 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -######################################################################### \ No newline at end of file +######################################################################### diff --git a/geonode/security/management/commands/sync_security_rules.py b/geonode/security/management/commands/sync_security_rules.py index cdeab64bbec..644e698563a 100644 --- a/geonode/security/management/commands/sync_security_rules.py +++ b/geonode/security/management/commands/sync_security_rules.py @@ -29,4 +29,3 @@ class Command(BaseCommand): def handle(self, *args, **options): sync_resources_with_guardian() - diff --git a/geonode/security/middleware.py b/geonode/security/middleware.py index c38d99208c9..503eba7a6ca 100644 --- a/geonode/security/middleware.py +++ b/geonode/security/middleware.py @@ -101,7 +101,7 @@ def process_request(self, request): if not any(path.match(request.path) for path in white_list): return HttpResponseRedirect( - "{login_path}?next={request_path}".format(login_path=self.redirect_to, request_path=request.path) + f"{self.redirect_to}?next={request.path}" ) @@ -143,6 +143,4 @@ def do_logout(self, request): if not any(path.match(request.path) for path in white_list): return HttpResponseRedirect( - '{login_path}?next={request_path}'.format( - login_path=self.redirect_to, - request_path=request.path)) + f'{self.redirect_to}?next={request.path}') diff --git a/geonode/security/oauth2_grant_types.py b/geonode/security/oauth2_grant_types.py index affded7fb07..ff3492a9956 100644 --- a/geonode/security/oauth2_grant_types.py +++ b/geonode/security/oauth2_grant_types.py @@ -361,7 +361,7 @@ def validate_authorization_request(self, request): duplicate_params = request.duplicate_params if param in duplicate_params: raise errors.InvalidRequestFatalError( - description='Duplicate %s parameter.' % param, + description=f'Duplicate {param} parameter.', request=request) except ValueError: log.exception( @@ -470,7 +470,7 @@ def validate_token_request(self, request): for param in ('client_id', 'grant_type', 'redirect_uri'): if param in request.duplicate_params: - raise errors.InvalidRequestError(description='Duplicate %s parameter.' % param, + raise errors.InvalidRequestError(description=f'Duplicate {param} parameter.', request=request) if self.request_validator.client_authentication_required(request): @@ -520,7 +520,7 @@ def validate_token_request(self, request): if challenge_method not in self._code_challenge_methods: raise errors.ServerError( - description="code_challenge_method {} is not supported.".format(challenge_method), + description=f"code_challenge_method {challenge_method} is not supported.", request=request ) @@ -565,7 +565,7 @@ def validate_token_request(self, request): def validate_code_challenge(self, challenge, challenge_method, verifier): if challenge_method in self._code_challenge_methods: return self._code_challenge_methods[challenge_method](verifier, challenge) - raise NotImplementedError('Unknown challenge_method %s' % challenge_method) + raise NotImplementedError(f'Unknown challenge_method {challenge_method}') class OpenIDAuthorizationCodeGrant(OpenIDGrantTypeBase): diff --git a/geonode/security/tests.py b/geonode/security/tests.py index 2f7976004c1..575908aaa87 100644 --- a/geonode/security/tests.py +++ b/geonode/security/tests.py @@ -153,7 +153,7 @@ def test_login_middleware(self): response = middleware.process_request(request) self.assertIsNone( response, - msg="Middleware activated for white listed path: {0}".format(path)) + msg=f"Middleware activated for white listed path: {path}") self.client.login(username='admin', password='admin') admin = get_user_model().objects.get(username='admin') @@ -196,7 +196,7 @@ def test_login_middleware_with_basic_auth(self): response = middleware.process_request(request) self.assertIsNone( response, - msg="Middleware activated for white listed path: {0}".format(black_listed_url)) + msg=f"Middleware activated for white listed path: {black_listed_url}") # Basic authorized request to black listed URL should be allowed request.path = black_listed_url @@ -204,7 +204,7 @@ def test_login_middleware_with_basic_auth(self): response = middleware.process_request(request) self.assertIsNone( response, - msg="Middleware activated for white listed path: {0}".format(black_listed_url)) + msg=f"Middleware activated for white listed path: {black_listed_url}") @on_ogc_backend(geoserver.BACKEND_PACKAGE) @dump_func_name @@ -377,11 +377,11 @@ def test_set_bulk_permissions(self): if check_ogc_backend(geoserver.BACKEND_PACKAGE): # Check GeoFence Rules have been correctly created geofence_rules_count = get_geofence_rules_count() - _log("1. geofence_rules_count: %s " % geofence_rules_count) + _log(f"1. geofence_rules_count: {geofence_rules_count} ") self.assertEqual(geofence_rules_count, 14) set_geofence_all(test_perm_layer) geofence_rules_count = get_geofence_rules_count() - _log("2. geofence_rules_count: %s " % geofence_rules_count) + _log(f"2. geofence_rules_count: {geofence_rules_count} ") self.assertEqual(geofence_rules_count, 15) self.client.logout() @@ -392,17 +392,17 @@ def test_set_bulk_permissions(self): self.assertEqual(len(self.deserialize(resp)['objects']), 7) perms = get_users_with_perms(test_perm_layer) - _log("3. perms: %s " % perms) + _log(f"3. perms: {perms} ") sync_geofence_with_guardian(test_perm_layer, perms, user='bobby') # Check GeoFence Rules have been correctly created geofence_rules_count = get_geofence_rules_count() - _log("4. geofence_rules_count: %s " % geofence_rules_count) + _log(f"4. geofence_rules_count: {geofence_rules_count} ") self.assertEqual(geofence_rules_count, 15) # Validate maximum priority geofence_rules_highest_priority = get_highest_priority() - _log("5. geofence_rules_highest_priority: %s " % geofence_rules_highest_priority) + _log(f"5. geofence_rules_highest_priority: {geofence_rules_highest_priority} ") self.assertTrue(geofence_rules_highest_priority > 0) url = settings.OGC_SERVER['default']['LOCATION'] @@ -411,7 +411,7 @@ def test_set_bulk_permissions(self): import requests from requests.auth import HTTPBasicAuth - r = requests.get(url + 'gwc/rest/seed/%s.json' % test_perm_layer.alternate, + r = requests.get(url + f'gwc/rest/seed/{test_perm_layer.alternate}.json', auth=HTTPBasicAuth(user, passwd)) self.assertEqual(r.status_code, 400) @@ -501,20 +501,20 @@ def test_perm_specs_synchronization(self): perm_spec = {'users': {'AnonymousUser': []}, 'groups': []} layer.set_permissions(perm_spec) geofence_rules_count = get_geofence_rules_count() - _log("1. geofence_rules_count: %s " % geofence_rules_count) + _log(f"1. geofence_rules_count: {geofence_rules_count} ") self.assertEqual(geofence_rules_count, 5) perm_spec = { "users": {"admin": ["view_resourcebase"]}, "groups": []} layer.set_permissions(perm_spec) geofence_rules_count = get_geofence_rules_count() - _log("2. geofence_rules_count: %s " % geofence_rules_count) + _log(f"2. geofence_rules_count: {geofence_rules_count} ") self.assertEqual(geofence_rules_count, 7) perm_spec = {'users': {"admin": ['change_layer_data']}, 'groups': []} layer.set_permissions(perm_spec) geofence_rules_count = get_geofence_rules_count() - _log("3. geofence_rules_count: %s " % geofence_rules_count) + _log(f"3. geofence_rules_count: {geofence_rules_count} ") self.assertEqual(geofence_rules_count, 7) # FULL WFS-T @@ -600,13 +600,13 @@ def test_perm_specs_synchronization(self): perm_spec = {'users': {}, 'groups': {'bar': ['view_resourcebase']}} layer.set_permissions(perm_spec) geofence_rules_count = get_geofence_rules_count() - _log("4. geofence_rules_count: %s " % geofence_rules_count) + _log(f"4. geofence_rules_count: {geofence_rules_count} ") self.assertEqual(geofence_rules_count, 7) perm_spec = {'users': {}, 'groups': {'bar': ['change_resourcebase']}} layer.set_permissions(perm_spec) geofence_rules_count = get_geofence_rules_count() - _log("5. geofence_rules_count: %s " % geofence_rules_count) + _log(f"5. geofence_rules_count: {geofence_rules_count} ") self.assertEqual(geofence_rules_count, 5) # Testing GeoLimits @@ -884,7 +884,7 @@ def test_layer_upload_with_time(self): e = dlxml.fromstring(wms_capabilities) for atype in e.findall( - "./[wms:Name='%s']/wms:Dimension[@name='time']" % (saved_layer.alternate), namespaces): + f"./[wms:Name='{saved_layer.alternate}']/wms:Dimension[@name='time']", namespaces): dim_name = atype.get('name') if dim_name: dim_name = str(dim_name).lower() @@ -1009,7 +1009,7 @@ def test_layer_permissions(self): check_layer(layer) geofence_rules_count = get_geofence_rules_count() - _log("0. geofence_rules_count: %s " % geofence_rules_count) + _log(f"0. geofence_rules_count: {geofence_rules_count} ") self.assertTrue(geofence_rules_count >= 2) # Set the layer private for not authenticated users @@ -1036,7 +1036,7 @@ def test_layer_permissions(self): # the layer must be not accessible (response is xml) request = Request(url) basic_auth = base64.b64encode(b'bobby:bob') - request.add_header("Authorization", "Basic {}".format(basic_auth.decode("utf-8"))) + request.add_header("Authorization", f"Basic {basic_auth.decode('utf-8')}") response = urlopen(request) _content_type = response.getheader('Content-Type').lower() self.assertEqual( @@ -1056,7 +1056,7 @@ def test_layer_permissions(self): layer.set_permissions(perm_spec) request = Request(url) basic_auth = base64.b64encode(b'bobby:bob') - request.add_header("Authorization", "Basic {}".format(basic_auth.decode("utf-8"))) + request.add_header("Authorization", f"Basic {basic_auth.decode('utf-8')}") response = urlopen(request) _content_type = response.getheader('Content-Type').lower() self.assertEqual( @@ -1337,7 +1337,7 @@ def test_not_superuser_permissions(self): if check_ogc_backend(geoserver.BACKEND_PACKAGE): # Check GeoFence Rules have been correctly created geofence_rules_count = get_geofence_rules_count() - _log("1. geofence_rules_count: %s " % geofence_rules_count) + _log(f"1. geofence_rules_count: {geofence_rules_count} ") self.assertEqual(geofence_rules_count, 14) self.assertTrue(self.client.login(username='bobby', password='bob')) @@ -1407,12 +1407,12 @@ def test_not_superuser_permissions(self): if check_ogc_backend(geoserver.BACKEND_PACKAGE): perms = get_users_with_perms(layer) - _log("2. perms: %s " % perms) + _log(f"2. perms: {perms} ") sync_geofence_with_guardian(layer, perms, user=bob, group=anonymous_group) # Check GeoFence Rules have been correctly created geofence_rules_count = get_geofence_rules_count() - _log("3. geofence_rules_count: %s " % geofence_rules_count) + _log(f"3. geofence_rules_count: {geofence_rules_count} ") self.assertEqual(geofence_rules_count, 14) # 5. change_resourcebase_permissions @@ -1582,7 +1582,7 @@ def test_save_and_delete_signals(self): if sld: if isinstance(sld, bytes): sld = sld.decode().strip('\n') - _log("sld. ------------ %s " % sld) + _log(f"sld. ------------ {sld} ") set_layer_style(test_perm_layer, test_perm_layer.alternate, sld) create_gs_thumbnail(test_perm_layer, overwrite=True) self.assertIsNotNone(test_perm_layer.get_thumbnail_url()) @@ -1615,7 +1615,7 @@ def setUp(self): with contextlib.ExitStack() as stack: input_files = [ stack.enter_context(_fp) for _fp in input_files] - files = dict(zip(['{}_file'.format(s) for s in suffixes], input_files)) + files = dict(zip([f'{s}_file' for s in suffixes], input_files)) files['base_file'] = files.pop('shp_file') files['permissions'] = '{}' files['charset'] = 'utf-8' @@ -1634,7 +1634,7 @@ def _get_input_paths(self): base_name = 'single_point' suffixes = 'shp shx dbf prj'.split(' ') base_path = gisdata.GOOD_DATA - paths = [os.path.join(base_path, 'vector', '{}.{}'.format(base_name, suffix)) for suffix in suffixes] + paths = [os.path.join(base_path, 'vector', f'{base_name}.{suffix}') for suffix in suffixes] return paths, suffixes, @dump_func_name diff --git a/geonode/security/utils.py b/geonode/security/utils.py index 11240f74eb2..4ed4fdc6d6e 100644 --- a/geonode/security/utils.py +++ b/geonode/security/utils.py @@ -172,7 +172,7 @@ def get_geofence_rules(page=0, entries=1, count=False): curl -X GET -u admin:geoserver \ http://:/geoserver/rest/geofence/rules.json?page={page}&entries={entries} """ - _url = url + 'rest/geofence/rules.json?page={}&entries={}'.format(page, entries) + _url = f'{url}rest/geofence/rules.json?page={page}&entries={entries}' r = requests.get(_url, headers=_headers, auth=HTTPBasicAuth(user, passwd), @@ -247,12 +247,12 @@ def purge_geofence_all(): headers=headers, auth=HTTPBasicAuth(user, passwd)) if (r.status_code < 200 or r.status_code > 201): - msg = "Could not DELETE GeoServer Rule id[%s]" % rule['id'] + msg = f"Could not DELETE GeoServer Rule id[{rule['id']}]" e = Exception(msg) - logger.debug("Response [{}] : {}".format(r.status_code, r.text)) + logger.debug(f"Response [{r.status_code}] : {r.text}") raise e except Exception: - logger.debug("Response [{}] : {}".format(r.status_code, r.text)) + logger.debug(f"Response [{r.status_code}] : {r.text}") except Exception: tb = traceback.format_exc() logger.debug(tb) @@ -273,8 +273,7 @@ def purge_geofence_layer_rules(resource): workspace = get_layer_workspace(resource.layer) try: r = requests.get( - "{}rest/geofence/rules.json?workspace={}&layer={}".format( - url, workspace, resource.layer.name), + f"{url}rest/geofence/rules.json?workspace={workspace}&layer={resource.layer.name}", headers=headers, auth=HTTPBasicAuth(user, passwd), timeout=10, @@ -299,7 +298,7 @@ def purge_geofence_layer_rules(resource): msg = "Could not DELETE GeoServer Rule for Layer " msg = msg + str(resource.layer.name) e = Exception(msg) - logger.debug("Response [{}] : {}".format(r.status_code, r.text)) + logger.debug(f"Response [{r.status_code}] : {r.text}") raise e except Exception as e: logger.exception(e) @@ -342,11 +341,11 @@ def toggle_layer_cache(layer_name, enable=True, filters=None, formats=None): curl -v -u admin:geoserver -XGET \ "http://:/geoserver/gwc/rest/layers/geonode:tasmania_roads.xml" """ - r = requests.get(url + 'gwc/rest/layers/{}.xml'.format(layer_name), + r = requests.get(f'{url}gwc/rest/layers/{layer_name}.xml', auth=HTTPBasicAuth(user, passwd)) if (r.status_code < 200 or r.status_code > 201): - logger.debug("Could not Retrieve {} Cache.".format(layer_name)) + logger.debug(f"Could not Retrieve {layer_name} Cache.") return False try: xml_content = r.content @@ -403,12 +402,12 @@ def toggle_layer_cache(layer_name, enable=True, filters=None, formats=None): """ headers = {'Content-type': 'text/xml'} payload = ET.tostring(tree) - r = requests.post(url + 'gwc/rest/layers/{}.xml'.format(layer_name), + r = requests.post(f'{url}gwc/rest/layers/{layer_name}.xml', headers=headers, data=payload, auth=HTTPBasicAuth(user, passwd)) if (r.status_code < 200 or r.status_code > 201): - logger.debug("Could not Update {} Cache.".format(layer_name)) + logger.debug(f"Could not Update {layer_name} Cache.") return False except Exception: tb = traceback.format_exc() @@ -433,11 +432,11 @@ def delete_layer_cache(layer_name): curl -v -u admin:geoserver -XDELETE \ "http://:/geoserver/gwc/rest/layers/geonode:tasmania_roads.xml" """ - r = requests.delete(url + 'gwc/rest/layers/{}.xml'.format(layer_name), + r = requests.delete(f'{url}gwc/rest/layers/{layer_name}.xml', auth=HTTPBasicAuth(user, passwd)) if (r.status_code < 200 or r.status_code > 201): - logger.debug("Could not Delete {} Cache.".format(layer_name)) + logger.debug(f"Could not Delete {layer_name} Cache.") return False return True except Exception: @@ -462,14 +461,14 @@ def set_geowebcache_invalidate_cache(layer_alternate, cat=None): http://localhost:8080/geoserver/gwc/rest/masstruncate """ headers = {'Content-type': 'text/xml'} - payload = "%s" % layer_alternate + payload = f"{layer_alternate}" r = requests.post( url + 'gwc/rest/masstruncate', headers=headers, data=payload, auth=HTTPBasicAuth(user, passwd)) if (r.status_code < 200 or r.status_code > 201): - logger.debug("Could not Truncate GWC Cache for Layer '%s'." % layer_alternate) + logger.debug(f"Could not Truncate GWC Cache for Layer '{layer_alternate}'.") except Exception: tb = traceback.format_exc() logger.debug(tb) @@ -489,9 +488,9 @@ def set_geofence_all(instance): """ resource = instance.get_self_resource() - logger.debug("Inside set_geofence_all for instance {}".format(instance)) + logger.debug(f"Inside set_geofence_all for instance {instance}") workspace = get_layer_workspace(resource.layer) - logger.debug("going to work in workspace {!r}".format(workspace)) + logger.debug(f"going to work in workspace {workspace}") try: url = settings.OGC_SERVER['default']['LOCATION'] user = settings.OGC_SERVER['default']['USER'] @@ -518,9 +517,9 @@ def set_geofence_all(instance): ) if response.status_code not in (200, 201): logger.debug( - "Response {!r} : {}".format(response.status_code, response.text)) + f"Response {response.status_code} : {response.text}") raise RuntimeError("Could not ADD GeoServer ANONYMOUS Rule " - "for Layer {}".format(resource.layer.name)) + f"for Layer {resource.layer.name}") except Exception: tb = traceback.format_exc() logger.debug(tb) @@ -593,7 +592,6 @@ def sync_geofence_with_guardian(layer, perms, user=None, group=None, group_perms _disable_layer_cache = anonymous_geolimits.count() > 0 if _disable_layer_cache: - # delete_layer_cache('{}:{}'.format(_layer_workspace, _layer_name)) filters = None formats = None # Re-order dictionary @@ -615,12 +613,12 @@ def sync_geofence_with_guardian(layer, perms, user=None, group=None, group_perms 'image/gif', 'image/png8' ] - toggle_layer_cache('{}:{}'.format(_layer_workspace, _layer_name), enable=True, filters=filters, formats=formats) + toggle_layer_cache(f'{_layer_workspace}:{_layer_name}', enable=True, filters=filters, formats=formats) for service, allowed in gf_services.items(): if layer and layer.name and allowed: if _user: - logger.debug("Adding 'user' to geofence the rule: %s %s %s" % (layer, service, _user)) + logger.debug(f"Adding 'user' to geofence the rule: {layer} {service} {_user}") _wkt = None if users_geolimits and users_geolimits.count(): _wkt = users_geolimits.last().wkt @@ -630,7 +628,7 @@ def sync_geofence_with_guardian(layer, perms, user=None, group=None, group_perms service, request=request, user=_user, allow=enabled) _update_geofence_rule(layer, _layer_name, _layer_workspace, service, user=_user, geo_limit=_wkt) elif not _group: - logger.debug("Adding to geofence the rule: %s %s *" % (layer, service)) + logger.debug(f"Adding to geofence the rule: {layer} {service} *") _wkt = None if anonymous_geolimits and anonymous_geolimits.count(): _wkt = anonymous_geolimits.last().wkt @@ -644,7 +642,7 @@ def sync_geofence_with_guardian(layer, perms, user=None, group=None, group_perms _update_geofence_rule(layer, _layer_name, _layer_workspace, service, request=request, user=_user, allow=enabled) if _group: - logger.debug("Adding 'group' to geofence the rule: %s %s %s" % (layer, service, _group)) + logger.debug(f"Adding 'group' to geofence the rule: {layer} {service} {_group}") _wkt = None if groups_geolimits and groups_geolimits.count(): _wkt = groups_geolimits.last().wkt @@ -736,7 +734,7 @@ def _get_geofence_payload(layer, layer_name, workspace, access, user=None, group priority_el.text = str(highest_priority if highest_priority >= 0 else 0) if group is not None: role_el = etree.SubElement(root_el, "roleName") - role_el.text = "ROLE_{}".format(group.upper()) + role_el.text = f"ROLE_{group.upper()}" workspace_el = etree.SubElement(root_el, "workspace") workspace_el.text = workspace layer_el = etree.SubElement(root_el, "layer") @@ -776,10 +774,9 @@ def _update_geofence_rule(layer, layer_name, workspace, request=request, geo_limit=geo_limit ) - logger.debug("request data: {}".format(payload)) + logger.debug(f"request data: {payload}") response = requests.post( - "{base_url}rest/geofence/rules".format( - base_url=settings.OGC_SERVER['default']['LOCATION']), + f"{settings.OGC_SERVER['default']['LOCATION']}rest/geofence/rules", data=payload, headers={ 'Content-type': 'application/xml' @@ -789,10 +786,10 @@ def _update_geofence_rule(layer, layer_name, workspace, password=settings.OGC_SERVER['default']['PASSWORD'] ) ) - logger.debug("response status_code: {}".format(response.status_code)) + logger.debug(f"response status_code: {response.status_code}") if response.status_code not in (200, 201): - msg = ("Could not ADD GeoServer User {!r} Rule for " - "Layer {!r}: '{!r}'".format(user, layer, response.text)) + msg = (f"Could not ADD GeoServer User {user} Rule for " + f"Layer {layer}: '{response.text}'") if 'Duplicate Rule' in response.text: logger.debug(msg) else: @@ -820,7 +817,6 @@ def sync_resources_with_guardian(resource=None): purge_geofence_layer_rules(r) layer = Layer.objects.get(id=r.id) perm_spec = layer.get_all_level_info() - logger.debug(" %s --------------------------- %s " % (layer, perm_spec)) # All the other users if 'users' in perm_spec: for user, perms in perm_spec['users'].items(): @@ -839,7 +835,7 @@ def sync_resources_with_guardian(resource=None): r.clear_dirty_state() except Exception as e: logger.exception(e) - logger.warn("!WARNING! - Failure Synching-up Security Rules for Resource [%s]" % (r)) + logger.warn(f"!WARNING! - Failure Synching-up Security Rules for Resource [{r}]") def spec_perms_is_empty(perm_spec): diff --git a/geonode/security/views.py b/geonode/security/views.py index 8d8cc274cd8..8716882aecf 100644 --- a/geonode/security/views.py +++ b/geonode/security/views.py @@ -85,9 +85,9 @@ def resource_permissions_handle_post(request, resource): not view_any: success = False - message = "User {} has download permissions but cannot " \ + message = f"User {user.username} has download permissions but cannot " \ "access the resource. Please update permission " \ - "consistently!".format(user.username) + "consistently!" return HttpResponse( json.dumps({'success': success, 'message': message}), @@ -337,7 +337,7 @@ def attributes_sats_refresh(request): json.dumps( { 'success': 'false', - 'message': 'Error trying to fetch the resource "%s" from GeoServer!' % layer.store + 'message': f'Error trying to fetch the resource "{layer.store}" from GeoServer!' }), status=302, content_type='text/plain') @@ -355,7 +355,7 @@ def attributes_sats_refresh(request): json.dumps( { 'success': 'false', - 'message': 'Exception occurred: "%s"' % str(e) + 'message': f'Exception occurred: "{str(e)}"' }), status=302, content_type='text/plain') @@ -468,8 +468,8 @@ def send_email_owner_on_view(owner, viewer, layer_id, geonode_email="email@geo.n from django.core.mail import EmailMessage # TODO: Copy edit message. subject_email = "Your Layer has been seen." - msg = ("Your layer called {0} with uuid={1}" - " was seen by {2}").format(layer.name, layer.uuid, viewer) + msg = (f"Your layer called {layer.name} with uuid={layer.uuid}" + f" was seen by {viewer}") try: email = EmailMessage( subject=subject_email, diff --git a/geonode/services/management/commands/importservice.py b/geonode/services/management/commands/importservice.py index ddf8fe58f0c..6e499f8284d 100644 --- a/geonode/services/management/commands/importservice.py +++ b/geonode/services/management/commands/importservice.py @@ -103,10 +103,10 @@ def handle(self, url, name, type, method, console=sys.stdout, **options): json_response = json.loads(ensure_string(response.content)) if "id" in json_response: - print("Service created with id of {}".format(json_response["id"])) + print(f"Service created with id of {json_response['id']}") service = Service.objects.get(id=json_response["id"]) else: - logger.error("Something went wrong: {}".format(ensure_string(response.content))) + logger.error(f"Something went wrong: {ensure_string(response.content)}") return print(service.id) diff --git a/geonode/services/models.py b/geonode/services/models.py index ba1de3b7259..87930d519a8 100644 --- a/geonode/services/models.py +++ b/geonode/services/models.py @@ -152,7 +152,7 @@ class Service(ResourceBase): # Supported Capabilities def __str__(self): - return "{0}".format(self.name) + return str(self.name) @property def service_url(self): diff --git a/geonode/services/serviceprocessors/arcgis.py b/geonode/services/serviceprocessors/arcgis.py index 5bf950db2de..b035622af75 100644 --- a/geonode/services/serviceprocessors/arcgis.py +++ b/geonode/services/serviceprocessors/arcgis.py @@ -188,7 +188,7 @@ def harvest_resource(self, resource_id, geonode_service): ) if existance_test_qs.exists(): raise RuntimeError( - "Resource {!r} has already been harvested".format(resource_id)) + f"Resource {resource_id} has already been harvested") resource_fields["keywords"] = keywords resource_fields["is_approved"] = True resource_fields["is_published"] = True @@ -202,7 +202,7 @@ def harvest_resource(self, resource_id, geonode_service): # self._create_layer_legend_link(geonode_layer) else: raise RuntimeError( - "Resource {!r} cannot be harvested".format(resource_id)) + f"Resource {resource_id} cannot be harvested") def has_resources(self): try: @@ -213,20 +213,15 @@ def has_resources(self): def _offers_geonode_projection(self, srs): geonode_projection = getattr(settings, "DEFAULT_MAP_CRS", "EPSG:3857") - return geonode_projection in "EPSG:{}".format(srs) + return geonode_projection in f"EPSG:{srs}" def _get_indexed_layer_fields(self, layer_meta): - srs = "EPSG:%s" % layer_meta.extent.spatialReference.wkid + srs = f"EPSG:{layer_meta.extent.spatialReference.wkid}" bbox = utils.decimal_encode([layer_meta.extent.xmin, layer_meta.extent.ymin, layer_meta.extent.xmax, layer_meta.extent.ymax]) - - typename = slugify("{}-{}".format( - layer_meta.id, - ''.join(c for c in layer_meta.title if ord(c) < 128) - )) - + typename = slugify(f"{layer_meta.id}-{''.join(c for c in layer_meta.title if ord(c) < 128)}") return { "name": layer_meta.title, "store": self.name, @@ -262,7 +257,6 @@ def _create_layer(self, geonode_service, **resource_fields): def _create_layer_thumbnail(self, geonode_layer): """Create a thumbnail with a WMS request.""" - create_thumbnail( instance=geonode_layer, wms_version=self.parsed_service.version, @@ -275,19 +269,13 @@ def _create_layer_service_link(self, geonode_layer): Link.objects.get_or_create( resource=geonode_layer.resourcebase_ptr, url=geonode_layer.ows_url, - name="ESRI {}: {} Service".format( - geonode_layer.remote_service.type, - geonode_layer.store - ), + name=f"ESRI {geonode_layer.remote_service.type}: {geonode_layer.store} Service", defaults={ "extension": "html", - "name": "ESRI {}: {} Service".format( - geonode_layer.remote_service.type, - geonode_layer.store - ), + "name": f"ESRI {geonode_layer.remote_service.type}: {geonode_layer.store} Service", "url": geonode_layer.ows_url, "mime": "text/html", - "link_type": "ESRI:{}".format(geonode_layer.remote_service.type), + "link_type": f"ESRI:{geonode_layer.remote_service.type}", } ) diff --git a/geonode/services/serviceprocessors/base.py b/geonode/services/serviceprocessors/base.py index f1b736fea0f..a2500f3f5d4 100644 --- a/geonode/services/serviceprocessors/base.py +++ b/geonode/services/serviceprocessors/base.py @@ -89,7 +89,7 @@ def get_geoserver_cascading_workspace(create=True): name = getattr(settings, "CASCADE_WORKSPACE", "cascaded-services") workspace = catalog.get_workspace(name) if workspace is None and create: - uri = "http://www.geonode.org/{}".format(name) + uri = f"http://www.geonode.org/{name}" workspace = catalog.create_workspace(name, uri) return workspace diff --git a/geonode/services/serviceprocessors/handler.py b/geonode/services/serviceprocessors/handler.py index 70295cc9b65..49637ea7b05 100644 --- a/geonode/services/serviceprocessors/handler.py +++ b/geonode/services/serviceprocessors/handler.py @@ -51,7 +51,7 @@ def get_service_handler(base_url, proxy_base=None, service_type=enumerations.AUT else: to_check = (k for k, v in handlers.items() if v["OWS"]) for type_ in to_check: - logger.debug("Checking {}...".format(type_)) + logger.debug(f"Checking {type_}...") try: service = get_service_handler(base_url, type_) except Exception: @@ -59,14 +59,14 @@ def get_service_handler(base_url, proxy_base=None, service_type=enumerations.AUT else: break else: - raise RuntimeError("Could not parse service {!r} with any of the " - "available service handlers".format(base_url)) + raise RuntimeError(f"Could not parse service {base_url} with any of the " + "available service handlers") else: handler = handlers.get(service_type, {}).get("handler") try: service = handler(base_url) except Exception: logger.exception( - msg="Could not parse service {!r}".format(base_url)) + msg=f"Could not parse service {base_url}") raise return service diff --git a/geonode/services/serviceprocessors/wms.py b/geonode/services/serviceprocessors/wms.py index 5e4cc424bbf..eb4c7ebbfd9 100644 --- a/geonode/services/serviceprocessors/wms.py +++ b/geonode/services/serviceprocessors/wms.py @@ -230,7 +230,7 @@ def harvest_resource(self, resource_id, geonode_service): """ layer_meta = self.get_resource(resource_id) - logger.debug("layer_meta: {}".format(layer_meta)) + logger.debug(f"layer_meta: {layer_meta}") if self.indexing_method == CASCADED: logger.debug("About to import cascaded layer...") geoserver_resource = self._import_cascaded_resource(layer_meta) @@ -247,7 +247,7 @@ def harvest_resource(self, resource_id, geonode_service): ) if existance_test_qs.exists(): raise RuntimeError( - "Resource {!r} has already been harvested".format(resource_id)) + f"Resource {resource_id} has already been harvested") resource_fields["keywords"] = keywords resource_fields["is_approved"] = True resource_fields["is_published"] = True @@ -290,7 +290,6 @@ def _create_layer(self, geonode_service, **resource_fields): def _create_layer_thumbnail(self, geonode_layer): """Create a thumbnail with a WMS request.""" - create_thumbnail( instance=geonode_layer, wms_version=self.parsed_service.version, @@ -320,9 +319,8 @@ def _create_layer_legend_link(self, geonode_layer): "fontAntiAliasing:true;fontSize:12;forceLabels:on") } kvp = "&".join("{}={}".format(*item) for item in params.items()) - legend_url = "{}{}{}".format( - geonode_layer.remote_service.service_url, _q_separator, kvp) - logger.debug("legend_url: {}".format(legend_url)) + legend_url = f"{geonode_layer.remote_service.service_url}{_q_separator}{kvp}" + logger.debug(f"legend_url: {legend_url}") Link.objects.get_or_create( resource=geonode_layer.resourcebase_ptr, url=legend_url, @@ -338,7 +336,7 @@ def _create_layer_legend_link(self, geonode_layer): def _create_layer_service_link(self, geonode_layer): ogc_wms_url = geonode_layer.ows_url - ogc_wms_name = 'OGC WMS: %s Service' % geonode_layer.store + ogc_wms_name = f'OGC WMS: {geonode_layer.store} Service' ogc_wms_link_type = 'OGC:WMS' if Link.objects.filter(resource=geonode_layer.resourcebase_ptr, name=ogc_wms_name, @@ -365,8 +363,8 @@ def _get_cascaded_layer_fields(self, geoserver_resource): "name": name, "workspace": workspace or "remoteWorkspace", "store": store.name if store and hasattr(store, 'name') else self.name, - "typename": "{}:{}".format(workspace, name) if workspace not in name else name, - "alternate": "{}:{}".format(workspace, name) if workspace not in name else name, + "typename": f"{workspace}:{name}" if workspace not in name else name, + "alternate": f"{workspace}:{name}" if workspace not in name else name, "storeType": "remoteStore", "title": geoserver_resource.title, "abstract": geoserver_resource.abstract, @@ -401,8 +399,8 @@ def _get_store(self, create=True): workspace = base.get_geoserver_cascading_workspace(create=create) cat = workspace.catalog store = cat.get_store(self.name, workspace=workspace) - logger.debug("name: {}".format(self.name)) - logger.debug("store: {}".format(store)) + logger.debug(f"name: {self.name}") + logger.debug(f"store: {store}") if store is None and create: # store did not exist. Create it store = cat.create_wmsstore( name=self.name, @@ -436,12 +434,11 @@ def _import_cascaded_resource(self, layer_meta): layer_resource.projection_policy = "REPROJECT_TO_DECLARED" cat.save(layer_resource) if layer_resource is None: - raise RuntimeError("Could not cascade resource {!r} through " - "geoserver".format(layer_meta)) + raise RuntimeError(f"Could not cascade resource {layer_meta} through " + "geoserver") layer_resource = layer_resource.resource else: - logger.debug("Layer {} is already present. Skipping...".format( - layer_meta.id)) + logger.debug(f"Layer {layer_meta.id} is already present. Skipping...") layer_resource.refresh() return layer_resource @@ -524,7 +521,7 @@ def harvest_resource(self, resource_id, geonode_service): ) if existance_test_qs.exists(): raise RuntimeError( - "Resource {!r} has already been harvested".format(resource_id)) + f"Resource {resource_id} has already been harvested") resource_fields["keywords"] = keywords resource_fields["is_approved"] = True resource_fields["is_published"] = True @@ -541,9 +538,9 @@ def harvest_resource(self, resource_id, geonode_service): def _probe_geonode_wms(self, raw_url): url = urlsplit(raw_url) - base_url = '%s://%s/' % (url.scheme, url.netloc) + base_url = f'{url.scheme}://{url.netloc}/' response = requests.get( - '%sapi/ows_endpoints/' % base_url, {}, + f'{base_url}api/ows_endpoints/', {}, timeout=30, verify=False) content = response.content @@ -566,16 +563,16 @@ def _probe_geonode_wms(self, raw_url): logger.exception(e) # OLD-style not OWS Enabled GeoNode - _url = "%s://%s/geoserver/ows" % (url.scheme, url.netloc) + _url = f"{url.scheme}://{url.netloc}/geoserver/ows" return _url def _enrich_layer_metadata(self, geonode_layer): workspace, layername = geonode_layer.name.split( ":") if ":" in geonode_layer.name else (None, geonode_layer.name) url = urlsplit(self.url) - base_url = '%s://%s/' % (url.scheme, url.netloc) + base_url = f'{url.scheme}://{url.netloc}/' response = requests.get( - '%sapi/layers/?name=%s' % (base_url, layername), {}, + f'{base_url}api/layers/?name={layername}', {}, timeout=10, verify=False) content = response.content @@ -607,20 +604,19 @@ def _enrich_layer_metadata(self, geonode_layer): thumbnail_remote_url = _layer["thumbnail_url"] _url = urlsplit(thumbnail_remote_url) if not _url.scheme: - thumbnail_remote_url = "{}{}".format( - geonode_layer.remote_service.service_url, _url.path) + thumbnail_remote_url = f"{geonode_layer.remote_service.service_url}{_url.path}" resp, image = http_client.request( thumbnail_remote_url) if 'ServiceException' in str(image) or \ resp.status_code < 200 or resp.status_code > 299: - msg = 'Unable to obtain thumbnail: %s' % image + msg = f'Unable to obtain thumbnail: {image}' logger.debug(msg) # Replace error message with None. image = None if image is not None: - thumbnail_name = 'layer-%s-thumb.png' % geonode_layer.uuid + thumbnail_name = f'layer-{geonode_layer.uuid}-thumb.png' geonode_layer.save_thumbnail( thumbnail_name, image=image) else: diff --git a/geonode/services/signals.py b/geonode/services/signals.py index db6c16f6aea..4bfd2658cca 100644 --- a/geonode/services/signals.py +++ b/geonode/services/signals.py @@ -45,7 +45,7 @@ def remove_harvest_job(sender, **kwargs): resource_id = layer.alternate.split(":")[1] if re.match(f'^{workspace.name}:', resource_id) else layer.alternate if HarvestJob.objects.filter(resource_id=resource_id): for job in HarvestJob.objects.filter(resource_id=resource_id, service=layer.remote_service): - logger.debug("job: {}".format(job.id)) + logger.debug(f"job: {job.id}") job.delete() else: pass # layer was not harvested from a service, we've nothing to do diff --git a/geonode/services/tasks.py b/geonode/services/tasks.py index 70977d08951..b06375b91a9 100644 --- a/geonode/services/tasks.py +++ b/geonode/services/tasks.py @@ -75,7 +75,7 @@ def harvest_resource(self, harvest_job_id): time.sleep(10) except Exception as err: logger.exception(msg="An error has occurred while harvesting " - "resource {!r}".format(harvest_job.resource_id)) + f"resource {harvest_job.resource_id}") details = str(err) # TODO: pass more context about the error finally: if not details: diff --git a/geonode/services/tests.py b/geonode/services/tests.py index c4334c02949..c83dab6355e 100644 --- a/geonode/services/tests.py +++ b/geonode/services/tests.py @@ -98,7 +98,7 @@ def test_get_cascading_workspace_creates_new_workspace(self, mock_settings, cat.get_workspace.assert_called_with(mock_settings.CASCADE_WORKSPACE) cat.create_workspace.assert_called_with( mock_settings.CASCADE_WORKSPACE, - "http://www.geonode.org/{}".format(mock_settings.CASCADE_WORKSPACE) + f"http://www.geonode.org/{mock_settings.CASCADE_WORKSPACE}" ) @mock.patch("geonode.services.serviceprocessors.handler.WmsServiceHandler", diff --git a/geonode/services/utils.py b/geonode/services/utils.py index 2d0cb439ed7..ad5ca6a5b16 100644 --- a/geonode/services/utils.py +++ b/geonode/services/utils.py @@ -26,7 +26,7 @@ def flip_coordinates(c1, c2): if c1 > c2: - logger.debug('Flipping coordinates %s, %s' % (c1, c2)) + logger.debug(f'Flipping coordinates {c1}, {c2}') temp = c1 c1 = c2 c2 = temp @@ -124,7 +124,7 @@ def decimal_encode(bbox): except Exception: o = None if 'EPSG' not in o else o if o and isinstance(o, float): - _bbox.append("{0:.15f}".format(round(o, 2))) + _bbox.append(f"{round(o, 2):.15f}") elif o and 'EPSG' in o: _srid = o _bbox = _bbox if not _srid else _bbox + [_srid] diff --git a/geonode/services/views.py b/geonode/services/views.py index b30daeb4bb2..1c99a427dcf 100644 --- a/geonode/services/views.py +++ b/geonode/services/views.py @@ -56,11 +56,9 @@ def service_proxy(request, service_id): if not service.proxy_base: service_url = service.base_url else: - service_url = "{ows_url}?{ows_request}".format( - ows_url=service.base_url, ows_request=request.META['QUERY_STRING']) + service_url = f"{service.base_url}?{request.META['QUERY_STRING']}" if urljoin(settings.SITEURL, reverse('proxy')) != service.proxy_base: - service_url = "{proxy_base}?url={service_url}".format(proxy_base=service.proxy_base, - service_url=quote(service_url, safe='')) + service_url = f"{service.proxy_base}?url={quote(service_url, safe='')}" return proxy(request, url=service_url, sec_chk_hosts=False) @@ -178,7 +176,7 @@ def harvest_resources_handle_post(request, service, handler): requested = list(set(requested)) resources_to_harvest = [] for id in _gen_harvestable_ids(requested, available_resources): - logger.debug("id: {}".format(id)) + logger.debug(f"id: {id}") harvest_job, created = HarvestJob.objects.get_or_create( service=service, resource_id=id @@ -188,7 +186,7 @@ def harvest_resources_handle_post(request, service, handler): tasks.harvest_resource.apply_async((harvest_job.id,)) else: logger.warning( - "resource {} already has a harvest job".format(id)) + f"resource {id} already has a harvest job") msg_async = _("The selected resources are being imported") msg_sync = _("The selected resources have been imported") messages.add_message( @@ -239,7 +237,7 @@ def harvest_single_resource(request, service_id, resource_id): messages.add_message( request, messages.SUCCESS, - _("Resource {} is being processed".format(resource_id)) + _(f"Resource {resource_id} is being processed") ) return redirect( reverse("service_detail", @@ -377,6 +375,6 @@ def remove_service(request, service_id): messages.add_message( request, messages.INFO, - _("Service {} has been deleted".format(service.name)) + _(f"Service {service.name} has been deleted") ) return HttpResponseRedirect(reverse("services")) diff --git a/geonode/settings.py b/geonode/settings.py index 1def17f08a2..65f4b1020c3 100644 --- a/geonode/settings.py +++ b/geonode/settings.py @@ -36,6 +36,9 @@ from django.conf.global_settings import DATETIME_INPUT_FORMATS from geonode import get_version from kombu import Queue, Exchange +from kombu.serialization import register + +from . import serializer SILENCED_SYSTEM_CHECKS = [ '1_8.W001', @@ -88,9 +91,8 @@ SITE_HOST_SCHEMA = os.getenv('SITE_HOST_SCHEMA', 'http') SITE_HOST_NAME = os.getenv('SITE_HOST_NAME', 'localhost') SITE_HOST_PORT = os.getenv('SITE_HOST_PORT', 8000) -_default_siteurl = "%s://%s:%s/" % (SITE_HOST_SCHEMA, - SITE_HOST_NAME, - SITE_HOST_PORT) if SITE_HOST_PORT else "%s://%s/" % (SITE_HOST_SCHEMA, SITE_HOST_NAME) +_default_siteurl = f"{SITE_HOST_SCHEMA}://{SITE_HOST_NAME}:{SITE_HOST_PORT}/" \ + if SITE_HOST_PORT else f"{SITE_HOST_SCHEMA}://{SITE_HOST_NAME}/" SITEURL = os.getenv('SITEURL', _default_siteurl) # we need hostname for deployed @@ -99,13 +101,12 @@ # add trailing slash to site url. geoserver url will be relative to this if not SITEURL.endswith('/'): - SITEURL = '{}/'.format(SITEURL) + SITEURL = f'{SITEURL}/' +_DB_PATH = os.path.join(PROJECT_ROOT, 'development.db') DATABASE_URL = os.getenv( 'DATABASE_URL', - 'spatialite:///{path}'.format( - path=os.path.join(PROJECT_ROOT, 'development.db') - ) + f'spatialite:///{_DB_PATH}' ) if DATABASE_URL.startswith("spatialite"): @@ -115,6 +116,7 @@ if spatialite_version < 5: # To workaround Shapely/Spatialite interaction bug for Spatialite < 5 from shapely import speedups + speedups.enable() except FileNotFoundError as ex: print(ex) @@ -289,8 +291,8 @@ # URL that handles the media served from MEDIA_ROOT. Make sure to use a # trailing slash if there is a path component (optional in other cases). # Examples: "http://media.lawrence.com", "http://example.com/media/" -MEDIA_URL = os.getenv('MEDIA_URL', '%s/%s/' % (FORCE_SCRIPT_NAME, MEDIAFILES_LOCATION)) -LOCAL_MEDIA_URL = os.getenv('LOCAL_MEDIA_URL', '%s/%s/' % (FORCE_SCRIPT_NAME, MEDIAFILES_LOCATION)) +MEDIA_URL = os.getenv('MEDIA_URL', f'{FORCE_SCRIPT_NAME}/{MEDIAFILES_LOCATION}/') +LOCAL_MEDIA_URL = os.getenv('LOCAL_MEDIA_URL', f'{FORCE_SCRIPT_NAME}/{MEDIAFILES_LOCATION}/') # Absolute path to the directory that holds static files like app media. # Example: "/home/media/media.lawrence.com/apps/" @@ -300,7 +302,7 @@ # URL that handles the static files like app media. # Example: "http://media.lawrence.com" -STATIC_URL = os.getenv('STATIC_URL', '%s/%s/' % (FORCE_SCRIPT_NAME, STATICFILES_LOCATION)) +STATIC_URL = os.getenv('STATIC_URL', f'{FORCE_SCRIPT_NAME}/{STATICFILES_LOCATION}/') # Additional directories which hold static files _DEFAULT_STATICFILES_DIRS = [ @@ -327,18 +329,17 @@ AWS_STORAGE_BUCKET_NAME = os.environ.get('S3_BUCKET_NAME', '') AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID', '') AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY', '') -AWS_S3_BUCKET_DOMAIN = '%s.s3.amazonaws.com' % AWS_STORAGE_BUCKET_NAME +AWS_S3_BUCKET_DOMAIN = f'{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com' AWS_QUERYSTRING_AUTH = False if not DEBUG and S3_STATIC_ENABLED: STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' - STATIC_URL = "https://%s/%s/" % (AWS_S3_BUCKET_DOMAIN, - STATICFILES_LOCATION) + STATIC_URL = f"https://{AWS_S3_BUCKET_DOMAIN}/{STATICFILES_LOCATION}/" if not DEBUG and S3_MEDIA_ENABLED: MEDIAFILES_LOCATION = 'media' DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' - MEDIA_URL = "https://%s/%s/" % (AWS_S3_BUCKET_DOMAIN, MEDIAFILES_LOCATION) + MEDIA_URL = f"https://{AWS_S3_BUCKET_DOMAIN}/{MEDIAFILES_LOCATION}/" # Cache Bustin Settings CACHE_BUSTING_STATIC_ENABLED = ast.literal_eval(os.environ.get('CACHE_BUSTING_STATIC_ENABLED', 'False')) @@ -562,7 +563,7 @@ # ENABLE_SERIALIZER_OPTIMIZATIONS: enable/disable representation speedups 'ENABLE_SERIALIZER_OPTIMIZATIONS': True, - # DEFER_MANY_RELATIONS: automatically defer many-relations, unless + # DEFER_MANY_RELATIONS: automatically defer many-relations, unless # `deferred=False` is explicitly set on the field. 'DEFER_MANY_RELATIONS': False, @@ -845,16 +846,16 @@ # - authorized exempt urls needed for oauth when GeoNode is set to lockdown AUTH_EXEMPT_URLS = ( r'^%s/?$' % FORCE_SCRIPT_NAME, - '%s/o/*' % FORCE_SCRIPT_NAME, - '%s/gs/*' % FORCE_SCRIPT_NAME, - '%s/account/*' % FORCE_SCRIPT_NAME, - '%s/static/*' % FORCE_SCRIPT_NAME, - '%s/api/o/*' % FORCE_SCRIPT_NAME, - '%s/api/roles' % FORCE_SCRIPT_NAME, - '%s/api/adminRole' % FORCE_SCRIPT_NAME, - '%s/api/users' % FORCE_SCRIPT_NAME, - '%s/api/layers' % FORCE_SCRIPT_NAME, - '%s/monitoring' % FORCE_SCRIPT_NAME, + f'{FORCE_SCRIPT_NAME}/o/*', + f'{FORCE_SCRIPT_NAME}/gs/*', + f'{FORCE_SCRIPT_NAME}/account/*', + f'{FORCE_SCRIPT_NAME}/static/*', + f'{FORCE_SCRIPT_NAME}/api/o/*', + f'{FORCE_SCRIPT_NAME}/api/roles', + f'{FORCE_SCRIPT_NAME}/api/adminRole', + f'{FORCE_SCRIPT_NAME}/api/users', + f'{FORCE_SCRIPT_NAME}/api/layers', + f'{FORCE_SCRIPT_NAME}/monitoring', r'^/i18n/setlang/?$', ) @@ -916,8 +917,8 @@ # per-deployment settings should go here # Login and logout urls override -LOGIN_URL = os.getenv('LOGIN_URL', '{}account/login/'.format(SITEURL)) -LOGOUT_URL = os.getenv('LOGOUT_URL', '{}account/logout/'.format(SITEURL)) +LOGIN_URL = os.getenv('LOGIN_URL', f'{SITEURL}account/login/') +LOGOUT_URL = os.getenv('LOGOUT_URL', f'{SITEURL}account/logout/') ACCOUNT_LOGIN_REDIRECT_URL = os.getenv('LOGIN_REDIRECT_URL', SITEURL) ACCOUNT_LOGOUT_REDIRECT_URL = os.getenv('LOGOUT_REDIRECT_URL', SITEURL) @@ -946,8 +947,8 @@ # add trailing slash to geoserver location url. if not GEOSERVER_LOCATION.endswith('/'): - GEOSERVER_LOCATION = '{}/'.format(GEOSERVER_LOCATION) - + GEOSERVER_LOCATION = f'{GEOSERVER_LOCATION}/' + GEOSERVER_PUBLIC_SCHEMA = os.getenv( 'GEOSERVER_PUBLIC_SCHEMA', SITE_HOST_SCHEMA ) @@ -960,10 +961,10 @@ 'GEOSERVER_PUBLIC_PORT', 8080 ) -_default_public_location = '{}://{}:{}/geoserver/'.format( - GEOSERVER_PUBLIC_SCHEMA, - GEOSERVER_PUBLIC_HOST, - GEOSERVER_PUBLIC_PORT) if GEOSERVER_PUBLIC_PORT else '{}://{}/geoserver/'.format(GEOSERVER_PUBLIC_SCHEMA, GEOSERVER_PUBLIC_HOST) +if GEOSERVER_PUBLIC_PORT: + _default_public_location = f'{GEOSERVER_PUBLIC_SCHEMA}://{GEOSERVER_PUBLIC_HOST}:{GEOSERVER_PUBLIC_PORT}/geoserver/' +else: + _default_public_location = f'{GEOSERVER_PUBLIC_SCHEMA}://{GEOSERVER_PUBLIC_HOST}/geoserver/' GEOSERVER_PUBLIC_LOCATION = os.getenv( 'GEOSERVER_PUBLIC_LOCATION', _default_public_location @@ -981,7 +982,8 @@ 'GEOSERVER_ADMIN_PASSWORD', 'geoserver' ) -GEOFENCE_SECURITY_ENABLED = False if TEST and not INTEGRATION else ast.literal_eval(os.getenv('GEOFENCE_SECURITY_ENABLED', 'True')) +GEOFENCE_SECURITY_ENABLED = False if TEST and not INTEGRATION \ + else ast.literal_eval(os.getenv('GEOFENCE_SECURITY_ENABLED', 'True')) # OGC (WMS/WFS/WCS) Server Settings # OGC (WMS/WFS/WCS) Server Settings @@ -1234,8 +1236,11 @@ PROXY_ALLOWED_HOSTS = ast.literal_eval(os.getenv('PROXY_ALLOWED_HOSTS')) except ValueError: # fallback to regular list of values separated with misc chars - PROXY_ALLOWED_HOSTS = [HOSTNAME, 'localhost', 'django', 'geonode', 'spatialreference.org', 'nominatim.openstreetmap.org', 'dev.openlayers.org'] if os.getenv('PROXY_ALLOWED_HOSTS') is None \ - else re.split(r' *[,|:|;] *', os.getenv('PROXY_ALLOWED_HOSTS')) + PROXY_ALLOWED_HOSTS = [ + HOSTNAME, 'localhost', 'django', 'geonode', + 'spatialreference.org', 'nominatim.openstreetmap.org', 'dev.openlayers.org'] \ + if os.getenv('PROXY_ALLOWED_HOSTS') is None \ + else re.split(r' *[,|:|;] *', os.getenv('PROXY_ALLOWED_HOSTS')) # The proxy to use when making cross origin requests. PROXY_URL = os.environ.get('PROXY_URL', '/proxy/?url=') @@ -1413,15 +1418,15 @@ def get_geonode_catalogue_service(): if PYCSW: pycsw_config = PYCSW["CONFIGURATION"] if pycsw_config: - pycsw_catalogue = { - ("%s" % pycsw_config['metadata:main']['identification_title']): { - "url": CATALOGUE['default']['URL'], - "type": "csw", - "title": pycsw_config['metadata:main']['identification_title'], - "autoload": True - } - } - return pycsw_catalogue + pycsw_catalogue = { + f"{pycsw_config['metadata:main']['identification_title']}": { + "url": CATALOGUE['default']['URL'], + "type": "csw", + "title": pycsw_config['metadata:main']['identification_title'], + "autoload": True + } + } + return pycsw_catalogue return None GEONODE_CATALOGUE_SERVICE = get_geonode_catalogue_service() @@ -1444,7 +1449,7 @@ def get_geonode_catalogue_service(): MAPSTORE_CATALOGUE_SELECTED_SERVICE = "Demo WMS Service" if GEONODE_CATALOGUE_SERVICE: - MAPSTORE_CATALOGUE_SERVICES[list(list(GEONODE_CATALOGUE_SERVICE.keys()))[0]] = GEONODE_CATALOGUE_SERVICE[list(list(GEONODE_CATALOGUE_SERVICE.keys()))[0]] + MAPSTORE_CATALOGUE_SERVICES[list(list(GEONODE_CATALOGUE_SERVICE.keys()))[0]] = GEONODE_CATALOGUE_SERVICE[list(list(GEONODE_CATALOGUE_SERVICE.keys()))[0]] # noqa MAPSTORE_CATALOGUE_SELECTED_SERVICE = list(list(GEONODE_CATALOGUE_SERVICE.keys()))[0] DEFAULT_MS2_BACKGROUNDS = [ @@ -1471,7 +1476,7 @@ def get_geonode_catalogue_service(): "name": "s2cloudless:s2cloudless", "url": "https://maps.geo-solutions.it/geoserver/wms", "group": "background", - "thumbURL": "%sstatic/mapstorestyle/img/s2cloudless-s2cloudless.png" % SITEURL, + "thumbURL": f"{SITEURL}static/mapstorestyle/img/s2cloudless-s2cloudless.png", "visibility": False }, { "source": "ol", @@ -1483,19 +1488,6 @@ def get_geonode_catalogue_service(): "visibility": False, "args": ["Empty Background", {"visibility": False}] } - # Custom XYZ Tile Provider - # { - # "type": "tileprovider", - # "title": "Title", - # "provider": "custom", // or undefined - # "name": "Name", - # "group": "background", - # "visibility": false, - # "url": "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", - # "options": { - # "subdomains": [ "a", "b"] - # } - # } ] if MAPBOX_ACCESS_TOKEN: @@ -1504,13 +1496,13 @@ def get_geonode_catalogue_service(): "title": "MapBox streets-v11", "provider": "MapBoxStyle", "name": "MapBox streets-v11", - "accessToken": "%s" % MAPBOX_ACCESS_TOKEN, + "accessToken": f"{MAPBOX_ACCESS_TOKEN}", "source": "streets-v11", - "thumbURL": "https://api.mapbox.com/styles/v1/mapbox/streets-v11/tiles/256/6/33/23?access_token=%s" % MAPBOX_ACCESS_TOKEN, + "thumbURL": f"https://api.mapbox.com/styles/v1/mapbox/streets-v11/tiles/256/6/33/23?access_token={MAPBOX_ACCESS_TOKEN}", # noqa "group": "background", "visibility": True } - DEFAULT_MS2_BACKGROUNDS = [BASEMAP,] + DEFAULT_MS2_BACKGROUNDS + DEFAULT_MS2_BACKGROUNDS = [BASEMAP, ] + DEFAULT_MS2_BACKGROUNDS if BING_API_KEY: BASEMAP = { @@ -1522,7 +1514,7 @@ def get_geonode_catalogue_service(): "apiKey": "{{apiKey}}", "visibility": False } - DEFAULT_MS2_BACKGROUNDS = [BASEMAP,] + DEFAULT_MS2_BACKGROUNDS + DEFAULT_MS2_BACKGROUNDS = [BASEMAP, ] + DEFAULT_MS2_BACKGROUNDS MAPSTORE_BASELAYERS = DEFAULT_MS2_BACKGROUNDS @@ -1549,10 +1541,10 @@ def get_geonode_catalogue_service(): TINYMCE_DEFAULT_CONFIG = { "theme": "silver", "height": 500, - "plugins": 'print preview paste importcss searchreplace autolink autosave save directionality code visualblocks visualchars fullscreen image link media template codesample table charmap hr pagebreak nonbreaking anchor toc insertdatetime advlist lists wordcount imagetools textpattern noneditable help charmap quickbars emoticons', + "plugins": 'print preview paste importcss searchreplace autolink autosave save directionality code visualblocks visualchars fullscreen image link media template codesample table charmap hr pagebreak nonbreaking anchor toc insertdatetime advlist lists wordcount imagetools textpattern noneditable help charmap quickbars emoticons', # noqa "imagetools_cors_hosts": ['picsum.photos'], "menubar": 'file edit view insert format tools table help', - "toolbar": 'undo redo | bold italic underline strikethrough | fontselect fontsizeselect formatselect | alignleft aligncenter alignright alignjustify | outdent indent | numlist bullist | forecolor backcolor removeformat | pagebreak | charmap emoticons | fullscreen preview save | insertfile image media template link anchor codesample | ltr rtl', + "toolbar": 'undo redo | bold italic underline strikethrough | fontselect fontsizeselect formatselect | alignleft aligncenter alignright alignjustify | outdent indent | numlist bullist | forecolor backcolor removeformat | pagebreak | charmap emoticons | fullscreen preview save | insertfile image media template link anchor codesample | ltr rtl', # noqa "toolbar_sticky": "true", "autosave_ask_before_unload": "true", "autosave_interval": "30s", @@ -1571,7 +1563,7 @@ def get_geonode_catalogue_service(): { "title": 'New Table', "description": 'creates a new table', - "content": '
' + "content": '
' # noqa }, { "title": 'Starting my story', @@ -1581,7 +1573,7 @@ def get_geonode_catalogue_service(): { "title": 'New list with dates', "description": 'New List with dates', - "content": '
cdate
mdate

My List

' + "content": '
cdate
mdate

My List

' # noqa } ], "template_cdate_format": '[Date Created (CDATE): %m/%d/%Y : %H:%M:%S]', @@ -1638,7 +1630,7 @@ def get_geonode_catalogue_service(): 'CELERY_RESULT_BACKEND_PATH', os.path.join(PROJECT_ROOT, '.celery_results')) if not os.path.exists(CELERY_RESULT_BACKEND_PATH): os.makedirs(CELERY_RESULT_BACKEND_PATH) - CELERY_RESULT_BACKEND = 'file:///%s' % CELERY_RESULT_BACKEND_PATH + CELERY_RESULT_BACKEND = f'file:///{CELERY_RESULT_BACKEND_PATH}' CELERY_BROKER_URL = os.environ.get('BROKER_URL', _BROKER_URL) CELERY_RESULT_PERSISTENT = ast.literal_eval(os.environ.get('CELERY_RESULT_PERSISTENT', 'False')) @@ -1656,9 +1648,6 @@ def get_geonode_catalogue_service(): CELERY_TASK_EAGER_PROPAGATES = ast.literal_eval(os.environ.get('CELERY_TASK_EAGER_PROPAGATES', 'True')) CELERY_TASK_IGNORE_RESULT = ast.literal_eval(os.environ.get('CELERY_TASK_IGNORE_RESULT', 'True')) - -from . import serializer -from kombu.serialization import register register('geonode_json', serializer.dumps, serializer.loads, content_type='application/json', content_encoding='utf-8') # I use these to debug kombu crashes; we get a more informative message. @@ -1757,7 +1746,7 @@ def get_geonode_catalogue_service(): # NOTIFICATIONS SETTINGS # ########################################################################### # NOTIFICATION_ENABLED = ast.literal_eval(os.environ.get('NOTIFICATION_ENABLED', 'True')) or TEST -#PINAX_NOTIFICATIONS_LANGUAGE_MODEL = "people.Profile" +# PINAX_NOTIFICATIONS_LANGUAGE_MODEL = "people.Profile" # notifications backends NOTIFICATIONS_BACKEND = os.environ.get('NOTIFICATIONS_BACKEND', 'geonode.notifications_backend.EmailBackend') @@ -1808,7 +1797,7 @@ def get_geonode_catalogue_service(): GDAL_LIBRARY_PATH = os.environ.get('GDAL_LIBRARY_PATH') else: # maybe it will be found regardless if not it will throw 500 error - from django.contrib.gis.geos import GEOSGeometry # flake8: noqa + from django.contrib.gis.geos import GEOSGeometry # noqa # Keywords thesauri # e.g. THESAURUS = {'name':'inspire_themes', 'required':True, 'filter':True} @@ -1836,7 +1825,7 @@ def get_geonode_catalogue_service(): 3. Superusers can do enything. - if [ GROUP_PRIVATE_RESOURCES == True ] - The "unapproved" and "unpublished" Resources will be accessible **ONLY** by owners, superusers and member of + The "unapproved" and "unpublished" Resources will be accessible **ONLY** by owners, superusers and member of the belonging groups. - if [ GROUP_MANDATORY_RESOURCES == True ] @@ -1886,7 +1875,7 @@ def get_geonode_catalogue_service(): SOCIALACCOUNT_ADAPTER = 'geonode.people.adapters.SocialAccountAdapter' SOCIALACCOUNT_AUTO_SIGNUP = ast.literal_eval(os.environ.get('SOCIALACCOUNT_AUTO_SIGNUP', 'True')) -#This will hide or show local registration form in allauth view. True will show form +# This will hide or show local registration form in allauth view. True will show form SOCIALACCOUNT_WITH_GEONODE_LOCAL_SINGUP = strtobool(os.environ.get('SOCIALACCOUNT_WITH_GEONODE_LOCAL_SINGUP', 'True')) # Uncomment this to enable Linkedin and Facebook login @@ -1966,7 +1955,7 @@ def get_geonode_catalogue_service(): PUBLIC_GEOSERVER = { "source": { "title": "GeoServer - Public Layers", - "attribution": "© %s" % SITEURL, + "attribution": f"© {SITEURL}", "ptype": LOCAL_GXP_PTYPE, "url": OGC_SERVER['default']['PUBLIC_LOCATION'] + "ows", "restUrl": "/gs/rest" @@ -1975,7 +1964,7 @@ def get_geonode_catalogue_service(): LOCAL_GEOSERVER = { "source": { "title": "GeoServer - Private Layers", - "attribution": "© %s" % SITEURL, + "attribution": f"© {SITEURL}", "ptype": LOCAL_GXP_PTYPE, "url": "/gs/ows", "restUrl": "/gs/rest" @@ -2036,9 +2025,8 @@ def get_geonode_catalogue_service(): USER_ANALYTICS_GZIP = ast.literal_eval(os.getenv('USER_ANALYTICS_GZIP', 'False')) GEOIP_PATH = os.getenv('GEOIP_PATH', os.path.join(PROJECT_ROOT, 'GeoIPCities.dat')) -#This controls if tastypie search on resourches is performed only with titles +# This controls if tastypie search on resourches is performed only with titles SEARCH_RESOURCES_EXTENDED = strtobool(os.getenv('SEARCH_RESOURCES_EXTENDED', 'True')) # -- END Settings for MONITORING plugin CATALOG_METADATA_TEMPLATE = os.getenv("CATALOG_METADATA_TEMPLATE", "catalogue/full_metadata.xml") - diff --git a/geonode/social/signals.py b/geonode/social/signals.py index 3e3b5118f0c..86baf51fb4a 100644 --- a/geonode/social/signals.py +++ b/geonode/social/signals.py @@ -136,12 +136,11 @@ def activity_post_modify_object(sender, instance, created=None, **kwargs): if verb: try: activity.send(action.get('actor'), - verb="{verb}".format(verb=verb), + verb=str(verb), action_object=action.get('action_object'), target=action.get('target', None), object_name=action.get('object_name'), - raw_action=raw_action, - ) + raw_action=raw_action) # except ModelNotActionable: except Exception: logger.warning('The activity received a non-actionable Model or None as the actor/action.') @@ -175,7 +174,7 @@ def relationship_post_save(instance, sender, created, **kwargs): def rating_post_save(instance, sender, created, **kwargs): """ Send a notification when rating a layer, map or document """ - notice_type_label = '%s_rated' % instance.content_object.class_name.lower() + notice_type_label = f'{instance.content_object.class_name.lower()}_rated' recipients = get_notification_recipients(notice_type_label, instance.user, resource=instance.content_object) @@ -188,7 +187,7 @@ def comment_post_save(instance, sender, created, **kwargs): """ Send a notification when a comment to a layer, map or document has been submitted """ - notice_type_label = '%s_comment' % instance.content_type.model.lower() + notice_type_label = f'{instance.content_type.model.lower()}_comment' recipients = get_comment_notification_recipients(notice_type_label, instance.author, resource=instance.content_object) diff --git a/geonode/tests/csw.py b/geonode/tests/csw.py index a5aa66a51f7..6d8b1ea9ea6 100644 --- a/geonode/tests/csw.py +++ b/geonode/tests/csw.py @@ -156,11 +156,11 @@ def test_csw_outputschema_dc(self): for link in record.references: if check_ogc_backend(geoserver.BACKEND_PACKAGE): if link['scheme'] == 'OGC:WMS': - self.assertEqual(link['url'], "{}ows".format(settings.GEOSERVER_PUBLIC_LOCATION)) + self.assertEqual(link['url'], f"{settings.GEOSERVER_PUBLIC_LOCATION}ows") elif link['scheme'] == 'OGC:WFS': - self.assertEqual(link['url'], "{}ows".format(settings.GEOSERVER_PUBLIC_LOCATION)) + self.assertEqual(link['url'], f"{settings.GEOSERVER_PUBLIC_LOCATION}ows") elif link['scheme'] == 'OGC:WCS': - self.assertEqual(link['url'], "{}ows".format(settings.GEOSERVER_PUBLIC_LOCATION)) + self.assertEqual(link['url'], f"{settings.GEOSERVER_PUBLIC_LOCATION}ows") def test_csw_outputschema_iso(self): """Verify that GeoNode CSW can handle ISO metadata with ISO outputSchema""" @@ -195,12 +195,12 @@ def test_csw_outputschema_iso(self): if link.protocol == 'OGC:WMS': self.assertEqual( link.url, - '{}ows'.format(settings.GEOSERVER_PUBLIC_LOCATION), + f'{settings.GEOSERVER_PUBLIC_LOCATION}ows', 'Expected a specific OGC:WMS URL') elif link.protocol == 'OGC:WFS': self.assertEqual( link.url, - '{}wfs'.format(settings.GEOSERVER_PUBLIC_LOCATION), + f'{settings.GEOSERVER_PUBLIC_LOCATION}wfs', 'Expected a specific OGC:WFS URL') def test_csw_outputschema_dc_bbox(self): diff --git a/geonode/tests/integration.py b/geonode/tests/integration.py index 016b87c4183..1ec4b2e7705 100644 --- a/geonode/tests/integration.py +++ b/geonode/tests/integration.py @@ -158,7 +158,7 @@ def test_layer_upload(self): import requests from requests.auth import HTTPBasicAuth - r = requests.get(url + 'gwc/rest/seed/%s.json' % saved_layer.alternate, + r = requests.get(url + f'gwc/rest/seed/{saved_layer.alternate}.json', auth=HTTPBasicAuth(user, passwd)) self.assertEqual(r.status_code, 200) o = json.loads(r.text) @@ -170,14 +170,14 @@ def test_layer_upload(self): set_styles, create_gs_thumbnail) - _log("0. ------------ %s " % saved_layer) + _log(f"0. ------------ {saved_layer} ") self.assertIsNotNone(saved_layer) workspace, name = saved_layer.alternate.split(':') self.assertIsNotNone(workspace) self.assertIsNotNone(name) ws = gs_catalog.get_workspace(workspace) self.assertIsNotNone(ws) - _log("1. ------------ %s " % saved_layer.store) + _log(f"1. ------------ {saved_layer.store} ") self.assertIsNotNone(saved_layer.store) # Save layer attributes @@ -189,7 +189,7 @@ def test_layer_upload(self): # set SLD sld = saved_layer.default_style.sld_body if saved_layer.default_style else None self.assertIsNotNone(sld) - _log("2. ------------ %s " % sld) + _log(f"2. ------------ {sld} ") set_layer_style(saved_layer, saved_layer.alternate, sld) create_gs_thumbnail(saved_layer, overwrite=True) @@ -305,9 +305,8 @@ def test_layer_upload(self): if item['file'] in not_expected_layers: continue else: - msg = ('Could not upload file "%s", ' - 'and it is not in %s' % ( - item['file'], not_expected_layers)) + msg = (f"Could not upload file '{item['file']}', " + f"and it is not in {not_expected_layers}") assert errors, msg else: msg = ('Upload should have returned either "name" or ' @@ -340,7 +339,7 @@ def test_layer_upload(self): 'rest/layers'), username=gs_username, password=gs_password) - if page.find('rest/layers/%s.html' % layer_name) > 0: + if page.find(f'rest/layers/{layer_name}.html') > 0: found = True if not found: msg = ( @@ -381,7 +380,7 @@ def test_layer_upload_metadata(self): gisdata.PROJECT_ROOT, 'both/good/sangis.org/Airport/Air_Runways.shp') - self.assertTrue('%s.xml' % thelayer, + self.assertTrue(f'{thelayer}.xml', 'Expected layer XML metadata to exist') try: if os.path.exists(thelayer): @@ -680,10 +679,10 @@ def test_repeated_upload(self): uploaded3 = file_upload(thefile, overwrite=False) check_layer(uploaded3) try: - msg = ('Expected %s but got %s' % (uploaded1.name, uploaded2.name)) + msg = (f'Expected {uploaded1.name} but got {uploaded2.name}') assert uploaded1.name == uploaded2.name, msg - msg = ('Expected a different name when uploading %s using ' - 'overwrite=False but got %s' % (thefile, uploaded3.name)) + msg = (f'Expected a different name when uploading {thefile} using ' + f'overwrite=False but got {uploaded3.name}') assert uploaded1.name != uploaded3.name, msg finally: # Clean up and completely delete the layers @@ -837,12 +836,12 @@ def test_keywords_upload(self): overwrite=True) try: keywords = uploaded.keyword_list() - msg = 'No keywords found in layer %s' % uploaded.name + msg = f'No keywords found in layer {uploaded.name}' assert len(keywords) > 0, msg assert 'foo' in uploaded.keyword_list( - ), 'Could not find "foo" in %s' % keywords + ), f'Could not find "foo" in {keywords}' assert 'bar' in uploaded.keyword_list( - ), 'Could not find "bar" in %s' % keywords + ), f'Could not find "bar" in {keywords}' finally: # Clean up and completely delete the layers uploaded.delete() diff --git a/geonode/tests/smoke.py b/geonode/tests/smoke.py index 4251f0cfe8b..c8e58cf58b1 100644 --- a/geonode/tests/smoke.py +++ b/geonode/tests/smoke.py @@ -307,9 +307,10 @@ def test_inbox_renders(self): def test_inbox_redirects_when_not_logged_in(self): target_url = reverse("messages_inbox") response = self.client.get(target_url) + account_login_url = reverse("account_login") self.assertRedirects( response, - "{}{}?next=http%3A//testserver{}".format(settings.SITEURL[:-1], reverse("account_login"), target_url) + f"{settings.SITEURL[:-1]}{account_login_url}?next=http%3A//testserver{target_url}" ) @on_ogc_backend(geoserver.BACKEND_PACKAGE) @@ -328,9 +329,10 @@ def test_new_message_renders(self): def test_new_message_redirects_when_not_logged_in(self): target_url = reverse("message_create", args=(self.first_user.id,)) response = self.client.get(target_url) + account_login_url = reverse("account_login") self.assertRedirects( response, - "{}{}?next=http%3A//testserver{}".format(settings.SITEURL[:-1], reverse("account_login"), target_url) + f"{settings.SITEURL[:-1]}{account_login_url}?next=http%3A//testserver{target_url}" ) @on_ogc_backend(geoserver.BACKEND_PACKAGE) @@ -349,7 +351,8 @@ def test_thread_detail_renders(self): def test_thread_detail_redirects_when_not_logged_in(self): target_url = reverse("messages_thread_detail", args=(self.thread.id,)) response = self.client.get(target_url) + account_login_url = reverse("account_login") self.assertRedirects( response, - "{}{}?next=http%3A//testserver{}".format(settings.SITEURL[:-1], reverse("account_login"), target_url) + f"{settings.SITEURL[:-1]}{account_login_url}?next=http%3A//testserver{target_url}" ) diff --git a/geonode/tests/suite/base.py b/geonode/tests/suite/base.py index 739cbc79653..5970bbed2c8 100644 --- a/geonode/tests/suite/base.py +++ b/geonode/tests/suite/base.py @@ -47,9 +47,9 @@ def setup_test_db(worker_index, fixtures, fn, *args): old_name = settings.DATABASES['default']['NAME'] if worker_index is not None: - test_database_name = 'test_%d_%s' % (worker_index, old_name) + test_database_name = f'test_{worker_index}_{old_name}' else: - test_database_name = 'test_%s' % (old_name) + test_database_name = f'test_{old_name}' create_test_db(test_database_name) if fixtures: diff --git a/geonode/tests/suite/runner.py b/geonode/tests/suite/runner.py index 03934f6ec9c..00ce256770b 100644 --- a/geonode/tests/suite/runner.py +++ b/geonode/tests/suite/runner.py @@ -136,7 +136,7 @@ def _run_tests(self, tests, **kwargs): group_tests = tests[group] del tests[group] - logger.debug('Running tests in a main process: %s' % (group_tests)) + logger.debug(f'Running tests in a main process: {group_tests}') pending_not_thread_safe_tests[group] = group_tests result = self._tests_func(tests=group_tests, worker_index=None) results_queue.put((group, result), block=False) @@ -166,13 +166,13 @@ def _run_tests(self, tests, **kwargs): worker_count = worker_max worker_args = (tests_queue, results_queue, stop_event) - logger.debug("Number of workers %s " % worker_count) + logger.debug(f"Number of workers {worker_count} ") workers = self._create_worker_pool(pool_size=worker_count, target_func=self._run_tests_worker, worker_args=worker_args) for index, worker in enumerate(workers): - logger.debug('Staring worker %s' % (index)) + logger.debug(f'Staring worker {index}') worker.start() if workers: @@ -190,7 +190,7 @@ def _run_tests(self, tests, **kwargs): else: pending_not_thread_safe_tests.pop(group) except KeyError: - logger.debug('Got a result for unknown group: %s' % (group)) + logger.debug(f'Got a result for unknown group: {group}') else: completed_tests[group] = result self._print_result(result) @@ -234,19 +234,13 @@ def pop_item(): if stop_event.is_set(): # We should stop break - try: - result = None logger.debug( - 'Worker %s is running tests %s' % - (index, tests)) - result = self._tests_func( - tests=tests, worker_index=index) - + f'Worker {index} is running tests {tests}') + result = self._tests_func(tests=tests, worker_index=index) results_queue.put((group, result)) logger.debug( - 'Worker %s has finished running tests %s' % - (index, tests)) + f'Worker {index} has finished running tests {tests}') except (KeyboardInterrupt, SystemExit): if isinstance(self, TwistedParallelTestSuiteRunner): # Twisted raises KeyboardInterrupt when the tests @@ -255,21 +249,16 @@ def pop_item(): else: raise except Exception as e: - import traceback - tb = traceback.format_exc() - logger.error(tb) - logger.debug('Running tests failed, reason: %s' % (str(e))) - + logger.debug(f'Running tests failed, reason: {e}') result = TestResult().from_exception(e) results_queue.put((group, result)) except Empty: logger.debug( - 'Worker %s timed out while waiting for tests to run' % - (index)) + f'Worker {index} timed out while waiting for tests to run') finally: tests_queue.close() results_queue.close() - logger.debug('Worker %s is stopping' % (index)) + logger.debug(f'Worker {index} is stopping') def _pre_tests_func(self): # This method gets called before _tests_func is called @@ -289,7 +278,7 @@ def _print_result(self, result): def _exit(self, start_time, end_time, failure_count, error_count): time_difference = (end_time - start_time) - print("Total run time: {} seconds".format(time_difference), file=sys.stderr) + print(f"Total run time: {time_difference} seconds", file=sys.stderr) try: sys.exit(failure_count + error_count) except Exception: @@ -430,8 +419,7 @@ def setup_databases_18(self, **kwargs): worker_index = kwargs.get('worker_index', None) for alias in connections: connection = connections[alias] - database_name = 'test_%d_%s' % ( - worker_index, connection.settings_dict['NAME']) + database_name = f"test_{worker_index}_{connection.settings_dict['NAME']}" connection.settings_dict['TEST_NAME'] = database_name item = test_databases.setdefault( diff --git a/geonode/tests/utils.py b/geonode/tests/utils.py index 7c6109a75e4..2943d21425e 100644 --- a/geonode/tests/utils.py +++ b/geonode/tests/utils.py @@ -104,12 +104,12 @@ def make_request(self, path, data=None, ajax=False, debug=True, force_login=Fals logger.error(f" make_request ----------> url: {url}") if ajax: - url += "{}force_ajax=true".format('&' if '?' in url else '?') + url += f"{('&' if '?' in url else '?')}force_ajax=true" self._session.headers['X_REQUESTED_WITH'] = "XMLHttpRequest" cookie_value = self._session.cookies.get(settings.SESSION_COOKIE_NAME) if force_login and cookie_value: - self.response_cookies += "; {}={}".format(settings.SESSION_COOKIE_NAME, cookie_value) + self.response_cookies += f"; {settings.SESSION_COOKIE_NAME}={cookie_value}" if self.csrf_token: self._session.headers['X-CSRFToken'] = self.csrf_token @@ -172,7 +172,7 @@ def make_request(self, path, data=None, ajax=False, debug=True, force_login=Fals message = '' if hasattr(ex, 'message'): if debug: - logger.error('error in request to %s' % path) + logger.error(f'error in request to {path}') logger.error(ex.message) message = ex.message[ex.message.index(':')+2:] else: @@ -252,7 +252,7 @@ def upload_file(self, _file): try: return resp, resp.json() except ValueError: - logger.exception(ValueError("probably not json, status {}".format(resp.status_code))) + logger.exception(ValueError(f"probably not json, status {resp.status_code}")) return resp, resp.content def get_html(self, path, debug=True): @@ -321,7 +321,7 @@ def get_web_page(url, username=None, password=None, login_url=None): e.args = (msg,) raise except URLError as e: - msg = 'Could not open URL "%s": %s' % (url, e) + msg = f'Could not open URL "{url}": {e}' e.args = (msg,) raise else: @@ -333,9 +333,9 @@ def get_web_page(url, username=None, password=None, login_url=None): def check_layer(uploaded): """Verify if an object is a valid Layer. """ - msg = ('Was expecting layer object, got %s' % (type(uploaded))) + msg = (f'Was expecting layer object, got {type(uploaded)}') assert isinstance(uploaded, Layer), msg - msg = ('The layer does not have a valid name: %s' % uploaded.name) + msg = (f'The layer does not have a valid name: {uploaded.name}') assert len(uploaded.name) > 0, msg diff --git a/geonode/themes/models.py b/geonode/themes/models.py index 86f1f9392c1..37e4c3172b9 100644 --- a/geonode/themes/models.py +++ b/geonode/themes/models.py @@ -38,16 +38,16 @@ class Partner(models.Model): @property def logo_class(self): - _logo_class = slugify("logo_%s" % self.name) - return "{0}".format(_logo_class) + _logo_class = slugify(f"logo_{self.name}") + return str(_logo_class) @property def partner_link(self): - _href = self.href if self.href.startswith('http') else 'http://%s' % self.href - return "{0}".format(_href) + _href = self.href if self.href.startswith('http') else f'http://{self.href}' + return str(_href) def __str__(self): - return "{0}".format(self.title) + return str(self.title) class Meta: ordering = ("name", ) @@ -72,8 +72,9 @@ class JumbotronThemeSlide(models.Model): def __str__(self): get_icon = (lambda arg: '[✓]' if arg else '[✗]') - return '{} | '.format( - self.slide_name, get_icon(self.is_enabled), get_icon(self.hide_jumbotron_slide_content)) + _enabled_icon = get_icon(self.is_enabled) + _slide_content_icon = get_icon(self.hide_jumbotron_slide_content) + return f'{self.slide_name} | ' class GeoNodeThemeCustomization(models.Model): @@ -206,7 +207,7 @@ class GeoNodeThemeCustomization(models.Model): def file_link(self): if self.logo: - return "download" % (self.logo.url,) + return f"download" else: return "No attachment" @@ -215,11 +216,11 @@ def file_link(self): @property def theme_uuid(self): if not self.identifier: - self.identifier = slugify("theme id %s %s" % (self.id, self.date)) - return "{0}".format(self.identifier) + self.identifier = slugify(f"theme id {self.id} {self.date}") + return str(self.identifier) def __str__(self): - return "{0}".format(self.name) + return str(self.name) class Meta: ordering = ("date", ) diff --git a/geonode/themes/tests.py b/geonode/themes/tests.py index a3fc0e4be55..c12a5a03520 100644 --- a/geonode/themes/tests.py +++ b/geonode/themes/tests.py @@ -73,11 +73,11 @@ def test_theme_customization(self): response = self.client.get('/') self.assertNotContains(response, "welcome_2") self.assertContains(response, "welcome_3") - self.assertContains(response, "background: %s;" % theme_3.search_bg_color) - self.assertContains(response, "color: %s;" % theme_3.search_title_color) - self.assertContains(response, "color: %s;" % theme_3.search_link_color) - self.assertContains(response, "color: %s;" % theme_3.footer_text_color) - self.assertContains(response, "color: %s;" % theme_3.footer_href_color) + self.assertContains(response, f"background: {theme_3.search_bg_color};") + self.assertContains(response, f"color: {theme_3.search_title_color};") + self.assertContains(response, f"color: {theme_3.search_link_color};") + self.assertContains(response, f"color: {theme_3.footer_text_color};") + self.assertContains(response, f"color: {theme_3.footer_href_color};") # We should have only one active theme active_themes = GeoNodeThemeCustomization.objects.filter(is_enabled=True) diff --git a/geonode/thumbs/thumbnails.py b/geonode/thumbs/thumbnails.py index fb74be38d4e..8781d3b0073 100644 --- a/geonode/thumbs/thumbnails.py +++ b/geonode/thumbs/thumbnails.py @@ -213,7 +213,7 @@ def _generate_thumbnail_name(instance: Union[Layer, Map]) -> Optional[str]: """ if isinstance(instance, Layer): - file_name = "layer-%s-thumb.png" % instance.uuid + file_name = f"layer-{instance.uuid}-thumb.png" elif isinstance(instance, Map): # if a Map is empty - nothing to do here @@ -221,7 +221,7 @@ def _generate_thumbnail_name(instance: Union[Layer, Map]) -> Optional[str]: logger.debug(f"Thumbnail generation skipped - Map {instance.title} has no defined layers") return None - file_name = "map-%s-thumb.png" % instance.uuid + file_name = f"map-{instance.uuid}-thumb.png" else: raise ThumbnailError( "Thumbnail generation didn't recognize the provided instance: it's neither a Layer nor a Map." diff --git a/geonode/upload/admin.py b/geonode/upload/admin.py index eca61b48d5a..8753e9efec0 100644 --- a/geonode/upload/admin.py +++ b/geonode/upload/admin.py @@ -24,7 +24,7 @@ def import_link(obj): - return "Geoserver Importer Link" % obj.get_import_url() + return f"Geoserver Importer Link" import_link.short_description = 'Link' diff --git a/geonode/upload/files.py b/geonode/upload/files.py index 91a8346a284..5cb7069de52 100644 --- a/geonode/upload/files.py +++ b/geonode/upload/files.py @@ -75,8 +75,8 @@ def all_files(self): return [self.base_file] + self.auxillary_files def __repr__(self): - return "" % ( - self.base_file, self.file_type, self.auxillary_files, self.sld_files, self.xml_files) + return f"" class FileType(object): @@ -114,7 +114,7 @@ def find_auxillary_files(self, base, others): return aux_files, slds, xmls def __repr__(self): - return "" % self.code + return f"" TYPE_UNKNOWN = FileType("unknown", None, None) diff --git a/geonode/upload/models.py b/geonode/upload/models.py index b503ee03e50..e30d512876a 100644 --- a/geonode/upload/models.py +++ b/geonode/upload/models.py @@ -109,14 +109,13 @@ def update_from_session(self, upload_session): self.save() def get_resume_url(self): - return reverse('data_upload') + "?id=%s" % self.import_id + return f"{reverse('data_upload')}?id={self.import_id}" def get_delete_url(self): return reverse('data_upload_delete', args=[self.import_id]) def get_import_url(self): - return "%srest/imports/%s" % ( - ogc_server_settings.LOCATION, self.import_id) + return f"{ogc_server_settings.LOCATION}rest/imports/{self.import_id}" def delete(self, cascade=True): models.Model.delete(self) @@ -134,10 +133,7 @@ def delete(self, cascade=True): shutil.rmtree(self.upload_dir) def __str__(self): - return 'Upload [%s] gs%s - %s, %s' % (self.pk, - self.import_id, - self.name, - self.user) + return f'Upload [{self.pk}] gs{self.import_id} - {self.name}, {self.user}' class UploadFile(models.Model): @@ -146,7 +142,7 @@ class UploadFile(models.Model): slug = models.SlugField(max_length=50, blank=True) def __str__(self): - return "{0}".format(self.slug) + return str(self.slug) def get_absolute_url(self): return reverse('data_upload_new', args=[self.slug, ]) diff --git a/geonode/upload/tests/__init__.py b/geonode/upload/tests/__init__.py index 7cb1e85e918..719b69c6ff1 100644 --- a/geonode/upload/tests/__init__.py +++ b/geonode/upload/tests/__init__.py @@ -93,7 +93,7 @@ def test_scan_file(self): """ exts = ('.shp', '.shx', '.sld', '.xml', '.prj', '.dbf') - with create_files(['san_andres_y_providencia_location{0}'.format(s) for s in exts]) as tests: + with create_files([f'san_andres_y_providencia_location{s}' for s in exts]) as tests: shp = [s for s in tests if s.endswith('.shp')][0] spatial_files = scan_file(shp) self.assertTrue(isinstance(spatial_files, SpatialFiles)) @@ -127,7 +127,7 @@ def test_scan_file(self): basedir = os.path.dirname(spatial_file.base_file) for f in file_names: - path = os.path.join(basedir, '_%s' % f) + path = os.path.join(basedir, f'_{f}') self.assertTrue(os.path.exists(path)) # Test the scan_file function with a raster spatial file takes SLD also. diff --git a/geonode/upload/tests/integration.py b/geonode/upload/tests/integration.py index fafd5e14938..48aa5a4e132 100644 --- a/geonode/upload/tests/integration.py +++ b/geonode/upload/tests/integration.py @@ -71,13 +71,7 @@ DB_NAME = settings.DATABASES['default']['NAME'] DB_USER = settings.DATABASES['default']['USER'] DB_PASSWORD = settings.DATABASES['default']['PASSWORD'] -DATASTORE_URL = 'postgis://{}:{}@{}:{}/{}'.format( - DB_USER, - DB_PASSWORD, - DB_HOST, - DB_PORT, - DB_NAME -) +DATASTORE_URL = f'postgis://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}' postgis_db = dj_database_url.parse(DATASTORE_URL, conn_max_age=0) logging.getLogger('south').setLevel(logging.WARNING) @@ -100,7 +94,7 @@ def get_wms(version='1.1.1', type_name=None, username=None, password=None): # requests. Either we should roll our own or fix owslib if type_name: url = GEOSERVER_URL + \ - '%swms?request=getcapabilities' % type_name.replace(':', '/') + f"{type_name.replace(':', '/')}wms?request=getcapabilities" else: url = GEOSERVER_URL + \ 'wms?request=getcapabilities' @@ -192,7 +186,7 @@ def check_layer_geoserver_caps(self, type_name): type_name=type_name, username=GEOSERVER_USER, password=GEOSERVER_PASSWD) ws, layer_name = type_name.split(':') self.assertTrue(layer_name in wms.contents, - '%s is not in %s' % (layer_name, wms.contents)) + f'{layer_name} is not in {wms.contents}') def check_layer_geoserver_rest(self, layer_name): """ Check that a layer shows up in GeoServer rest api after @@ -223,7 +217,7 @@ def check_save_step(self, resp, data): self.assertEqual(resp.status_code, 200) self.assertTrue(isinstance(data, dict)) # make that the upload returns a success True key - self.assertTrue(data['success'], 'expected success but got %s' % data) + self.assertTrue(data['success'], f'expected success but got {data}') self.assertTrue('redirect_to' in data) def complete_upload(self, file_path, resp, data, is_raster=False): @@ -262,8 +256,7 @@ def finish_upload( if data['success']: self.assertTrue( data['success'], - 'expected success but got %s' % - data) + f'expected success but got {data}') self.assertTrue('redirect_to' in data) current_step = data['redirect_to'] # self.wait_for_progress(data.get('progress')) @@ -289,8 +282,7 @@ def finish_upload( # @todo - make the check match completely (endswith at least) # currently working around potential 'orphaned' db tables self.assertTrue( - layer_name in url, 'expected %s in URL, got %s' % - (layer_name, url)) + layer_name in url, f'expected {layer_name} in URL, got {url}') return url except Exception: return current_step @@ -306,10 +298,9 @@ def check_upload_model(self, original_name): # the import session is COMPLETE if upload and not upload.complete: logger.warning( - "Upload not complete for Layer %s" % - original_name) + f"Upload not complete for Layer {original_name}") except Upload.DoesNotExist: - self.fail('expected to find Upload object for %s' % original_name) + self.fail(f'expected to find Upload object for {original_name}') def check_layer_complete(self, layer_page, original_name): '''check everything to verify the layer is complete''' @@ -335,8 +326,7 @@ def check_layer_complete(self, layer_page, original_name): pass if not caps_found: logger.warning( - "Could not recognize Layer %s on GeoServer WMS Capa" % - original_name) + f"Could not recognize Layer {original_name} on GeoServer WMS Capa") self.check_upload_model(layer_name) def check_invalid_projection(self, layer_name, resp, data): @@ -447,12 +437,12 @@ def test_shp_upload(self): fname = os.path.join( GOOD_DATA, 'vector', - '%s.shp' % layer_name) + f'{layer_name}.shp') self.upload_file(fname, self.complete_upload, - check_name='%s' % layer_name) + check_name=f'{layer_name}') - test_layer = Layer.objects.filter(name__icontains='%s' % layer_name).last() + test_layer = Layer.objects.filter(name__icontains=f'{layer_name}').last() if test_layer: layer_attributes = test_layer.attributes self.assertIsNotNone(layer_attributes) @@ -484,10 +474,8 @@ def test_shp_upload(self): # self.assertIn( # _link_orig.url, # test_layer.csw_anytext, - # "The link URL {0} is not present in the 'csw_anytext' attribute of the layer '{1}'".format( - # _link_orig.url, - # test_layer.alternate - # ) + # f"The link URL {_link_orig.url} is not present in the 'csw_anytext' \ + # attribute of the layer '{test_layer.alternate}'" # ) # Check catalogue catalogue = get_catalogue() @@ -495,9 +483,7 @@ def test_shp_upload(self): self.assertIsNotNone(record) self.assertTrue( hasattr(record, 'links'), - "No records have been found in the catalogue for the resource '{}'".format( - test_layer.alternate - ) + f"No records have been found in the catalogue for the resource '{test_layer.alternate}'" ) # Check 'metadata' links for each record for mime, name, metadata_url in record.links['metadata']: @@ -512,10 +498,7 @@ def test_shp_upload(self): ) self.assertIsNotNone( _post_migrate_link_meta, - "No '{}' links have been found in the catalogue for the resource '{}'".format( - name, - test_layer.alternate - ) + f"No '{name}' links have been found in the catalogue for the resource '{test_layer.alternate}'" ) except Link.DoesNotExist: _post_migrate_link_meta = None @@ -685,7 +668,7 @@ def test_time(self): timedir = os.path.join(GOOD_DATA, 'time') layer_name = 'boxes_with_date' - shp = os.path.join(timedir, '%s.shp' % layer_name) + shp = os.path.join(timedir, f'{layer_name}.shp') # get to time step resp, data = self.client.upload_file(shp) @@ -713,15 +696,13 @@ def test_time(self): self.assertTrue( url.endswith(layer_name), - 'expected url to end with %s, but got %s' % - (layer_name, - url)) + f'expected url to end with {layer_name}, but got {url}') self.assertEqual(resp.status_code, 200) url = unquote(url) self.check_layer_complete(url, layer_name) wms = get_wms( - type_name='geonode:%s' % layer_name, username=GEOSERVER_USER, password=GEOSERVER_PASSWD) + type_name=f'geonode:{layer_name}', username=GEOSERVER_USER, password=GEOSERVER_PASSWD) layer_info = list(wms.items())[0][1] self.assertEqual(100, len(layer_info.timepositions)) else: @@ -733,7 +714,7 @@ def test_configure_time(self): cascading_delete(layer_name=layer_name, catalog=gs_catalog) def get_wms_timepositions(): - alternate_name = 'geonode:%s' % layer_name + alternate_name = f'geonode:{layer_name}' if alternate_name in get_wms().contents: metadata = get_wms().contents[alternate_name] self.assertTrue(metadata is not None) @@ -742,7 +723,7 @@ def get_wms_timepositions(): return None thefile = os.path.join( - GOOD_DATA, 'time', '%s.shp' % layer_name + GOOD_DATA, 'time', f'{layer_name}.shp' ) resp, data = self.client.upload_file(thefile) @@ -775,15 +756,13 @@ def get_wms_timepositions(): self.assertTrue( url.endswith(layer_name), - 'expected url to end with %s, but got %s' % - (layer_name, - url)) + f'expected url to end with {layer_name}, but got {url}') self.assertEqual(resp.status_code, 200) url = unquote(url) self.check_layer_complete(url, layer_name) wms = get_wms( - type_name='geonode:%s' % layer_name, username=GEOSERVER_USER, password=GEOSERVER_PASSWD) + type_name=f'geonode:{layer_name}', username=GEOSERVER_USER, password=GEOSERVER_PASSWD) layer_info = list(wms.items())[0][1] self.assertEqual(100, len(layer_info.timepositions)) else: diff --git a/geonode/upload/tests/test_settings.py b/geonode/upload/tests/test_settings.py index 9bc0fc302c6..4f45988a6b0 100644 --- a/geonode/upload/tests/test_settings.py +++ b/geonode/upload/tests/test_settings.py @@ -17,10 +17,12 @@ # along with this program. If not, see . # ######################################################################### - import os -from urllib.parse import urlparse, urlunparse -from geonode.settings import * +import re +import ast +from datetime import timedelta +from urllib.parse import urlparse +from geonode import settings PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__)) @@ -40,7 +42,7 @@ # add trailing slash to site url. geoserver url will be relative to this if not SITEURL.endswith('/'): - SITEURL = '{}/'.format(SITEURL) + SITEURL = f'{SITEURL}/' ALLOWED_HOSTS = ['localhost', 'geonode', 'django', 'geonode.example.com'] @@ -98,18 +100,20 @@ ) GEOSERVER_PUBLIC_HOST = os.getenv( - 'GEOSERVER_PUBLIC_HOST', SITE_HOST_NAME + 'GEOSERVER_PUBLIC_HOST', settings.SITE_HOST_NAME ) GEOSERVER_PUBLIC_PORT = os.getenv( 'GEOSERVER_PUBLIC_PORT', 8080 ) -_default_public_location = 'http://{}:{}/geoserver/'.format( - GEOSERVER_PUBLIC_HOST, GEOSERVER_PUBLIC_PORT) if GEOSERVER_PUBLIC_PORT else 'http://{}/geoserver/'.format(GEOSERVER_PUBLIC_HOST) +if GEOSERVER_PUBLIC_PORT: + _default_public_location = f'{settings.GEOSERVER_PUBLIC_SCHEMA}://{settings.GEOSERVER_PUBLIC_HOST}:{settings.GEOSERVER_PUBLIC_PORT}/geoserver/' # noqa +else: + _default_public_location = f'{settings.GEOSERVER_PUBLIC_SCHEMA}://{settings.GEOSERVER_PUBLIC_HOST}/geoserver/' GEOSERVER_WEB_UI_LOCATION = os.getenv( - 'GEOSERVER_WEB_UI_LOCATION', GEOSERVER_LOCATION + 'GEOSERVER_WEB_UI_LOCATION', settings.GEOSERVER_LOCATION ) GEOSERVER_PUBLIC_LOCATION = os.getenv( @@ -145,7 +149,7 @@ 'WMST_ENABLED': False, 'BACKEND_WRITE_ENABLED': True, 'WPS_ENABLED': False, - 'LOG_FILE': '%s/geoserver/data/logs/geoserver.log' % os.path.abspath(os.path.join(PROJECT_ROOT, os.pardir)), + 'LOG_FILE': f'{os.path.abspath(os.path.join(PROJECT_ROOT, os.pardir))}/geoserver/data/logs/geoserver.log', # Set to dictionary identifier of database containing spatial data in DATABASES dictionary to enable 'DATASTORE': 'datastore', 'TIMEOUT': int(os.getenv('OGC_REQUEST_TIMEOUT', '60')), @@ -190,8 +194,9 @@ MONITORING_ENABLED = ast.literal_eval(os.environ.get('MONITORING_ENABLED', 'False')) USER_ANALYTICS_ENABLED = ast.literal_eval( os.getenv('USER_ANALYTICS_ENABLED', os.environ.get('MONITORING_ENABLED', 'False'))) -USER_ANALYTICS_GZIP = ast.literal_eval(os.getenv('USER_ANALYTICS_GZIP', - os.environ.get('MONITORING_ENABLED', 'False'))) +USER_ANALYTICS_GZIP = ast.literal_eval( + os.getenv('USER_ANALYTICS_GZIP', + os.environ.get('MONITORING_ENABLED', 'False'))) MONITORING_CONFIG = os.getenv("MONITORING_CONFIG", None) MONITORING_HOST_NAME = os.getenv("MONITORING_HOST_NAME", HOSTNAME) @@ -205,10 +210,10 @@ MONITORING_DISABLE_CSRF = ast.literal_eval(os.environ.get('MONITORING_DISABLE_CSRF', 'False')) if MONITORING_ENABLED: - if 'geonode.monitoring' not in INSTALLED_APPS: - INSTALLED_APPS += ('geonode.monitoring',) - if 'geonode.monitoring.middleware.MonitoringMiddleware' not in MIDDLEWARE: - MIDDLEWARE += \ + if 'geonode.monitoring' not in settings.INSTALLED_APPS: + settings.INSTALLED_APPS += ('geonode.monitoring',) + if 'geonode.monitoring.middleware.MonitoringMiddleware' not in settings.MIDDLEWARE: + settings.MIDDLEWARE += \ ('geonode.monitoring.middleware.MonitoringMiddleware',) # skip certain paths to not to mud stats too much @@ -216,8 +221,8 @@ '/monitoring/', '/admin', '/jsi18n', - STATIC_URL, - MEDIA_URL, + settings.STATIC_URL, + settings.MEDIA_URL, re.compile('^/[a-z]{2}/admin/'), ) @@ -232,7 +237,7 @@ (timedelta(days=14), timedelta(days=1),), ) - CELERY_BEAT_SCHEDULE['collect_metrics'] = { + settings.CELERY_BEAT_SCHEDULE['collect_metrics'] = { 'task': 'geonode.monitoring.tasks.collect_metrics', 'schedule': 60.0, } diff --git a/geonode/upload/tests/test_upload_preprocessing.py b/geonode/upload/tests/test_upload_preprocessing.py index 631169c5d1f..26891556021 100644 --- a/geonode/upload/tests/test_upload_preprocessing.py +++ b/geonode/upload/tests/test_upload_preprocessing.py @@ -57,18 +57,18 @@ def test_preprocess_files_kml_ground_overlay(self, mock_handler): def test_extract_bbox_param(self): fake_north = "70.000" - kml_bytes = """ + kml_bytes = f""" - {} + {fake_north} - """.format(fake_north).strip() + """.strip() kml_doc, ns = get_kml_doc(kml_bytes) result = upload_preprocessing._extract_bbox_param( kml_doc, ns, "north") diff --git a/geonode/upload/upload.py b/geonode/upload/upload.py index c57c29c2906..dc605c54728 100644 --- a/geonode/upload/upload.py +++ b/geonode/upload/upload.py @@ -149,7 +149,7 @@ def __init__(self, **kw): if hasattr(self, k): setattr(self, k, v) else: - raise Exception('not handled : %s' % k) + raise Exception(f'not handled : {k}') def cleanup(self): """do what we should at the given state of the upload""" @@ -259,10 +259,9 @@ def _check_geoserver_store(store_name, layer_type, overwrite): _("Name already in use and overwrite is False")) existing_type = resource.resource_type if existing_type != layer_type: - msg = ("Type of uploaded file {} ({}) does not " + msg = (f"Type of uploaded file {store_name} ({layer_type}) does not " "match type of existing resource type " - "{}".format(store_name, layer_type, - existing_type)) + f"{existing_type}") logger.error(msg) raise GeoNodeException(msg) @@ -283,7 +282,7 @@ def save_step(user, layer, spatial_files, overwrite=True, mosaic=False, time_presentation_reference_value=None, charset_encoding="UTF-8"): logger.debug( - 'Uploading layer: {}, files {!r}'.format(layer, spatial_files)) + f'Uploading layer: {layer}, files {spatial_files}') if len(spatial_files) > 1: # we only support more than one file if they're rasters for mosaicing if not all( @@ -291,7 +290,7 @@ def save_step(user, layer, spatial_files, overwrite=True, mosaic=False, raise UploadException( "Please upload only one type of file at a time") name = get_valid_layer_name(layer, overwrite) - logger.debug('Name for layer: {!r}'.format(name)) + logger.debug(f'Name for layer: {name}') if not any(spatial_files.all_files()): raise UploadException("Unable to recognize the uploaded file(s)") the_layer_type = _get_layer_type(spatial_files) @@ -300,10 +299,10 @@ def save_step(user, layer, spatial_files, overwrite=True, mosaic=False, FeatureType.resource_type, Coverage.resource_type): raise RuntimeError("Expected layer type to FeatureType or " - "Coverage, not {}".format(the_layer_type)) + f"Coverage, not {the_layer_type}") files_to_upload = preprocess_files(spatial_files) - logger.debug("files_to_upload: {}".format(files_to_upload)) - logger.debug('Uploading {}'.format(the_layer_type)) + logger.debug(f"files_to_upload: {files_to_upload}") + logger.debug(f'Uploading {the_layer_type}') error_msg = None try: next_id = _get_next_id() @@ -508,7 +507,7 @@ def build_att_remap_transform(att): ) if transforms: - logger.debug('Setting transforms %s' % transforms) + logger.debug(f'Setting transforms {transforms}') upload_session.import_session.tasks[0].add_transforms(transforms) try: upload_session.time_transforms = transforms @@ -578,7 +577,7 @@ def final_step(upload_session, user, charset="UTF-8"): if import_session.state == 'INCOMPLETE': if task.state != 'ERROR': - raise Exception('unknown item state: %s' % task.state) + raise Exception(f'unknown item state: {task.state}') elif import_session.state == 'READY': import_session.commit() elif import_session.state == 'PENDING': @@ -588,8 +587,7 @@ def final_step(upload_session, user, charset="UTF-8"): if not publishing: raise LayerNotReady( - "Expected to find layer named '%s' in geoserver" % - name) + f"Expected to find layer named '{name}' in geoserver") _log('Creating Django record for [%s]', name) target = task.target @@ -749,8 +747,7 @@ def _store_file(saved_layer, name=file_name, base=base, file=File( - f, name='%s%s' % - (assigned_name or saved_layer.name, type_name))) + f, name=f'{assigned_name or saved_layer.name}{type_name}')) # save the system assigned name for the remaining files if not assigned_name: the_file = geonode_upload_session.layerfile_set.all()[0].file.name diff --git a/geonode/upload/upload_preprocessing.py b/geonode/upload/upload_preprocessing.py index 0963166d975..e8fce07d651 100644 --- a/geonode/upload/upload_preprocessing.py +++ b/geonode/upload/upload_preprocessing.py @@ -110,6 +110,6 @@ def preprocess_files(spatial_files): def _extract_bbox_param(kml_doc, namespaces, param): return kml_doc.xpath( "kml:Document/kml:GroundOverlay/kml:LatLonBox/" - "kml:{}/text()".format(param), + f"kml:{param}/text()", namespaces=namespaces )[0] diff --git a/geonode/upload/upload_validators.py b/geonode/upload/upload_validators.py index 24160da4183..6d0e7ba3905 100644 --- a/geonode/upload/upload_validators.py +++ b/geonode/upload/upload_validators.py @@ -46,7 +46,7 @@ def _supported_type(ext, supported_types): def validate_uploaded_files(cleaned, uploaded_files, field_spatial_types): - logger.debug("uploaded_files: {}".format(uploaded_files)) + logger.debug(f"uploaded_files: {uploaded_files}") requires_datastore = () if ogc_server_settings.DATASTORE else ( 'csv', 'kml') @@ -152,11 +152,10 @@ def _validate_shapefile_components(possible_filenames): else: if additional_component.mandatory: raise forms.ValidationError( - "Could not find {!r} file, which is mandatory for " - "shapefile uploads".format( - additional_component.extension) + f"Could not find {additional_component.extension} file, which is mandatory for " + "shapefile uploads" ) - logger.debug("shapefile components: {}".format(components)) + logger.debug(f"shapefile components: {components}") return components @@ -174,8 +173,8 @@ def _validate_kml_bytes(kml_bytes, other_files): "kml:Icon/kml:href/text()", namespaces=namespaces)[0].strip() except IndexError: image_path = "" - logger.debug("image_path: {}".format(image_path)) - logger.debug("other_files: {}".format(other_files)) + logger.debug(f"image_path: {image_path}") + logger.debug(f"other_files: {other_files}") if image_path not in other_files: raise forms.ValidationError( _("Ground overlay image declared in kml file cannot be found")) @@ -259,10 +258,10 @@ def dupes(_a): valid_extensions = None raster_types = [t for t in files.types if t.layer_type == files.raster] - raster_exts = [".%s" % t.code for t in raster_types] + raster_exts = [f".{t.code}" for t in raster_types] raster_aliases = [] for alias in [aliases for aliases in [t.aliases for t in raster_types] if aliases]: - raster_aliases.extend([".%s" % a for a in alias]) + raster_aliases.extend([f".{a}" for a in alias]) raster_exts.extend(raster_aliases) raster_files = [ diff --git a/geonode/upload/utils.py b/geonode/upload/utils.py index ae209f3766e..5939c8a0b72 100644 --- a/geonode/upload/utils.py +++ b/geonode/upload/utils.py @@ -135,7 +135,7 @@ def error_response(req, exception=None, errors=None, force_ajax=True): if exception: logger.exception('Unexpected error in upload step') else: - logger.error('upload error: %s', errors) + logger.error(f'upload error: {errors}') if req.is_ajax() or force_ajax: content_type = 'text/html' if not req.is_ajax() else None return json_response(exception=exception, errors=errors, @@ -146,7 +146,7 @@ def error_response(req, exception=None, errors=None, force_ajax=True): return render( req, 'upload/layer_upload_error.html', - context={'error_msg': 'Unexpected error : %s,' % exception}) + context={'error_msg': f'Unexpected error : {exception}'}) def json_load_byteified(file_handle): @@ -257,7 +257,7 @@ def get_next_step(upload_session, offset=1): try: pages = _pages[upload_session.upload_type] except KeyError as e: - raise Exception(_('Unsupported file type: %s' % e.message)) + raise Exception(_(f'Unsupported file type: {e.message}')) index = -1 if upload_session.completed_step and upload_session.completed_step != 'save': index = pages.index(upload_session.completed_step) @@ -299,7 +299,7 @@ def next_step_response(req, upload_session, force_ajax=True): {'status': 'error', 'success': False, 'id': import_session.id, - 'error_msg': "%s" % upload_session.error_msg, + 'error_msg': str(upload_session.error_msg), } ) @@ -310,13 +310,13 @@ def next_step_response(req, upload_session, force_ajax=True): upload_session.completed_step = 'check' return next_step_response(req, upload_session, force_ajax=True) if next == 'check' and force_ajax: - url = reverse('data_upload') + "?id=%s" % (import_session.id) + url = f"{reverse('data_upload')}?id={import_session.id}" return json_response( {'url': url, 'status': 'incomplete', 'success': True, 'id': import_session.id, - 'redirect_to': settings.SITEURL + 'upload/check' + "?id=%s%s" % (import_session.id, _force_ajax), + 'redirect_to': f"{settings.SITEURL}upload/check?id={import_session.id}{_force_ajax}", } ) @@ -335,46 +335,46 @@ def next_step_response(req, upload_session, force_ajax=True): upload_session.completed_step = 'time' return next_step_response(req, upload_session, force_ajax) if next == 'time' and force_ajax: - url = reverse('data_upload') + "?id=%s" % (import_session.id) + url = f"{reverse('data_upload')}?id={import_session.id}" return json_response( {'url': url, 'status': 'incomplete', 'success': True, 'id': import_session.id, - 'redirect_to': settings.SITEURL + 'upload/time' + "?id=%s%s" % (import_session.id, _force_ajax), + 'redirect_to': f"{settings.SITEURL}upload/time?id={import_session.id}{_force_ajax}", } ) if next == 'mosaic' and force_ajax: - url = reverse('data_upload') + "?id=%s" % (import_session.id) + url = f"{reverse('data_upload')}?id={import_session.id}" return json_response( {'url': url, 'status': 'incomplete', 'success': True, 'id': import_session.id, - 'redirect_to': settings.SITEURL + 'upload/mosaic' + "?id=%s%s" % (import_session.id, _force_ajax), + 'redirect_to': f"{settings.SITEURL}upload/mosaic?id={import_session.id}{_force_ajax}", } ) if next == 'srs' and force_ajax: - url = reverse('data_upload') + "?id=%s" % (import_session.id) + url = f"{reverse('data_upload')}?id={import_session.id}" return json_response( {'url': url, 'status': 'incomplete', 'success': True, 'id': import_session.id, - 'redirect_to': settings.SITEURL + 'upload/srs' + "?id=%s%s" % (import_session.id, _force_ajax), + 'redirect_to': f"{settings.SITEURL}upload/srs?id={import_session.id}{_force_ajax}", } ) if next == 'csv' and force_ajax: - url = reverse('data_upload') + "?id=%s" % (import_session.id) + url = f"{reverse('data_upload')}?id={import_session.id}" return json_response( {'url': url, 'status': 'incomplete', 'success': True, 'id': import_session.id, - 'redirect_to': settings.SITEURL + 'upload/csv' + "?id=%s%s" % (import_session.id, _force_ajax), + 'redirect_to': f"{settings.SITEURL}upload/csv?id={import_session.id}{_force_ajax}", } ) @@ -391,9 +391,9 @@ def next_step_response(req, upload_session, force_ajax=True): force_ajax=force_ajax) session_id = None if 'id' in req.GET: - session_id = "?id=%s" % (req.GET['id']) + session_id = f"?id={req.GET['id']}" elif import_session and import_session.id: - session_id = "?id=%s" % (import_session.id) + session_id = f"?id={import_session.id}" if req.is_ajax() or force_ajax: content_type = 'text/html' if not req.is_ajax() else None @@ -444,9 +444,8 @@ def check_import_session_is_valid(request, upload_session, import_session): layer = import_session.tasks[0].layer invalid = [a for a in layer.attributes if str(a.name).find(' ') >= 0] if invalid: - att_list = "
%s
" % '. '.join( - [a.name for a in invalid]) - msg = "Attributes with spaces are not supported : %s" % att_list + att_list = f"
{'. '.join([a.name for a in invalid])}
" + msg = f"Attributes with spaces are not supported : {att_list}" upload_session.completed_step = 'error' upload_session.error_msg = msg return layer @@ -522,7 +521,7 @@ def _fixup_base_file(absolute_base_file, tempdir=None): if os.path.exists(absolute_base_file): return absolute_base_file else: - raise Exception(_('File does not exist: %s' % absolute_base_file)) + raise Exception(_(f'File does not exist: {absolute_base_file}')) def _get_layer_values(layer, upload_session, expand=0): @@ -579,16 +578,14 @@ def run_import(upload_session, async_upload=_ASYNC_UPLOAD): import_execution_requested = False if import_session.state == 'INCOMPLETE': if task.state != 'ERROR': - raise Exception(_('unknown item state: %s' % task.state)) + raise Exception(_(f'unknown item state: {task.state}')) elif import_session.state == 'PENDING' and task.target.store_type == 'coverageStore': if task.state == 'READY': import_session.commit(async_upload) import_execution_requested = True if task.state == 'ERROR': progress = task.get_progress() - raise Exception(_( - 'error during import: %s' % - progress.get('message'))) + raise Exception(_(f"error during import: {progress.get('message')}")) # if a target datastore is configured, ensure the datastore exists # in geoserver and set the uploader target appropriately @@ -599,15 +596,13 @@ def run_import(upload_session, async_upload=_ASYNC_UPLOAD): workspace=settings.DEFAULT_WORKSPACE ) _log( - 'setting target datastore %s %s', - target.name, - target.workspace.name) + f'setting target datastore {target.name} {target.workspace.name}') task.set_target(target.name, target.workspace.name) else: target = task.target if upload_session.update_mode: - _log('setting updateMode to %s', upload_session.update_mode) + _log(f'setting updateMode to {upload_session.update_mode}') task.set_update_mode(upload_session.update_mode) _log('running import session') @@ -624,8 +619,8 @@ def progress_redirect(step, upload_id): return json_response(dict( success=True, id=upload_id, - redirect_to=reverse('data_upload', args=[step]) + "?id=%s" % upload_id, - progress=reverse('data_upload_progress') + "?id=%s" % upload_id + redirect_to=f"{reverse('data_upload', args=[step])}?id={upload_id}", + progress=f"{reverse('data_upload_progress')}?id={upload_id}" )) diff --git a/geonode/upload/views.py b/geonode/upload/views.py index dca7d66d00b..e00a819c4cf 100644 --- a/geonode/upload/views.py +++ b/geonode/upload/views.py @@ -165,24 +165,24 @@ def save_step_view(req, session): form = LayerUploadForm(req.POST, req.FILES) if form.is_valid(): tempdir = tempfile.mkdtemp(dir=settings.STATIC_ROOT) - logger.debug("valid_extensions: {}".format(form.cleaned_data["valid_extensions"])) + logger.debug(f"valid_extensions: {form.cleaned_data['valid_extensions']}") relevant_files = _select_relevant_files( form.cleaned_data["valid_extensions"], iter(req.FILES.values()) ) - logger.debug("relevant_files: {}".format(relevant_files)) + logger.debug(f"relevant_files: {relevant_files}") _write_uploaded_files_to_disk(tempdir, relevant_files) base_file = os.path.join(tempdir, form.cleaned_data["base_file"].name) name, ext = os.path.splitext(os.path.basename(base_file)) - logger.debug('Name: {0}, ext: {1}'.format(name, ext)) - logger.debug("base_file: {}".format(base_file)) + logger.debug(f'Name: {name}, ext: {ext}') + logger.debug(f"base_file: {base_file}") scan_hint = get_scan_hint(form.cleaned_data["valid_extensions"]) spatial_files = scan_file( base_file, scan_hint=scan_hint, charset=form.cleaned_data["charset"] ) - logger.debug("spatial_files: {}".format(spatial_files)) + logger.debug(f"spatial_files: {spatial_files}") import_session = save_step( req.user, name, @@ -211,7 +211,7 @@ def save_step_view(req, session): form.cleaned_data["charset"], tempdir=tempdir) - _log('provided sld is %s' % sld) + _log(f'provided sld is {sld}') # upload_type = get_upload_type(base_file) upload_session = UploaderSession( tempdir=tempdir, @@ -308,7 +308,7 @@ def srs_step_view(request, upload_session): {'status': 'error', 'success': False, 'id': upload_session.import_session.id, - 'error_msg': "%s" % error, + 'error_msg': f"{error}", } ) else: @@ -402,7 +402,7 @@ def csv_step_view(request, upload_session): {'status': 'error', 'success': False, 'id': upload_session.import_session.id, - 'error_msg': "%s" % error, + 'error_msg': f"{error}", } ) else: @@ -596,7 +596,7 @@ def final_step_view(req, upload_session): 'status': 'pending', 'success': True, 'id': req.GET['id'], - 'redirect_to': '/upload/final' + "?id=%s%s" % (req.GET['id'], force_ajax) + 'redirect_to': f"/upload/final?id={req.GET['id']}{force_ajax}" } ) except Exception as e: diff --git a/geonode/urls.py b/geonode/urls.py index a110b5ef091..f4f8a280e59 100644 --- a/geonode/urls.py +++ b/geonode/urls.py @@ -227,7 +227,7 @@ ] if settings.NOTIFICATIONS_MODULE in settings.INSTALLED_APPS: - notifications_urls = '{}.urls'.format(settings.NOTIFICATIONS_MODULE) + notifications_urls = f'{settings.NOTIFICATIONS_MODULE}.urls' urlpatterns += [ # '', url(r'^notifications/', include(notifications_urls)), ] diff --git a/geonode/utils.py b/geonode/utils.py index df6d050854a..3bb4fad89f1 100755 --- a/geonode/utils.py +++ b/geonode/utils.py @@ -211,7 +211,7 @@ def get_headers(request, url, raw_url, allowed_hosts=[]): value = request.COOKIES.get(name) if name == 'csrftoken': csrftoken = value - cook = "%s=%s" % (name, value) + cook = f"{name}={value}" cookies = cook if not cookies else (cookies + '; ' + cook) csrftoken = get_token(request) if not csrftoken else csrftoken @@ -219,7 +219,7 @@ def get_headers(request, url, raw_url, allowed_hosts=[]): if csrftoken: headers['X-Requested-With'] = "XMLHttpRequest" headers['X-CSRFToken'] = csrftoken - cook = "%s=%s" % ('csrftoken', csrftoken) + cook = f"csrftoken={csrftoken}" cookies = cook if not cookies else (cookies + '; ' + cook) if cookies: @@ -254,18 +254,19 @@ def get_headers(request, url, raw_url, allowed_hosts=[]): access_token = get_or_create_token(request.user) if access_token: - headers['Authorization'] = 'Bearer %s' % access_token + headers['Authorization'] = f'Bearer {access_token}' pragma = "no-cache" referer = request.META[ "HTTP_REFERER"] if "HTTP_REFERER" in request.META else \ - "{scheme}://{netloc}/".format(scheme=site_url.scheme, - netloc=site_url.netloc) + f"{site_url.scheme}://{site_url.netloc}/" encoding = request.META["HTTP_ACCEPT_ENCODING"] if "HTTP_ACCEPT_ENCODING" in request.META else "gzip" - headers.update({"Pragma": pragma, - "Referer": referer, - "Accept-encoding": encoding, - }) + headers.update( + { + "Pragma": pragma, + "Referer": referer, + "Accept-encoding": encoding, + }) return (headers, access_token) @@ -327,11 +328,11 @@ def bbox_to_wkt(x0, x1, y0, y1, srid="4326", include_srid=True): float(x1), float(y0), float(x0), float(y0)) if include_srid: - wkt = 'SRID=%s;%s' % (srid, wkt) + wkt = f'SRID={srid};{wkt}' else: wkt = 'POLYGON((-180 -90,-180 90,180 90,180 -90,-180 -90))' if include_srid: - wkt = 'SRID=4326;%s' % wkt + wkt = f'SRID=4326;{wkt}' return wkt @@ -390,10 +391,9 @@ def bbox_to_projection(native_bbox, target_srid=4326): projected_bbox = [str(x) for x in g.GetEnvelope()] # Must be in the form : [x0, x1, y0, y1, EPSG:) return tuple([projected_bbox[0], projected_bbox[1], projected_bbox[2], projected_bbox[3]]) + \ - ("EPSG:%s" % target_srid,) - except Exception: - tb = traceback.format_exc() - logger.error(tb) + (f"EPSG:{target_srid}",) + except Exception as e: + logger.exception(e) return native_bbox @@ -652,7 +652,7 @@ def _base_source(source): 'remote': True, 'ptype': service.ptype, 'name': service.name, - 'title': "[R] %s" % service.title + 'title': f"[R] {service.title}" } if remote_source['url'] not in source_urls: index += 1 @@ -1012,7 +1012,7 @@ def json_response(body=None, errors=None, url=None, redirect_to=None, exception= } elif exception: if body is None: - body = "Unexpected exception %s" % exception + body = f"Unexpected exception {exception}" else: body = body % exception body = { @@ -1070,8 +1070,7 @@ def format_urls(a, values): def build_abstract(resourcebase, url=None, includeURL=True): if resourcebase.abstract and url and includeURL: - return "{abstract} -- [{url}]({url})".format( - abstract=resourcebase.abstract, url=url) + return f"{resourcebase.abstract} -- [{url}]({url})" else: return resourcebase.abstract @@ -1091,10 +1090,10 @@ def build_caveats(resourcebase): def build_social_links(request, resourcebase): - social_url = "{protocol}://{host}{path}".format( - protocol=("https" if request.is_secure() else "http"), - host=request.get_host(), - path=request.get_full_path()) + netschema = ("https" if request.is_secure() else "http") + host = request.get_host() + path = request.get_full_path() + social_url = f"{netschema}://{host}{path}" # Don't use datetime strftime() because it requires year >= 1900 # see # https://docs.python.org/2/library/datetime.html#strftime-strptime-behavior @@ -1173,7 +1172,7 @@ def fixup_shp_columnnames(inShapefile, charset, tempdir=None): inDataSource = None if inDataSource is None: - logger.debug("Could not open {}".format(inShapefile)) + logger.debug(f"Could not open {inShapefile}") return False, None, None else: inLayer = inDataSource.GetLayer() @@ -1233,7 +1232,7 @@ def fixup_shp_columnnames(inShapefile, charset, tempdir=None): except Exception as e: logger.exception(e) raise GeoNodeException( - "Could not decode SHAPEFILE attributes by using the specified charset '{}'.".format(charset)) + f"Could not decode SHAPEFILE attributes by using the specified charset '{charset}'.") return True, None, list_col @@ -1249,7 +1248,7 @@ def id_to_obj(id_): def printsignals(): for signalname in signalnames: - logger.debug("SIGNALNAME: %s" % signalname) + logger.debug(f"SIGNALNAME: {signalname}") signaltype = getattr(models.signals, signalname) signals = signaltype.receivers[:] for signal in signals: @@ -1332,10 +1331,9 @@ def parse_datetime(value): else: return datetime.datetime.strptime(value, patt) except Exception: - # tb = traceback.format_exc() - # logger.error(tb) - pass - raise ValueError("Invalid datetime input: {}".format(value)) + tb = traceback.format_exc() + logger.debug(tb) + raise ValueError(f"Invalid datetime input: {value}") def _convert_sql_params(cur, query): @@ -1436,14 +1434,14 @@ def request(self, url, method='GET', data=None, headers={}, stream=False, _u = user or get_user_model().objects.get(username=self.username) access_token = get_or_create_token(_u) if access_token and not access_token.is_expired(): - headers['Authorization'] = 'Bearer %s' % access_token.token + headers['Authorization'] = f'Bearer {access_token.token}' except Exception: tb = traceback.format_exc() logger.debug(tb) elif user == self.username: valid_uname_pw = base64.b64encode( - "{}:{}".format(self.username, self.password).encode()).decode() - headers['Authorization'] = 'Basic {}'.format(valid_uname_pw) + f"{self.username}:{self.password}".encode()).decode() + headers['Authorization'] = f'Basic {valid_uname_pw}' response = None content = None @@ -1460,7 +1458,8 @@ def request(self, url, method='GET', data=None, headers={}, stream=False, pool_maxsize=self.pool_maxsize, pool_connections=self.pool_connections ) - session.mount("{scheme}://".format(scheme=urlsplit(url).scheme), adapter) + scheme = urlsplit(url).scheme + session.mount(f"{scheme}://", adapter) session.verify = False action = getattr(session, method.lower(), None) if action: @@ -1579,14 +1578,14 @@ def chmod_tree(dst, permissions=0o777): os.chmod(path, permissions) status = os.stat(path) if oct(status.st_mode & 0o777) != str(oct(permissions)): - raise Exception("Could not update permissions of {}".format(path)) + raise Exception(f"Could not update permissions of {path}") for dirname in dirnames: path = os.path.join(dirpath, dirname) os.chmod(path, permissions) status = os.stat(path) if oct(status.st_mode & 0o777) != str(oct(permissions)): - raise Exception("Could not update permissions of {}".format(path)) + raise Exception(f"Could not update permissions of {path}") def slugify_zh(text, separator='_'): @@ -1778,8 +1777,7 @@ def set_resource_default_links(instance, layer, prune=False, **kwargs): ) site_url = settings.SITEURL.rstrip('/') if settings.SITEURL.startswith('http') else settings.SITEURL - html_link_url = '%s%s' % ( - site_url, instance.get_absolute_url()) + html_link_url = f'{site_url}{instance.get_absolute_url()}' if (Link.objects.filter(resource=instance.resourcebase_ptr, url=html_link_url, @@ -1846,7 +1844,7 @@ def set_resource_default_links(instance, layer, prune=False, **kwargs): # ogc_wms_path = '%s/ows' % instance.workspace ogc_wms_path = 'ows' ogc_wms_url = urljoin(ogc_server_settings.public_url, ogc_wms_path) - ogc_wms_name = 'OGC WMS: %s Service' % instance.workspace + ogc_wms_name = f'OGC WMS: {instance.workspace} Service' if Link.objects.filter(resource=instance.resourcebase_ptr, name=ogc_wms_name, url=ogc_wms_url).count() < 2: Link.objects.get_or_create( resource=instance.resourcebase_ptr, @@ -1864,7 +1862,7 @@ def set_resource_default_links(instance, layer, prune=False, **kwargs): # ogc_wfs_path = '%s/wfs' % instance.workspace ogc_wfs_path = 'ows' ogc_wfs_url = urljoin(ogc_server_settings.public_url, ogc_wfs_path) - ogc_wfs_name = 'OGC WFS: %s Service' % instance.workspace + ogc_wfs_name = f'OGC WFS: {instance.workspace} Service' if Link.objects.filter(resource=instance.resourcebase_ptr, name=ogc_wfs_name, url=ogc_wfs_url).count() < 2: Link.objects.get_or_create( resource=instance.resourcebase_ptr, @@ -1882,7 +1880,7 @@ def set_resource_default_links(instance, layer, prune=False, **kwargs): # ogc_wcs_path = '%s/wcs' % instance.workspace ogc_wcs_path = 'ows' ogc_wcs_url = urljoin(ogc_server_settings.public_url, ogc_wcs_path) - ogc_wcs_name = 'OGC WCS: %s Service' % instance.workspace + ogc_wcs_name = f'OGC WCS: {instance.workspace} Service' if Link.objects.filter(resource=instance.resourcebase_ptr, name=ogc_wcs_name, url=ogc_wcs_url).count() < 2: Link.objects.get_or_create( resource=instance.resourcebase_ptr, diff --git a/geonode/version.py b/geonode/version.py index f0b851a88df..f389ab3cfb4 100644 --- a/geonode/version.py +++ b/geonode/version.py @@ -43,9 +43,9 @@ def get_version(version=None): sub = mapping[version[3]] + str(version[4]) if git_changeset: if version[3] == 'unstable': - sub += '.dev%s' % git_changeset + sub += f'.dev{git_changeset}' elif version[3] != 'final': - sub += '.build%s' % git_changeset + sub += f'.build{git_changeset}' return main + sub diff --git a/geonode/views.py b/geonode/views.py index 72cf604cef6..78605ec343f 100644 --- a/geonode/views.py +++ b/geonode/views.py @@ -147,7 +147,7 @@ def ident_json(request): json_data['poc'] = { 'name': settings.PYCSW['CONFIGURATION']['metadata:main']['contact_name'], 'email': settings.PYCSW['CONFIGURATION']['metadata:main']['contact_email'], - 'twitter': 'https://twitter.com/%s' % settings.TWITTER_SITE + 'twitter': f'https://twitter.com/{settings.TWITTER_SITE}' } json_data['version'] = get_version() diff --git a/pavement.py b/pavement.py index d246be4e368..1837b2cd11a 100644 --- a/pavement.py +++ b/pavement.py @@ -81,7 +81,7 @@ try: from geonode.settings import TEST_RUNNER_KEEPDB, TEST_RUNNER_PARALLEL _keepdb = '--keepdb' if TEST_RUNNER_KEEPDB else '' - _parallel = ('--parallel=%s' % TEST_RUNNER_PARALLEL) if TEST_RUNNER_PARALLEL else '' + _parallel = f'--parallel={TEST_RUNNER_PARALLEL}' if TEST_RUNNER_PARALLEL else '' except Exception: _keepdb = '' _parallel = '' @@ -102,9 +102,9 @@ def grab(src, dest, name): logger.info(f" src, dest, name --> {src} {dest} {name}") if not os.path.exists(dest): - logger.info("Downloading {}".format(name)) + logger.info(f"Downloading {name}") elif not zipfile.is_zipfile(dest): - logger.info("Downloading {} (corrupt file)".format(name)) + logger.info(f"Downloading {name} (corrupt file)") elif not src.startswith("file://"): r = requests.head(src) file_time = datetime.datetime.fromtimestamp(os.path.getmtime(dest)) @@ -116,18 +116,18 @@ def grab(src, dest, name): utc = pytz.utc url_date = url_date.replace(tzinfo=utc) file_time = file_time.replace(tzinfo=utc) - if url_date < file_time : + if url_date < file_time: # Do not download if older than the local one return - logger.info("Downloading updated {}".format(name)) + logger.info(f"Downloading updated {name}") # Local file does not exist or remote one is newer if src.startswith("file://"): src2 = src.replace("file://", '') if not os.path.exists(src2): - logger.info("Source location ({}) does not exist".format(src2)) + logger.info(f"Source location ({src2}) does not exist") else: - logger.info("Copying local file from {}".format(src2)) + logger.info(f"Copying local file from {src2}") shutil.copyfile(src2, dest) else: # urlretrieve(str(src), str(dest)) @@ -135,7 +135,7 @@ def grab(src, dest, name): r = requests.get(src, stream=True, timeout=10, verify=False) # Total size in bytes. total_size = int(r.headers.get('content-length', 0)) - logger.info("Requesting {}".format(src)) + logger.info(f"Requesting {src}") block_size = 1024 wrote = 0 with open("output.bin", 'wb') as f: @@ -146,7 +146,7 @@ def grab(src, dest, name): unit_scale=False): wrote += len(data) f.write(data) - logger.info(" total_size [{}] / wrote [{}] ".format(total_size, wrote)) + logger.info(f" total_size [{total_size}] / wrote [{wrote}] ") if total_size != 0 and wrote != total_size: logger.error("ERROR, something went wrong. Data could not be written. Expected to write " + wrote + " but wrote " + total_size + " instead") @@ -183,7 +183,7 @@ def setup_geoserver(options): jetty_runner = download_dir / \ os.path.basename(urlparse(dev_config['JETTY_RUNNER_URL']).path) geoserver_data = download_dir / \ - os.path.basename(urlparse(dev_config['DATA_DIR_URL']).path) + os.path.basename(urlparse(dev_config['DATA_DIR_URL']).path) grab( options.get( 'geoserver', @@ -234,7 +234,7 @@ def _robust_rmtree(path, logger=None, max_retries=5): return except OSError: if logger: - info('Unable to remove path: %s' % path) + info(f'Unable to remove path: {path}') info('Retrying after %d seconds' % i) time.sleep(i) @@ -374,7 +374,7 @@ def upgradedb(options): elif version is None: print("Please specify your GeoNode version") else: - print("Upgrades from version {} are not yet supported.".format(version)) + print(f"Upgrades from version {version} are not yet supported.") @task @@ -387,9 +387,9 @@ def updategeoip(options): """ settings = options.get('settings', '') if settings and 'DJANGO_SETTINGS_MODULE' not in settings: - settings = 'DJANGO_SETTINGS_MODULE=%s' % settings + settings = f'DJANGO_SETTINGS_MODULE={settings}' - sh("%s python -W ignore manage.py updategeoip -o" % settings) + sh(f"{settings} python -W ignore manage.py updategeoip -o") @task @@ -402,17 +402,17 @@ def sync(options): """ settings = options.get('settings', '') if settings and 'DJANGO_SETTINGS_MODULE' not in settings: - settings = 'DJANGO_SETTINGS_MODULE=%s' % settings + settings = f'DJANGO_SETTINGS_MODULE={settings}' - sh("%s python -W ignore manage.py makemigrations --noinput" % settings) - sh("%s python -W ignore manage.py migrate --noinput" % settings) - sh("%s python -W ignore manage.py loaddata sample_admin.json" % settings) - sh("%s python -W ignore manage.py loaddata geonode/base/fixtures/default_oauth_apps.json" % settings) - sh("%s python -W ignore manage.py loaddata geonode/base/fixtures/initial_data.json" % settings) + sh(f"{settings} python -W ignore manage.py makemigrations --noinput") + sh(f"{settings} python -W ignore manage.py migrate --noinput") + sh(f"{settings} python -W ignore manage.py loaddata sample_admin.json") + sh(f"{settings} python -W ignore manage.py loaddata geonode/base/fixtures/default_oauth_apps.json") + sh(f"{settings} python -W ignore manage.py loaddata geonode/base/fixtures/initial_data.json") if 'django_celery_beat' in INSTALLED_APPS: - sh("%s python -W ignore manage.py loaddata geonode/base/fixtures/django_celery_beat.json" % settings) - sh("%s python -W ignore manage.py set_all_layers_alternate" % settings) - sh("%s python -W ignore manage.py collectstatic --noinput" % settings) + sh(f"{settings} python -W ignore manage.py loaddata geonode/base/fixtures/django_celery_beat.json") + sh(f"{settings} python -W ignore manage.py set_all_layers_alternate") + sh(f"{settings} python -W ignore manage.py collectstatic --noinput") @task @@ -425,11 +425,11 @@ def package(options): version = geonode.get_version() # Use GeoNode's version for the package name. - pkgname = 'GeoNode-%s-all' % version + pkgname = f'GeoNode-{version}-all' # Create the output directory. out_pkg = path(pkgname) - out_pkg_tar = path("%s.tar.gz" % pkgname) + out_pkg_tar = path(f"{pkgname}.tar.gz") # Create a distribution in zip format for the geonode python package. dist_dir = path('dist') @@ -445,7 +445,7 @@ def package(options): old_package.remove() if out_pkg_tar.exists(): - info('There is already a package for version %s' % version) + info(f'There is already a package for version {version}') return # Clean anything that is in the oupout package tree. @@ -459,7 +459,7 @@ def package(options): justcopy(support_folder, out_pkg / 'support') justcopy(install_file, out_pkg) - geonode_dist = path('..') / 'dist' / 'GeoNode-%s.zip' % version + geonode_dist = path('..') / 'dist' / f'GeoNode-{version}.zip' justcopy(geonode_dist, out_pkg) # Create a tar file with all files in the output package folder. @@ -468,14 +468,14 @@ def package(options): tar.add(file) # Add the README with the license and important links to documentation. - tar.add('README', arcname=('%s/README.rst' % out_pkg)) + tar.add('README', arcname=f'{out_pkg}/README.rst') tar.close() # Remove all the files in the temporary output package directory. out_pkg.rmtree() # Report the info about the new package. - info("%s created" % out_pkg_tar.abspath()) + info(f"{out_pkg_tar.abspath()} created") @task @@ -531,7 +531,7 @@ def stop_geoserver(options): shell=True, stdout=subprocess.PIPE) for pid in map(int, proc.stdout): - info('Stopping geoserver (process number {})'.format(pid)) + info(f'Stopping geoserver (process number {pid})') os.kill(pid, signal.SIGKILL) # Check if the process that we killed is alive. @@ -566,27 +566,21 @@ def start_django(options): """ settings = options.get('settings', '') if settings and 'DJANGO_SETTINGS_MODULE' not in settings: - settings = 'DJANGO_SETTINGS_MODULE=%s' % settings + settings = f'DJANGO_SETTINGS_MODULE={settings}' bind = options.get('bind', '0.0.0.0:8000') port = bind.split(":")[1] foreground = '' if options.get('foreground', False) else '&' - sh('%s python -W ignore manage.py runserver %s %s' % (settings, bind, foreground)) + sh(f'{settings} python -W ignore manage.py runserver {bind} {foreground}') if ASYNC_SIGNALS: if 'django_celery_beat' not in INSTALLED_APPS: - sh("{} celery -A geonode.celery_app:app worker --without-gossip --without-mingle -Ofair -B -E \ + sh(f"{settings} celery -A geonode.celery_app:app worker --without-gossip --without-mingle -Ofair -B -E \ --statedb=worker.state -s celerybeat-schedule --loglevel=DEBUG \ - --concurrency=10 -n worker1@%h -f celery.log {}".format( # noqa - settings, - foreground - )) + --concurrency=10 -n worker1@%h -f celery.log {foreground}") else: - sh("{} celery -A geonode.celery_app:app worker -l DEBUG {} {}".format( - settings, - "-s django_celery_beat.schedulers:DatabaseScheduler", - foreground - )) - sh('%s python -W ignore manage.py runmessaging %s' % (settings, foreground)) + sh(f"{settings} celery -A geonode.celery_app:app worker -l DEBUG -s \ + django_celery_beat.schedulers:DatabaseScheduler {foreground}") + sh(f'{settings} python -W ignore manage.py runmessaging {foreground}') # wait for Django to start started = waitfor("http://localhost:" + port) @@ -602,9 +596,9 @@ def start_messaging(options): """ settings = options.get('settings', '') if settings and 'DJANGO_SETTINGS_MODULE' not in settings: - settings = 'DJANGO_SETTINGS_MODULE=%s' % settings + settings = f'DJANGO_SETTINGS_MODULE={settings}' foreground = '' if options.get('foreground', False) else '&' - sh('%s python -W ignore manage.py runmessaging %s' % (settings, foreground)) + sh(f'{settings} python -W ignore manage.py runmessaging {foreground}') @task @@ -652,7 +646,7 @@ def start_geoserver(options): except socket.error as e: socket_free = False if e.errno == 98: - info('Port %s is already in use' % jetty_port) + info(f'Port {jetty_port} is already in use') else: info( 'Something else raised the socket.error exception while checking port %s' % @@ -726,11 +720,11 @@ def start_geoserver(options): ' > %(loggernullpath)s &' % locals() )) - info('Starting GeoServer on %s' % url) + info(f'Starting GeoServer on {url}') # wait for GeoServer to start started = waitfor(url) - info('The logs are available at %s' % log_file) + info(f'The logs are available at {log_file}') if not started: # If applications did not start in time we will give the user a chance @@ -761,10 +755,8 @@ def test(options): 'geonode.monitoring' in INSTALLED_APPS and \ 'geonode.monitoring' not in _apps_to_test: _apps_to_test.append('geonode.monitoring') - sh("%s manage.py test geonode.tests.smoke %s.tests --noinput %s %s" % (options.get('prefix'), - '.tests '.join(_apps_to_test), - _keepdb, - _parallel)) + sh(f"{options.get('prefix')} manage.py test geonode.tests.smoke \ +{('.tests '.join(_apps_to_test))}.tests --noinput {_keepdb} {_parallel}") @task @@ -830,39 +822,32 @@ def test_integration(options): if local: sh("cp geonode/upload/tests/test_settings.py geonode/") settings = 'geonode.test_settings' - sh("DJANGO_SETTINGS_MODULE={} python -W ignore manage.py " - "makemigrations --noinput".format(settings)) - sh("DJANGO_SETTINGS_MODULE={} python -W ignore manage.py " - "migrate --noinput".format(settings)) - sh("DJANGO_SETTINGS_MODULE={} python -W ignore manage.py " - "loaddata sample_admin.json".format(settings)) - sh("DJANGO_SETTINGS_MODULE={} python -W ignore manage.py " - "loaddata geonode/base/fixtures/default_oauth_apps.json".format(settings)) - sh("DJANGO_SETTINGS_MODULE={} python -W ignore manage.py " - "loaddata geonode/base/fixtures/initial_data.json".format(settings)) + sh(f"DJANGO_SETTINGS_MODULE={settings} python -W ignore manage.py " + "makemigrations --noinput") + sh(f"DJANGO_SETTINGS_MODULE={settings} python -W ignore manage.py " + "migrate --noinput") + sh(f"DJANGO_SETTINGS_MODULE={settings} python -W ignore manage.py " + "loaddata sample_admin.json") + sh(f"DJANGO_SETTINGS_MODULE={settings} python -W ignore manage.py " + "loaddata geonode/base/fixtures/default_oauth_apps.json") + sh(f"DJANGO_SETTINGS_MODULE={settings} python -W ignore manage.py " + "loaddata geonode/base/fixtures/initial_data.json") call_task('start_geoserver') bind = options.get('bind', '0.0.0.0:8000') foreground = '' if options.get('foreground', False) else '&' - sh('DJANGO_SETTINGS_MODULE=%s python -W ignore manage.py runmessaging %s' % - (settings, foreground)) - sh('DJANGO_SETTINGS_MODULE=%s python -W ignore manage.py runserver %s %s' % - (settings, bind, foreground)) + sh(f'DJANGO_SETTINGS_MODULE={settings} python -W ignore manage.py runmessaging {foreground}') + sh(f'DJANGO_SETTINGS_MODULE={settings} python -W ignore manage.py runserver {bind} {foreground}') sh('sleep 30') - settings = 'REUSE_DB=1 DJANGO_SETTINGS_MODULE=%s' % settings + settings = f'REUSE_DB=1 DJANGO_SETTINGS_MODULE={settings}' else: call_task('sync', options={'settings': settings}) live_server_option = '' info("Running the tests now...") - sh(('%s %s manage.py test %s' - ' -v 3 %s --noinput %s' % (settings, - prefix, - name, - _keepdb, - live_server_option))) + sh(f'{settings} {prefix} manage.py test {name} -v 3 {_keepdb} --noinput {live_server_option}') except BuildFailure as e: - info('Tests failed! %s' % str(e)) + info(f'Tests failed! {str(e)}') else: success = True finally: @@ -898,17 +883,49 @@ def run_tests(options): call_task('test', options={'prefix': prefix}) elif integration_tests: if integration_upload_tests: - call_task('test_integration', options={'prefix': prefix, 'name': 'geonode.upload.tests.integration', 'local': local}) + call_task( + 'test_integration', + options={ + 'prefix': prefix, + 'name': 'geonode.upload.tests.integration', + 'local': local + }) elif integration_monitoring_tests: - call_task('test_integration', options={'prefix': prefix, 'name': 'geonode.monitoring.tests.integration', 'local': local}) + call_task( + 'test_integration', + options={ + 'prefix': prefix, + 'name': 'geonode.monitoring.tests.integration', + 'local': local + }) elif integration_csw_tests: - call_task('test_integration', options={'prefix': prefix, 'name': 'geonode.tests.csw', 'local': local}) + call_task( + 'test_integration', + options={ + 'prefix': prefix, + 'name': 'geonode.tests.csw', + 'local': local + }) elif integration_bdd_tests: - call_task('test_bdd', options={'local': local}) + call_task( + 'test_bdd', + options={'local': local}) elif integration_server_tests: - call_task('test_integration', options={'prefix': prefix, 'name': 'geonode.geoserver.tests.integration', 'local': local}) + call_task( + 'test_integration', + options={ + 'prefix': prefix, + 'name': 'geonode.geoserver.tests.integration', + 'local': local + }) else: - call_task('test_integration', options={'prefix': prefix, 'name': 'geonode.tests.integration', 'local': local}) + call_task( + 'test_integration', + options={ + 'prefix': prefix, + 'name': 'geonode.tests.integration', + 'local': local + }) sh('flake8 geonode') @@ -923,10 +940,8 @@ def reset(options): def _reset(): from geonode import settings - sh("rm -rf {path}".format( - path=os.path.join(settings.PROJECT_ROOT, 'development.db') - ) - ) + path = os.path.join(settings.PROJECT_ROOT, 'development.db') + sh(f"rm -rf {path}") sh("rm -rf geonode/development.db") sh("rm -rf geonode/uploaded/*") _configure_data_dir() @@ -960,9 +975,9 @@ def setup_data(options): settings = options.get('settings', '') if settings and 'DJANGO_SETTINGS_MODULE' not in settings: - settings = 'DJANGO_SETTINGS_MODULE=%s' % settings + settings = f'DJANGO_SETTINGS_MODULE={settings}' - sh("%s python -W ignore manage.py importlayers %s -v2" % (settings, data_dir)) + sh(f"{settings} python -W ignore manage.py importlayers {data_dir} -v2") @needs(['package']) @@ -984,7 +999,7 @@ def deb(options): version, simple_version = versions() - info('Creating package for GeoNode version %s' % version) + info(f'Creating package for GeoNode version {version}') # Get rid of any uncommitted changes to debian/changelog info('Getting rid of any uncommitted changes in debian/changelog') @@ -992,8 +1007,8 @@ def deb(options): # Workaround for git-dch bug # http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=594580 - sh('rm -rf %s/.git' % (os.path.realpath('package'))) - sh('ln -s %s %s' % (os.path.realpath('.git'), os.path.realpath('package'))) + sh(f"rm -rf {os.path.realpath('package')}/.git") + sh(f"ln -s {os.path.realpath('.git')} {os.path.realpath('package')}") with pushd('package'): @@ -1013,7 +1028,7 @@ def deb(options): deb_changelog = path('debian') / 'changelog' for idx, line in enumerate(fileinput.input([deb_changelog], inplace=True)): if idx == 0: - logger.info("geonode ({}) {}; urgency=high".format(simple_version, distribution), end='') + logger.info(f"geonode ({simple_version}) {distribution}; urgency=high", end='') else: print(line.replace("urgency=medium", "urgency=high"), end='') @@ -1028,13 +1043,13 @@ def deb(options): sh('debuild -S') elif key is not None and ppa is None: print("A signed installable package") - sh('debuild -k%s -A' % key) + sh(f'debuild -k{key} -A') elif key is not None and ppa is not None: print("A signed, source package") - sh('debuild -k%s -S' % key) + sh(f'debuild -k{key} -S') if ppa is not None: - sh('dput ppa:%s geonode_%s_source.changes' % (ppa, simple_version)) + sh(f'dput ppa:{ppa} geonode_{simple_version}_source.changes') @task @@ -1060,11 +1075,11 @@ def publish(options): version, simple_version = versions() if ppa: sh('git add package/debian/changelog') - sh('git commit -m "Updated changelog for version %s"' % version) - sh('git tag -f %s' % version) - sh('git push origin %s' % version) - sh('git tag -f debian/%s' % simple_version) - sh('git push origin debian/%s' % simple_version) + sh(f'git commit -m "Updated changelog for version {version}"') + sh(f'git tag -f {version}') + sh(f'git push origin {version}') + sh(f'git tag -f debian/{simple_version}') + sh(f'git push origin debian/{simple_version}') # sh('git push origin master') sh('python setup.py sdist upload -r pypi') @@ -1084,11 +1099,11 @@ def versions(): stage = 'thefinal' if stage == 'unstable': - tail = '%s%s' % (branch, timestamp) + tail = f'{branch}{timestamp}' else: - tail = '%s%s' % (stage, edition) + tail = f'{stage}{edition}' - simple_version = '%s.%s.%s+%s' % (major, minor, revision, tail) + simple_version = f'{major}.{minor}.{revision}+{tail}' return version, simple_version @@ -1105,10 +1120,10 @@ def kill(arg1, arg2): while running and time.time() - t0 < time_out: if os.name == 'nt': - p = Popen('tasklist | find "%s"' % arg1, shell=True, + p = Popen(f'tasklist | find "{arg1}"', shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=False) else: - p = Popen('ps aux | grep %s' % arg1, shell=True, + p = Popen(f'ps aux | grep {arg1}', shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) lines = p.stdout.readlines() @@ -1116,26 +1131,26 @@ def kill(arg1, arg2): running = False for line in lines: # this kills all java.exe and python including self in windows - if ('%s' % arg2 in str(line)) or (os.name == 'nt' and '%s' % arg1 in str(line)): + if (f'{arg2}' in str(line)) or (os.name == 'nt' and f'{arg1}' in str(line)): running = True # Get pid fields = line.strip().split() - info('Stopping %s (process number %s)' % (arg1, int(fields[1]))) + info(f'Stopping {arg1} (process number {int(fields[1])})') if os.name == 'nt': - kill = 'taskkill /F /PID "%s"' % int(fields[1]) + kill = f'taskkill /F /PID "{int(fields[1])}"' else: - kill = 'kill -9 %s 2> /dev/null' % int(fields[1]) + kill = f'kill -9 {int(fields[1])} 2> /dev/null' os.system(kill) # Give it a little more time time.sleep(1) if running: - raise Exception('Could not stop %s: ' - 'Running processes are\n%s' - % (arg1, '\n'.join([str(_l).strip() for _l in lines]))) + _procs = '\n'.join([str(_l).strip() for _l in lines]) + raise Exception(f"Could not stop {arg1}: " + f"Running processes are\n{_procs}") def waitfor(url, timeout=300): diff --git a/scripts/cloud/demo_site.py b/scripts/cloud/demo_site.py index 0ebf19b2af0..ff1fa437e14 100644 --- a/scripts/cloud/demo_site.py +++ b/scripts/cloud/demo_site.py @@ -73,7 +73,7 @@ def deleteNode(self, node_name): def buildJob(self, job): """Trigger a job build""" - print('Building %s job' % job) + print(f'Building {job} job') self.j.build_job(job) print('Build requested') diff --git a/scripts/cloud/ec2.py b/scripts/cloud/ec2.py index 5f4020bd527..e45883c39d4 100644 --- a/scripts/cloud/ec2.py +++ b/scripts/cloud/ec2.py @@ -221,13 +221,13 @@ def launch(): print("Firing up instance...") instance = wait_for_state(ec2, instance_id, 'running') dns = instance['PublicDnsName'] - print(("Instance running at %s" % dns)) + print(f"Instance running at {dns}") config.set('ec2', 'HOST', dns) config.set('ec2', 'INSTANCE', instance_id) writeconfig(config) - print(("ssh -i %s ubuntu@%s" % (key_path, dns))) + print(f"ssh -i {key_path} ubuntu@{dns}") print("Terminate the instance via the web interface.") time.sleep(20) @@ -267,6 +267,6 @@ def terminate(): print(config.get('ec2', 'KEY_PATH')) else: print("Usage:\n " + - "python %s launch_base\n " % sys.argv[0] + - "python %s launch_geonode\n " % sys.argv[0] + - "python %s terminate" % sys.argv[0]) + f"python {sys.argv[0]} launch_base\n " + + f"python {sys.argv[0]} launch_geonode\n " + + f"python {sys.argv[0]} terminate") diff --git a/scripts/cloud/fabfile.py b/scripts/cloud/fabfile.py index 3da8c7f83e7..8a64310ac6c 100644 --- a/scripts/cloud/fabfile.py +++ b/scripts/cloud/fabfile.py @@ -74,14 +74,14 @@ # Install GeoNode dependencies def install_depend(): - sudo('cd %s; virtualenv geonode --system-site-packages;' % INSTALLDIR) + sudo(f'cd {INSTALLDIR}; virtualenv geonode --system-site-packages;') sudo('apt-get install -y gcc python-pastescript python-dev libxml2-dev libxslt1-dev openjdk-6-jdk ' 'python-psycopg2 imagemagick') # Web server sudo('apt-get install -y apache2 tomcat6 libapache2-mod-wsgi maven2') sudo("a2enmod proxy_http") # Database - sudo('apt-get install -y postgis=%s postgresql-9.1-postgis postgresql-server-dev-9.1' % POSTGIS_VER) + sudo(f'apt-get install -y postgis={POSTGIS_VER} postgresql-9.1-postgis postgresql-server-dev-9.1') create_postgis_template() # sed('/etc/postgresql/9.1/main/pg_hba.conf', # 'local all all peer', @@ -94,17 +94,14 @@ def create_postgis_template(): sudo('createdb -E UTF8 template_postgis', user='postgres') sudo('createlang -d template_postgis plpgsql', user='postgres') cstring = "UPDATE pg_database SET datistemplate='true' WHERE datname='template_postgis'" - sudo('psql -d postgres -c %s' % cstring, user='postgres') - sudo('psql -d template_postgis -f /usr/share/postgresql/9.1/contrib/postgis-%s/postgis.sql' - % POSTGIS_VER[:3], user='postgres') - sudo('psql -d template_postgis -f /usr/share/postgresql/9.1/contrib/postgis-%s/spatial_ref_sys.sql' - % POSTGIS_VER[:3], user='postgres') + sudo(f'psql -d postgres -c {cstring}', user='postgres') + sudo(f'psql -d template_postgis -f /usr/share/postgresql/9.1/contrib/postgis-{POSTGIS_VER[:3]}/postgis.sql', user='postgres') + sudo(f'psql -d template_postgis -f /usr/share/postgresql/9.1/contrib/postgis-{POSTGIS_VER[:3]}/spatial_ref_sys.sql', user='postgres') sudo('psql -d template_postgis -c "GRANT ALL ON geometry_columns TO PUBLIC;"', user='postgres') sudo('psql -d template_postgis -c "GRANT ALL ON geography_columns TO PUBLIC;"', user='postgres') sudo('psql -d template_postgis -c "GRANT ALL ON spatial_ref_sys TO PUBLIC;"', user='postgres') if POSTGIS_VER[:1] == '2': - sudo('psql -d template_postgis -f /usr/share/postgresql/9.1/contrib/postgis-%s/rtpostgis.sql' - % POSTGIS_VER[:3], user='postgres') + sudo(f'psql -d template_postgis -f /usr/share/postgresql/9.1/contrib/postgis-{POSTGIS_VER[:3]}/rtpostgis.sql', user='postgres') # Install GeoNode library @@ -114,10 +111,10 @@ def deploy_geonode(): # Fetch from GitHub sudo('apt-get install -y git') sudo('rm -rf setup') - sudo('git clone -b %s %s setup' % (GEONODE_BRANCH, GEONODE_GIT_URL)) + sudo(f'git clone -b {GEONODE_BRANCH} {GEONODE_GIT_URL} setup') # Install requirements, setup sudo('pip install -e setup') - sudo('cp -r setup/geonode %s' % PYLIBS) + sudo(f'cp -r setup/geonode {PYLIBS}') sudo('cp setup/package/support/geonode.apache /etc/apache2/sites-available/geonode') sudo('rm -rf setup') sed('/etc/apache2/sites-available/geonode', 'REPLACE_WITH_SITEDIR', PYLIBS, use_sudo=True) @@ -143,17 +140,17 @@ def deploy_project(project): with cd(GEONODEDIR), prefix(ACT): sudo('pip install -r requirements.txt --upgrade') sudo('rm requirements.txt') - put('%s/%s.apache' % (project, project), '/etc/apache2/sites-available/%s' % project, use_sudo=True) - sed('/etc/apache2/sites-available/%s' % project, 'REPLACE_WITH_SITEDIR', PYLIBS, use_sudo=True) + put(f'{project}/{project}.apache', f'/etc/apache2/sites-available/{project}', use_sudo=True) + sed(f'/etc/apache2/sites-available/{project}', 'REPLACE_WITH_SITEDIR', PYLIBS, use_sudo=True) with cd(os.path.join(PYLIBS, project)): sudo('ln -s settings_production.py local_settings.py') def enable_site(project): with cd(PYLIBS), prefix(ACT): - sudo('a2ensite %s; service apache2 restart' % project) + sudo(f'a2ensite {project}; service apache2 restart') sudo('mkdir /var/www/geonode; chown www-data:www-data /var/www/geonode') - sudo('django-admin.py collectstatic --noinput --settings=%s.settings' % project) + sudo(f'django-admin.py collectstatic --noinput --settings={project}.settings') # TODO - test/fix this function @@ -168,32 +165,32 @@ def restore_data(project): run('rm geoserver-data.tar') sudo('service tomcat6 restart') with prefix(ACT): - sudo('django-admin.py cleardeadlayers --settings=%s.settings' % project) - sudo('django-admin.py updatelayers --settings=%s.settings' % project) + sudo(f'django-admin.py cleardeadlayers --settings={project}.settings') + sudo(f'django-admin.py updatelayers --settings={project}.settings') def create_database(db, user, password): - # sudo("dropdb %s" % db, user="postgres") - # sudo("dropuser %s" % user, user="postgres") + # sudo(f"dropdb {db}", user="postgres") + # sudo(f"dropuser {user}", user="postgres") with fab_settings(warn_only=True): - sudo("createuser -s %s" % user, user="postgres") - sudo("createdb -O %s %s -T template_postgis" % (user, db), user="postgres") - sudo("psql -c \"alter user %s with encrypted password '%s'\" " % (user, password), user="postgres") + sudo(f"createuser -s {user}", user="postgres") + sudo(f"createdb -O {user} {db} -T template_postgis", user="postgres") + sudo(f"psql -c \"alter user {user} with encrypted password '{password}'\" ", user="postgres") def setup_pgsql(project): import sys sys.path.append('.') - exec('import %s.settings_production as settings' % project) + exec(f'import {project}.settings_production as settings') db = settings.DATABASES['default']['NAME'] user = settings.DATABASES['default']['USER'] password = settings.DATABASES['default']['PASSWORD'] create_database(db, user, password) with prefix(ACT): - sudo('django-admin.py migrate --settings=%s.settings' % project) + sudo(f'django-admin.py migrate --settings={project}.settings') # Need to restore database and GeoServer data - # put('data/%s.sql' % db, GEONODEDIR, use_sudo=True) - # sudo('psql -d %s -f %s/%s.sql' % (db, GEONODEDIR, db), user='postgres') + # put(f'data/{db}.sql', GEONODEDIR, use_sudo=True) + # sudo(f'psql -d {db} -f {GEONODEDIR}/{db}.sql', user='postgres') def deploy(project): @@ -241,14 +238,14 @@ def deploy_geonode_dev_deb(): def change_admin_password(): put('../misc/changepw.py', '/home/ubuntu/') - run("perl -pi -e 's/replace.me.admin.user/%s/g' ~/changepw.py" % ADMIN_USER) - run("perl -pi -e 's/replace.me.admin.pw/%s/g' ~/changepw.py" % ADMIN_PASSWORD) + run(f"perl -pi -e 's/replace.me.admin.user/{ADMIN_USER}/g' ~/changepw.py") + run(f"perl -pi -e 's/replace.me.admin.pw/{ADMIN_PASSWORD}/g' ~/changepw.py") sudo('source /var/lib/geonode/bin/activate;cat ~/changepw.py | django-admin.py shell --settings=geonode.settings') run('rm ~/changepw.py') def geonode_updateip(server_name="demo.geonode.org"): - sudo('geonode-updateip %s' % server_name) + sudo(f'geonode-updateip {server_name}') def set_temp_hosts_entry(server_name="demo.geonode.org"): @@ -278,14 +275,13 @@ def cleanup_temp(): def copy_keys(): - sudo('rm -f ~/.ssh/*%s.pem' % KEY_BASE) - put(('%s*%s*' % (KEY_PATH, KEY_BASE)), '/home/ubuntu/.ssh/', mode=0o400) + sudo(f'rm -f ~/.ssh/*{KEY_BASE}.pem') + put((f'{KEY_PATH}*{KEY_BASE}*'), '/home/ubuntu/.ssh/', mode=0o400) def install_ec2_tools(): - sudo('add-apt-repository "deb http://us-east-1.ec2.archive.ubuntu.com/ubuntu/ %s multiverse"' % UBUNTU_VERSION) - sudo('add-apt-repository "deb http://us-east-1.ec2.archive.ubuntu.com/ubuntu/ %s-updates multiverse"' - % UBUNTU_VERSION) + sudo(f'add-apt-repository "deb http://us-east-1.ec2.archive.ubuntu.com/ubuntu/ {UBUNTU_VERSION} multiverse"') + sudo(f'add-apt-repository "deb http://us-east-1.ec2.archive.ubuntu.com/ubuntu/ {UBUNTU_VERSION}-updates multiverse"') sudo('apt-get -y update') sudo('apt-get install -y ec2-ami-tools') sudo('apt-get install -y ec2-api-tools') @@ -299,22 +295,19 @@ def build_geonode_ami(): update_instance() install_ec2_tools() with hide('running', 'stdout', 'stderr'): - sudo('export AWS_USER_ID=%s' % AWS_USER_ID) - sudo('export AWS_ACCESS_KEY_ID=%s' % AWS_ACCESS_KEY_ID) - sudo('export AWS_SECRET_ACCESS_KEY=%s' % AWS_SECRET_ACCESS_KEY) - sudo('export ARCH=%s' % ARCH) - prefix = 'geonode-%s-%s' % (VERSION, datetime.datetime.now().strftime('%Y%m%d%H%M%S')) + sudo(f'export AWS_USER_ID={AWS_USER_ID}') + sudo(f'export AWS_ACCESS_KEY_ID={AWS_ACCESS_KEY_ID}') + sudo(f'export AWS_SECRET_ACCESS_KEY={AWS_SECRET_ACCESS_KEY}') + sudo(f'export ARCH={ARCH}') + prefix = f"geonode-{VERSION}-{datetime.datetime.now().strftime('%Y%m%d%H%M%S')}" excludes = '/mnt,/root/.ssh,/home/ubuntu/.ssh,/tmp' - sudo("ec2-bundle-vol -r %s -d /mnt -p %s -u %s -k ~/.ssh/pk-*.pem -c ~/.ssh/cert-*.pem -e %s" - % (ARCH, prefix, AWS_USER_ID, excludes)) - sudo("ec2-upload-bundle -b %s -m /mnt/%s.manifest.xml -a %s -s %s" - % (AMI_BUCKET, prefix, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)) - output = sudo('ec2-register --name "%s/%s" "%s/%s.manifest.xml" -K ~/.ssh/pk-*.pem -C ~/.ssh/cert-*.pem' - % (AMI_BUCKET, prefix, AMI_BUCKET, prefix)) + sudo(f"ec2-bundle-vol -r {ARCH} -d /mnt -p {prefix} -u {AWS_USER_ID} -k ~/.ssh/pk-*.pem -c ~/.ssh/cert-*.pem -e {excludes}") + sudo(f"ec2-upload-bundle -b {AMI_BUCKET} -m /mnt/{prefix}.manifest.xml -a {AWS_ACCESS_KEY_ID} -s {AWS_SECRET_ACCESS_KEY}") + output = sudo(f'ec2-register --name "{AMI_BUCKET}/{prefix}" "{AMI_BUCKET}/{prefix}.manifest.xml" -K ~/.ssh/pk-*.pem -C ~/.ssh/cert-*.pem') ami_id = output.split('\t')[1] if MAKE_PUBLIC: - sudo("ec2-modify-image-attribute -l -a all -K ~/.ssh/pk-*.pem -C ~/.ssh/cert-*.pem %s" % ami_id) - logger.info("AMI %s Ready for Use" % ami_id) + sudo(f"ec2-modify-image-attribute -l -a all -K ~/.ssh/pk-*.pem -C ~/.ssh/cert-*.pem {ami_id}") + logger.info(f"AMI {ami_id} Ready for Use") def install_sample_data(): diff --git a/scripts/misc/create_full_geonode_db.py b/scripts/misc/create_full_geonode_db.py index 56a87776f3c..ac8441d252a 100755 --- a/scripts/misc/create_full_geonode_db.py +++ b/scripts/misc/create_full_geonode_db.py @@ -62,7 +62,7 @@ def assign_random_category(resource): def assign_keywords(resource): """ Assigns up to 5 keywords to resource """ for i in range(0, randint(0, 5)): - resource.keywords.add('keyword_%s' % randint(0, n_keywords)) + resource.keywords.add(f'keyword_{randint(0, n_keywords)}') def assign_regions(resource): @@ -77,7 +77,7 @@ def create_users(n_users): """ Create n users in the database """ for i in range(0, n_users): user = get_user_model() - user.username = 'user_%s' % i + user.username = f'user_{i}' user.save() @@ -91,11 +91,11 @@ def set_resource(resource): def create_document(number): """ Creates a new document """ - file_list = glob.glob('%s*.jpg' % doc_path) + file_list = glob.glob(f'{doc_path}*.jpg') random_index = randint(0, len(file_list) - 1) file_uri = file_list[random_index] - title = 'Document N. %s' % number - img_filename = '%s_img.jpg' % number + title = f'Document N. {number}' + img_filename = f'{number}_img.jpg' doc = Document(title=title, owner=get_random_user()) doc.save() with open(file_uri, 'r') as f: @@ -109,7 +109,7 @@ def create_document(number): def create_layer(number): """ Creates a new layer """ - file_list = glob.glob('%s*.shp' % shp_path) + file_list = glob.glob(f'{shp_path}*.shp') random_index = randint(0, len(file_list) - 1) file_uri = file_list[random_index] layer = file_upload(file_uri) @@ -140,7 +140,7 @@ def create_layer(number): Document.objects.all().delete() for d in range(0, n_docs): t = Timer(partial(create_document, d)) - print('Document %s generated in: %s' % (d, t.timeit(number=1))) + print(f'Document {d} generated in: {t.timeit(number=1)}') # 3. create layers # first we delete layers @@ -149,4 +149,4 @@ def create_layer(number): for l in range(0, n_layers): t = Timer(partial(create_layer, l)) - print('Layer %s generated in: %s' % (l, t.timeit(number=1))) + print(f'Layer {l} generated in: {t.timeit(number=1)}') diff --git a/scripts/misc/upload.py b/scripts/misc/upload.py index b748907b083..df830c6abef 100755 --- a/scripts/misc/upload.py +++ b/scripts/misc/upload.py @@ -50,7 +50,7 @@ def upload_file_s3(filename, bucket, obj_name=None): try: _, bucket_name, filepath = sys.argv except ValueError: - print(("Usage:\n python %s bucket_name filepath" % sys.argv[0])) + print(f"Usage:\n python {sys.argv[0]} bucket_name filepath") filename = os.path.basename(filepath) error = upload_file_s3(filepath, bucket_name) diff --git a/scripts/spcgeonode/django/initialize.py b/scripts/spcgeonode/django/initialize.py index 6fa4693aac6..c094c5e03bb 100644 --- a/scripts/spcgeonode/django/initialize.py +++ b/scripts/spcgeonode/django/initialize.py @@ -109,14 +109,14 @@ } if _port not in _protocols: redirect_uris = [ - 'http://{}:{}/geoserver'.format(_host, _port), - 'http://{}:{}/geoserver/index.html'.format(_host, _port), + f'http://{_host}:{_port}/geoserver', + f'http://{_host}:{_port}/geoserver/index.html' ] else: # Make sure protocol string match with GeoServer Redirect URL's protocol string redirect_uris = [ - '{}{}/geoserver'.format(_protocols[_port], _host), - '{}{}/geoserver/index.html'.format(_protocols[_port], _host), + f'{_protocols[_port]}{_host}/geoserver', + f'{_protocols[_port]}{_host}/geoserver/index.html' ] app.redirect_uris = "\n".join(redirect_uris) @@ -164,12 +164,12 @@ _geoserver_host = os.getenv('GEOSERVER_LOCATION', 'http://geoserver:8080/geoserver') for _ in range(60*5): try: - requests.head("{}".format(_geoserver_host)) + requests.head(str(_geoserver_host)) break except ConnectionError: time.sleep(1) else: - requests.head("{}".format(_geoserver_host)) + requests.head(str(_geoserver_host)) ######################################################### # 9. Securing GeoServer @@ -184,7 +184,7 @@ # Getting the old password try: - r1 = requests.get('{}/rest/security/masterpw.json'.format(_geoserver_host), + r1 = requests.get(f'{_geoserver_host}/rest/security/masterpw.json', auth=(geoserver_admin_username, geoserver_admin_password)) except requests.exceptions.ConnectionError: print("Unable to connect to GeoServer. Make sure GeoServer is started and accessible.") @@ -196,8 +196,9 @@ print("Randomizing master password") new_password = uuid.uuid4().hex data = json.dumps({"oldMasterPassword": old_password, "newMasterPassword": new_password}) - r2 = requests.put('{}/rest/security/masterpw.json'.format(_geoserver_host), data=data, - headers={'Content-Type': 'application/json'}, auth=(geoserver_admin_username, geoserver_admin_password)) + r2 = requests.put(f'{_geoserver_host}/rest/security/masterpw.json', data=data, + headers={'Content-Type': 'application/json'}, + auth=(geoserver_admin_username, geoserver_admin_password)) r2.raise_for_status() else: print("Master password was already changed. No changes made.") diff --git a/setup.cfg b/setup.cfg index a6ffedb7c6a..85541504db8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -186,5 +186,5 @@ universal = 1 [flake8] max-line-length = 120 -exclude=geonode/*/migrations/*,management,scripts,docs,static,migrations,geonode/*settings.py,node_modules +exclude=geonode/*/migrations/*,scripts,docs,static,migrations,node_modules extend-ignore=E122,E124 diff --git a/tasks.py b/tasks.py index 48bcb6f0dce..d63c9bf51bd 100755 --- a/tasks.py +++ b/tasks.py @@ -1,17 +1,35 @@ -import ast -import json +# -*- coding: utf-8 -*- +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### import os import re +import ast +import json import time -import datetime import docker import socket import logging +import datetime from urllib.parse import urlparse from invoke import task - BOOTSTRAP_IMAGE_CHEIP = 'codenvy/che-ip:nightly' logger = logging.getLogger(__name__) @@ -20,7 +38,8 @@ @task def waitfordbs(ctx): print("**************************databases*******************************") - ctx.run("/usr/bin/wait-for-databases {0}".format('db'), pty=True) + db_host = os.getenv('DATABASE_HOST', 'db') + ctx.run(f"/usr/bin/wait-for-databases {db_host}", pty=True) @task @@ -36,9 +55,9 @@ def update(ctx): print("***************************setting env*********************************") ctx.run("env", pty=True) pub_ip = _geonode_public_host_ip() - print("Public Hostname or IP is {0}".format(pub_ip)) + print(f"Public Hostname or IP is {pub_ip}") pub_port = _geonode_public_port() - print("Public PORT is {0}".format(pub_port)) + print(f"Public PORT is {pub_port}") pub_protocol = 'https' if pub_port == '443' else 'http' if pub_protocol == 'https' or pub_port == '80': pub_port = None @@ -56,19 +75,21 @@ def update(ctx): if os.path.exists(override_env): os.remove(override_env) else: - print("Can not delete the %s file as it doesn't exists" % override_env) + print(f"Can not delete the {override_env} file as it doesn't exists") + if pub_port: + siteurl = f'{pub_protocol}://{pub_ip}:{pub_port}/' + gs_pub_loc = f'http://{pub_ip}:{pub_port}/geoserver/' + else: + siteurl = f'{pub_protocol}://{pub_ip}/' + gs_pub_loc = f'http://{pub_ip}/geoserver/' envs = { - "local_settings": "{0}".format(_localsettings()), - "siteurl": os.environ.get('SITEURL', - '{0}://{1}:{2}/'.format( - pub_protocol, - pub_ip, - pub_port) if pub_port else '{0}://{1}/'.format(pub_protocol, pub_ip)), - "geonode_docker_host": "{0}".format(socket.gethostbyname('geonode')), + "local_settings": str(_localsettings()), + "siteurl": os.environ.get('SITEURL', siteurl), + "geonode_docker_host": str(socket.gethostbyname('geonode')), "public_protocol": pub_protocol, - "public_fqdn": "{0}{1}".format(pub_ip, ':' + pub_port if pub_port else ''), - "public_host": "{0}".format(pub_ip), + "public_fqdn": str(pub_ip) + str(f':{pub_port}' if pub_port else ''), + "public_host": str(pub_ip), "dburl": os.environ.get('DATABASE_URL', db_url), "geodburl": os.environ.get('GEODATABASE_URL', geodb_url), "static_root": os.environ.get('STATIC_ROOT', '/mnt/volumes/statics/static/'), @@ -86,14 +107,8 @@ def update(ctx): "geodb_url": os.environ.get('GEODATABASE_URL', 'postgis://geonode:geonode@db:5432/geonode_data'), "geonode_db": os.environ.get('GEONODE_DATABASE', 'geonode'), "gs_loc": os.environ.get('GEOSERVER_LOCATION', 'http://geoserver:8080/geoserver/'), - "gs_web_ui_loc": os.environ.get('GEOSERVER_WEB_UI_LOCATION', - 'http://{0}:{1}/geoserver/'.format( - pub_ip, - pub_port) if pub_port else 'http://{0}/geoserver/'.format(pub_ip)), - "gs_pub_loc": os.environ.get('GEOSERVER_PUBLIC_LOCATION', - 'http://{0}:{1}/geoserver/'.format( - pub_ip, - pub_port) if pub_port else 'http://{0}/geoserver/'.format(pub_ip)), + "gs_web_ui_loc": os.environ.get('GEOSERVER_WEB_UI_LOCATION', gs_pub_loc), + "gs_pub_loc": os.environ.get('GEOSERVER_PUBLIC_LOCATION', gs_pub_loc), "gs_admin_pwd": os.environ.get('GEOSERVER_ADMIN_PASSWORD', 'geoserver'), "override_fn": override_env } @@ -103,8 +118,8 @@ def update(ctx): "['{public_fqdn}', '{public_host}', 'localhost', 'django', 'geonode',]".format(**envs)) except ValueError: current_allowed = [] - current_allowed.extend(['{}'.format(pub_ip), '{}:{}'.format(pub_ip, pub_port)]) - allowed_hosts = ['"{}"'.format(c) for c in current_allowed] + ['"geonode"', '"django"'] + current_allowed.extend([str(pub_ip), f'{pub_ip}:{pub_port}']) + allowed_hosts = [str(c) for c in current_allowed] + ['"geonode"', '"django"'] ctx.run("echo export DJANGO_SETTINGS_MODULE=\ {local_settings} >> {override_fn}".format(**envs), pty=True) @@ -162,7 +177,7 @@ def update(ctx): {siteurl} >> {override_fn}".format(**envs), pty=True) ctx.run("echo export LOGOUT_REDIRECT_URL=\ {siteurl} >> {override_fn}".format(**envs), pty=True) - ctx.run("source %s" % override_env, pty=True) + ctx.run(f"source {override_env}", pty=True) print("****************************finalize env**********************************") ctx.run("env", pty=True) @@ -170,22 +185,12 @@ def update(ctx): @task def migrations(ctx): print("**************************migrations*******************************") - ctx.run("python manage.py makemigrations --noinput --merge --settings={0}".format( - _localsettings() - ), pty=True) - ctx.run("python manage.py makemigrations --noinput --settings={0}".format( - _localsettings() - ), pty=True) - ctx.run("python manage.py migrate --noinput --settings={0}".format( - _localsettings() - ), pty=True) - ctx.run("python manage.py updategeoip --settings={0}".format( - _localsettings() - ), pty=True) + ctx.run(f"python manage.py makemigrations --noinput --merge --settings={_localsettings()}", pty=True) + ctx.run(f"python manage.py makemigrations --noinput --settings={_localsettings()}", pty=True) + ctx.run(f"python manage.py migrate --noinput --settings={_localsettings()}", pty=True) + ctx.run(f"python manage.py updategeoip --settings={_localsettings()}", pty=True) try: - ctx.run("python manage.py rebuild_index --noinput --settings={0}".format( - _localsettings() - ), pty=True) + ctx.run(f"python manage.py rebuild_index --noinput --settings={_localsettings()}", pty=True) except Exception: pass @@ -194,9 +199,7 @@ def migrations(ctx): def statics(ctx): print("**************************statics*******************************") ctx.run('mkdir -p /mnt/volumes/statics/{static,uploads}') - ctx.run("python manage.py collectstatic --noinput --settings={0}".format( - _localsettings() - ), pty=True) + ctx.run(f"python manage.py collectstatic --noinput --settings={_localsettings()}", pty=True) @task @@ -207,50 +210,43 @@ def prepare(ctx): ctx.run("rm -rf /tmp/default_site.json", pty=True) _prepare_site_fixture() # Updating OAuth2 Service Config + new_ext_ip = os.environ['SITEURL'] + client_id = os.environ['OAUTH2_CLIENT_ID'] + client_secret = os.environ['OAUTH2_CLIENT_SECRET'] oauth_config = "/geoserver_data/data/security/filter/geonode-oauth2/config.xml" ctx.run( - 'sed -i "s|.*|{client_id}|g" {oauth_config}'.format( - client_id=os.environ['OAUTH2_CLIENT_ID'], - oauth_config=oauth_config - ), pty=True) + f'sed -i "s|.*|{client_id}|g" {oauth_config}', + pty=True) ctx.run( - 'sed -i "s|.*|{client_secret}|g" {oauth_config}'.format( # noqa - client_secret=os.environ['OAUTH2_CLIENT_SECRET'], - oauth_config=oauth_config - ), pty=True) + f'sed -i "s|.*|{client_secret}|g" {oauth_config}', + pty=True) ctx.run( - 'sed -i "s|.*|{new_ext_ip}o/authorize/|g" {oauth_config}'.format( - new_ext_ip=os.environ['SITEURL'], - oauth_config=oauth_config - ), pty=True) + f'sed -i "s|.*|{new_ext_ip}o/authorize/|g" {oauth_config}', # noqa + pty=True) ctx.run( - 'sed -i "s|.*|{new_ext_ip}geoserver/index.html|g" {oauth_config}'.format( - new_ext_ip=os.environ['SITEURL'], - oauth_config=oauth_config - ), pty=True) + f'sed -i "s|.*|{new_ext_ip}geoserver/index.html|g" {oauth_config}', # noqa + pty=True) ctx.run( - 'sed -i "s|.*|{new_ext_ip}account/logout/|g" {oauth_config}'.format( - new_ext_ip=os.environ['SITEURL'], - oauth_config=oauth_config - ), pty=True) + f'sed -i "s|.*|{new_ext_ip}account/logout/|g" {oauth_config}', + pty=True) @task def fixtures(ctx): print("**************************fixtures********************************") - ctx.run("python manage.py loaddata sample_admin \ ---settings={0}".format(_localsettings()), pty=True) - ctx.run("python manage.py loaddata /tmp/default_oauth_apps_docker.json \ ---settings={0}".format(_localsettings()), pty=True) - ctx.run("python manage.py loaddata geonode/base/fixtures/initial_data.json \ ---settings={0}".format(_localsettings()), pty=True) + ctx.run(f"python manage.py loaddata sample_admin \ +--settings={_localsettings()}", pty=True) + ctx.run(f"python manage.py loaddata /tmp/default_oauth_apps_docker.json \ +--settings={_localsettings()}", pty=True) + ctx.run(f"python manage.py loaddata geonode/base/fixtures/initial_data.json \ +--settings={_localsettings()}", pty=True) @task def collectstatic(ctx): print("************************static artifacts******************************") - ctx.run("django-admin.py collectstatic --noinput \ ---settings={0}".format(_localsettings()), pty=True) + ctx.run(f"django-admin.py collectstatic --noinput \ +--settings={_localsettings()}", pty=True) @task @@ -265,8 +261,8 @@ def monitoringfixture(ctx): ctx.run("rm -rf /tmp/default_monitoring_apps_docker.json", pty=True) _prepare_monitoring_fixture() try: - ctx.run("django-admin.py loaddata /tmp/default_monitoring_apps_docker.json \ ---settings={0}".format(_localsettings()), pty=True) + ctx.run(f"django-admin.py loaddata /tmp/default_monitoring_apps_docker.json \ +--settings={_localsettings()}", pty=True) except Exception as e: logger.error("ERROR installing monitoring fixture: " + str(e)) @@ -274,8 +270,8 @@ def monitoringfixture(ctx): @task def updategeoip(ctx): print("**************************update geoip*******************************") - ctx.run("django-admin.py updategeoip \ - --settings={0}".format(_localsettings()), pty=True) + ctx.run(f"django-admin.py updategeoip \ +--settings={_localsettings()}", pty=True) @task @@ -285,15 +281,15 @@ def updateadmin(ctx): _prepare_admin_fixture( os.environ.get('ADMIN_PASSWORD', 'admin'), os.environ.get('ADMIN_EMAIL', 'admin@example.org')) - ctx.run("django-admin.py loaddata /tmp/django_admin_docker.json \ ---settings={0}".format(_localsettings()), pty=True) + ctx.run(f"django-admin.py loaddata /tmp/django_admin_docker.json \ +--settings={_localsettings()}", pty=True) @task def collectmetrics(ctx): print("************************collect metrics******************************") - ctx.run("python -W ignore manage.py collect_metrics \ - --settings={0} -n -t xml".format(_localsettings()), pty=True) + ctx.run(f"python -W ignore manage.py collect_metrics \ +--settings={_localsettings()} -n -t xml", pty=True) @task @@ -305,22 +301,20 @@ def initialized(ctx): def _docker_host_ip(): try: client = docker.from_env(version='1.24') - ip_list = client.containers.run(BOOTSTRAP_IMAGE_CHEIP, - network_mode='host' - ).split("\n") + ip_list = client.containers.run( + BOOTSTRAP_IMAGE_CHEIP, + network_mode='host').split("\n") except Exception: import traceback traceback.print_exc() ip_list = ['127.0.0.1', ] if len(ip_list) > 1: - print("Docker daemon is running on more than one \ -address {0}".format(ip_list)) - print("Only the first address:{0} will be returned!".format( - ip_list[0] - )) + print(f"Docker daemon is running on more than one \ +address {ip_list}") + print(f"Only the first address:{ip_list[0]} will be returned!") else: - print("Docker daemon is running at the following \ -address {0}".format(ip_list[0])) + print(f"Docker daemon is running at the following \ +address {ip_list[0]}") return ip_list[0] @@ -331,10 +325,10 @@ def _container_exposed_port(component, instname): ports_dict = json.dumps( [c.attrs['Config']['ExposedPorts'] for c in client.containers.list( filters={ - 'label': 'org.geonode.component={0}'.format(component), + 'label': f'org.geonode.component={component}', 'status': 'running' } - ) if '{0}'.format(instname) in c.name][0] + ) if str(instname) in c.name][0] ) for key in json.loads(ports_dict): port = re.split('/tcp', key)[0] @@ -348,11 +342,9 @@ def _update_db_connstring(): user = os.getenv('GEONODE_DATABASE', 'geonode') pwd = os.getenv('GEONODE_DATABASE_PASSWORD', 'geonode') dbname = os.getenv('GEONODE_DATABASE', 'geonode') - connstr = 'postgis://{0}:{1}@db:5432/{2}'.format( - user, - pwd, - dbname - ) + dbhost = os.getenv('DATABASE_HOST', 'db') + dbport = os.getenv('DATABASE_PORT', 5432) + connstr = f'postgis://{user}:{pwd}@{dbhost}:{dbport}/{dbname}' return connstr @@ -360,11 +352,9 @@ def _update_geodb_connstring(): geouser = os.getenv('GEONODE_GEODATABASE', 'geonode_data') geopwd = os.getenv('GEONODE_GEODATABASE_PASSWORD', 'geonode_data') geodbname = os.getenv('GEONODE_GEODATABASE', 'geonode_data') - geoconnstr = 'postgis://{0}:{1}@db:5432/{2}'.format( - geouser, - geopwd, - geodbname - ) + dbhost = os.getenv('DATABASE_HOST', 'db') + dbport = os.getenv('DATABASE_PORT', 5432) + geoconnstr = f'postgis://{geouser}:{geopwd}@{dbhost}:{dbport}/{geodbname}' return geoconnstr @@ -379,10 +369,10 @@ def _rest_api_availability(url): r = requests.request('get', url, verify=False) r.raise_for_status() # Raises a HTTPError if the status is 4xx, 5xxx except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: - logger.error("GeoServer connection error is {0}".format(e)) + logger.error(f"GeoServer connection error is {e}") return False except requests.exceptions.HTTPError as er: - logger.error("GeoServer HTTP error is {0}".format(er)) + logger.error(f"GeoServer HTTP error is {er}") return False else: logger.info("GeoServer API are available!") @@ -421,26 +411,26 @@ def _geoserver_info_provision(url): "Content-type": "application/xml", "Accept": "application/xml" } - data = """ + data = f""" - {0} -""".format(os.getenv('GEOSERVER_ADMIN_PASSWORD', 'geoserver')) + {(os.getenv('GEOSERVER_ADMIN_PASSWORD', 'geoserver'))} +""" response = cat.http_request(cat.service_url + '/security/self/password', method="PUT", data=data, headers=headers) - print("Response Code: %s" % response.status_code) + print(f"Response Code: {response.status_code}") if response.status_code == 200: print("GeoServer admin password updated SUCCESSFULLY!") else: - logger.warning("WARNING: GeoServer admin password *NOT* updated: code [%s]" % response.status_code) + logger.warning(f"WARNING: GeoServer admin password *NOT* updated: code [{response.status_code}]") def _prepare_oauth_fixture(): upurl = urlparse(os.environ['SITEURL']) net_scheme = upurl.scheme pub_ip = _geonode_public_host_ip() - print("Public Hostname or IP is {0}".format(pub_ip)) + print(f"Public Hostname or IP is {pub_ip}") pub_port = _geonode_public_port() - print("Public PORT is {0}".format(pub_port)) + print(f"Public PORT is {pub_port}") default_fixture = [ { "model": "oauth2_provider.application", @@ -450,15 +440,13 @@ def _prepare_oauth_fixture(): "created": "2018-05-31T10:00:31.661Z", "updated": "2018-05-31T11:30:31.245Z", "algorithm": "RS256", - "redirect_uris": "{0}://{1}:{2}/geoserver/index.html".format( - net_scheme, - pub_ip, - pub_port) if pub_port else "{0}://{1}/geoserver/index.html".format(net_scheme, pub_ip), + "redirect_uris": f"{net_scheme}://{pub_ip}:{pub_port}/geoserver/index.html" + if pub_port else f"{net_scheme}://{pub_ip}/geoserver/index.html", "name": "GeoServer", "authorization_grant_type": "authorization-code", "client_type": "confidential", - "client_id": "{0}".format(os.environ['OAUTH2_CLIENT_ID']), - "client_secret": "{0}".format(os.environ['OAUTH2_CLIENT_SECRET']), + "client_id": str(os.environ['OAUTH2_CLIENT_ID']), + "client_secret": str(os.environ['OAUTH2_CLIENT_SECRET']), "user": [ "admin" ] @@ -476,8 +464,8 @@ def _prepare_site_fixture(): "model": "sites.site", "pk": 1, "fields": { - "domain": "{0}".format(upurl.hostname), - "name": "{0}".format(upurl.hostname) + "domain": str(upurl.hostname), + "name": str(upurl.hostname) } } ] @@ -490,9 +478,9 @@ def _prepare_monitoring_fixture(): # net_scheme = upurl.scheme # net_loc = upurl.netloc pub_ip = _geonode_public_host_ip() - print("Public Hostname or IP is {0}".format(pub_ip)) + print(f"Public Hostname or IP is {pub_ip}") pub_port = _geonode_public_port() - print("Public PORT is {0}".format(pub_port)) + print(f"Public PORT is {pub_port}") geonode_ip = pub_ip try: geonode_ip = socket.gethostbyname('geonode') @@ -509,8 +497,8 @@ def _prepare_monitoring_fixture(): { "fields": { "active": True, - "ip": "{0}".format(geonode_ip), - "name": "{0}".format(os.environ['MONITORING_HOST_NAME']) + "ip": str(geonode_ip), + "name": str(os.environ['MONITORING_HOST_NAME']) }, "model": "monitoring.host", "pk": 1 @@ -518,7 +506,7 @@ def _prepare_monitoring_fixture(): { "fields": { "active": True, - "ip": "{0}".format(geoserver_ip), + "ip": str(geoserver_ip), "name": "geoserver" }, "model": "monitoring.host", @@ -526,9 +514,8 @@ def _prepare_monitoring_fixture(): }, { "fields": { - "name": "{0}".format(os.environ['MONITORING_SERVICE_NAME']), - # "url": "{0}://{1}/".format(net_scheme, net_loc), - "url": "{0}".format(os.environ['SITEURL']), + "name": str(os.environ['MONITORING_SERVICE_NAME']), + "url": str(os.environ['SITEURL']), "notes": "", "last_check": d, "active": True, @@ -542,8 +529,7 @@ def _prepare_monitoring_fixture(): { "fields": { "name": "geoserver-hostgeonode", - # "url": "{0}://{1}/".format(net_scheme, net_loc), - "url": "{0}".format(os.environ['SITEURL']), + "url": str(os.environ['SITEURL']), "notes": "", "last_check": d, "active": True, @@ -557,7 +543,7 @@ def _prepare_monitoring_fixture(): { "fields": { "name": "geoserver-hostgeoserver", - "url": "{0}".format(os.environ['GEOSERVER_PUBLIC_LOCATION']), + "url": str(os.environ['GEOSERVER_PUBLIC_LOCATION']), "notes": "", "last_check": d, "active": True, @@ -593,22 +579,22 @@ def _prepare_admin_fixture(admin_password, admin_email): mdext_date = d.isoformat()[:23] + "Z" default_fixture = [ { - "fields": { - "date_joined": mdext_date, - "email": admin_email, - "first_name": "", - "groups": [], - "is_active": True, - "is_staff": True, - "is_superuser": True, - "last_login": mdext_date, - "last_name": "", - "password": make_password(admin_password), - "user_permissions": [], - "username": "admin" - }, - "model": "people.Profile", - "pk": 1000 + "fields": { + "date_joined": mdext_date, + "email": admin_email, + "first_name": "", + "groups": [], + "is_active": True, + "is_staff": True, + "is_superuser": True, + "last_login": mdext_date, + "last_name": "", + "password": make_password(admin_password), + "user_permissions": [], + "username": "admin" + }, + "model": "people.Profile", + "pk": 1000 } ] with open('/tmp/django_admin_docker.json', 'w') as fixturefile: