diff --git a/geonode/contrib/worldmap/gazetteer/admin.py b/geonode/contrib/worldmap/gazetteer/admin.py new file mode 100644 index 00000000000..bc20e18aecd --- /dev/null +++ b/geonode/contrib/worldmap/gazetteer/admin.py @@ -0,0 +1,27 @@ +from django.contrib import admin + +from .models import GazetteerEntry, GazetteerAttribute + + +class GazetteerEntryAdmin(admin.ModelAdmin): + list_display = ( + 'layer_name', + 'layer_attribute', + 'feature_type', + 'place_name', + 'project', + 'username', + ) + +class GazetteerAttributeAdmin(admin.ModelAdmin): + list_display = ( + 'layer_name', + 'attribute', + 'in_gazetteer', + 'is_start_date', + 'is_end_date', + 'date_format', + ) + +admin.site.register(GazetteerEntry, GazetteerEntryAdmin) +admin.site.register(GazetteerAttribute, GazetteerAttributeAdmin) diff --git a/geonode/contrib/worldmap/gazetteer/management/commands/updategazetteer.py b/geonode/contrib/worldmap/gazetteer/management/commands/updategazetteer.py index ef495bb55b1..7e0b7e70b9e 100644 --- a/geonode/contrib/worldmap/gazetteer/management/commands/updategazetteer.py +++ b/geonode/contrib/worldmap/gazetteer/management/commands/updategazetteer.py @@ -10,6 +10,8 @@ class Command(BaseCommand): """ args = '[none]' + # TODO update this to new version + def handle(self, *args, **kwargs): gaz_layers = GazetteerEntry.objects.filter( username__isnull=True).values('layer_name').distinct() diff --git a/geonode/contrib/worldmap/gazetteer/migrations/0002_gazetteerattribute.py b/geonode/contrib/worldmap/gazetteer/migrations/0002_gazetteerattribute.py new file mode 100644 index 00000000000..89f82015f4a --- /dev/null +++ b/geonode/contrib/worldmap/gazetteer/migrations/0002_gazetteerattribute.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('layers', '0029_layer_service'), + ('gazetteer', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='GazetteerAttribute', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('in_gazetteer', models.BooleanField(default=False)), + ('attribute', models.OneToOneField(to='layers.Attribute')), + ], + ), + ] diff --git a/geonode/contrib/worldmap/gazetteer/migrations/0003_auto_20180316_1109.py b/geonode/contrib/worldmap/gazetteer/migrations/0003_auto_20180316_1109.py new file mode 100644 index 00000000000..94c5cd73a87 --- /dev/null +++ b/geonode/contrib/worldmap/gazetteer/migrations/0003_auto_20180316_1109.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('gazetteer', '0002_gazetteerattribute'), + ] + + operations = [ + migrations.AddField( + model_name='gazetteerattribute', + name='date_format', + field=models.TextField(null=True, blank=True), + ), + migrations.AddField( + model_name='gazetteerattribute', + name='is_end_date', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='gazetteerattribute', + name='is_start_date', + field=models.BooleanField(default=False), + ), + ] diff --git a/geonode/contrib/worldmap/gazetteer/models.py b/geonode/contrib/worldmap/gazetteer/models.py index 6052e9e846a..2a16c45495c 100644 --- a/geonode/contrib/worldmap/gazetteer/models.py +++ b/geonode/contrib/worldmap/gazetteer/models.py @@ -1,6 +1,8 @@ from django.utils.translation import ugettext as _ from django.contrib.gis.db import models +from geonode.layers.models import Attribute + # Querying postgis database for features then saving as django model object is # significantly slower than doing everything via SQL on postgis database only. # from django.modelsinspector import add_introspection_rules @@ -27,3 +29,17 @@ class GazetteerEntry(models.Model): class Meta: unique_together = (("layer_name", "layer_attribute", "feature_fid")) + + +class GazetteerAttribute(models.Model): + attribute = models.OneToOneField( + Attribute, + blank=False, + null=False) + in_gazetteer = models.BooleanField(default=False) + is_start_date = models.BooleanField(default=False) + is_end_date = models.BooleanField(default=False) + date_format = models.TextField(blank=True, null=True) + + def layer_name(self): + return self.attribute.layer.name diff --git a/geonode/contrib/worldmap/gazetteer/templates/gazetteer/edit_layer_gazetteer.html b/geonode/contrib/worldmap/gazetteer/templates/gazetteer/edit_layer_gazetteer.html new file mode 100644 index 00000000000..dc4e7f2ca1f --- /dev/null +++ b/geonode/contrib/worldmap/gazetteer/templates/gazetteer/edit_layer_gazetteer.html @@ -0,0 +1,84 @@ +{% extends "geonode_base.html" %} + +{% load i18n %} +{% load staticfiles %} +{% load bootstrap_tags %} +{% load base_tags %} +{% load url from future %} + +{% block title %}{{ layer.title }} — {{ block.super }}{% endblock %} + +{% block body_outer %} + +
+

{{ layer.title }}

+
+ +

+{% trans "By adding your layer features to the gazetteer, they will be searchable and viewable by anyone regardless of your layer's security permissions. If you do not choose at least one depict-date field, the system will use the depict-date in the layer metadata (if any)." %}

+ +
+ +
+ + {% for attribute in gazetteer_attributes %} +
+ +
+ {% endfor %} +
+ + + +
+ + +
+ +
+ + + + + + +
+ +
+ + + + + + +
+ +
+ {% trans "Return to Layer" %} + +
+ +
+ + + +{{ block.super }} +{% endblock body_outer %} diff --git a/geonode/contrib/worldmap/gazetteer/urls.py b/geonode/contrib/worldmap/gazetteer/urls.py index 2eeb2483649..0207ab2bc0d 100644 --- a/geonode/contrib/worldmap/gazetteer/urls.py +++ b/geonode/contrib/worldmap/gazetteer/urls.py @@ -1,8 +1,11 @@ from django.conf.urls import * -from geonode.gazetteer.views import search +from .views import search, edit_layer_gazetteer urlpatterns = patterns('', - url(r'^(?P[^/]+)' + + url(r'^gazetteer/edit/(?P[^/]*)$', + edit_layer_gazetteer, + name = 'edit_layer_gazetteer'), + url(r'^gazetteer/(?P[^/]+)' + '(/Service/(?P[\w\,]+))?' + '(/Project/(?P[A-Za-z0-9_-]+))?' + '(/Map/(?P[\d]+))?' + @@ -10,5 +13,5 @@ '(/StartDate/(?P[\d\s\/\-\:]+(\sBC|\sAD)?))?' + '(/EndDate/(?P[\d\s\/\-\:]+(\sBC|\sAD)?))?' + '(/User/(?P[A-Za-z0-9_-]+))?' + - '(/(?P(json|xml)))?$', search) + '(/(?P(json|xml)))?$', search), ) diff --git a/geonode/contrib/worldmap/gazetteer/utils.py b/geonode/contrib/worldmap/gazetteer/utils.py index a3742b68656..99836f2639f 100644 --- a/geonode/contrib/worldmap/gazetteer/utils.py +++ b/geonode/contrib/worldmap/gazetteer/utils.py @@ -1,19 +1,23 @@ -from array import array +import re import logging +import psycopg2 +from array import array + +from geopy import geocoders + from django.contrib.gis.geos.geometry import GEOSGeometry from django.core.urlresolvers import reverse from django.shortcuts import get_object_or_404 -from geonode.gazetteer.models import GazetteerEntry -#from psycopg2 import extras -from geonode.gazetteer.models import GazetteerEntry -from geopy import geocoders + from django.conf import settings -import psycopg2 from django.db.models import Q -from geonode.maps.models import Layer, MapLayer, Map from django.core.cache import cache -from geonode.flexidates import parse_julian_date -import re + +from geonode.maps.models import Layer, MapLayer, Map +from geonode.layers.models import Attribute +from .flexidates import parse_julian_date +from .models import GazetteerEntry +from .models import GazetteerEntry GAZETTEER_TABLE = 'gazetteer_gazetteerentry' @@ -162,7 +166,7 @@ def delete_from_gazetteer(layer_name): GazetteerEntry.objects.filter(layer_name__exact=layer_name).delete() -def add_to_gazetteer(layer_name, name_attributes, start_attribute=None, +def add_to_gazetteer(layer, name_attributes, start_attribute=None, end_attribute=None, project=None, user=None): """ Add placenames from a WorldMap layer into the gazetteer. @@ -176,16 +180,16 @@ def add_to_gazetteer(layer_name, name_attributes, start_attribute=None, def get_date_format(date_attribute): field_name = "l.\"" + date_attribute.attribute + "\"" date_format = [] - if "xsd:date" not in date_attribute.attribute_type and date_attribute.date_format is not None: + if "xsd:date" not in date_attribute.attribute_type and date_attribute.gazetteerattribute.date_format is not None: # This could be in any of multiple formats, and postgresql needs a format pattern to convert it. # User should supply this format when adding the layer attribute to the gazetteer date_format.append( "TO_CHAR(TO_DATE(CAST({name} AS TEXT), '{format}'), 'YYYY-MM-DD BC')".format( - name=field_name, format=date_attribute.date_format) + name=field_name, format=date_attribute.gazetteerattribute.date_format) ) date_format.append( "CAST(TO_CHAR(TO_DATE(CAST({name} AS TEXT), '{format}'), 'J') AS integer)".format( - name=field_name, format=date_attribute.date_format) + name=field_name, format=date_attribute.gazetteerattribute.date_format) ) elif "xsd:date" in date_attribute.attribute_type: # It's a date, convert to string @@ -206,7 +210,7 @@ def get_metadata_format(metadata_date): metadata_date)) return date_format - layer = get_object_or_404(Layer, name=layer_name) + layer_name = layer.name layer_type, geocolumn, projection = get_geometry_type(layer) namelist = "'" + "','".join(name_attributes) + "'" @@ -237,7 +241,7 @@ def get_metadata_format(metadata_date): start_format, julian_start = None, None if start_attribute is not None: - start_attribute_obj = get_object_or_404(Layer.Attribute, layer=layer, attribute=start_attribute) + start_attribute_obj = get_object_or_404(Attribute, layer=layer, attribute=start_attribute) start_dates = get_date_format(start_attribute_obj) start_format = start_dates[0] julian_start = start_dates[1] @@ -246,7 +250,7 @@ def get_metadata_format(metadata_date): end_format, julian_end = None, None if end_attribute is not None: - end_attribute_obj = get_object_or_404(Layer.Attribute, layer=layer, attribute=end_attribute) + end_attribute_obj = get_object_or_404(Attribute, layer=layer, attribute=end_attribute) end_dates = get_date_format(end_attribute_obj) end_format = end_dates[0] julian_end = end_dates[1] @@ -286,7 +290,7 @@ def get_metadata_format(metadata_date): """ for name in name_attributes: - attribute = get_object_or_404(Layer.Attribute, layer=layer, attribute=name) + attribute = get_object_or_404(Attribute, layer=layer, attribute=name) # detect column type, needed by dblink cur = conn.cursor(layer.store) @@ -298,7 +302,6 @@ def get_metadata_format(metadata_date): Update layer placenames where placename FID = layer FID and placename layer attribute = name attribute """ - updateQuery =updateTemplate.format( table=GAZETTEER_TABLE, attribute=attribute.attribute, @@ -361,8 +364,6 @@ def get_metadata_format(metadata_date): conn.close() - - def getExternalServiceResults(place_name, services): results = [] for service in services.split(',', ): @@ -379,8 +380,11 @@ def getExternalServiceResults(place_name, services): def getGoogleResults(place_name): - g = geocoders.GoogleV3(client_id=settings.GOOGLE_API_KEY, - secret_key=settings.GOOGLE_SECRET_KEY) if settings.GOOGLE_SECRET_KEY is not None else geocoders.GoogleV3() + if hasattr(settings, 'GOOGLE_SECRET_KEY'): + g = geocoders.GoogleV3(client_id=settings.GOOGLE_API_KEY, + secret_key=settings.GOOGLE_SECRET_KEY) + else: + g = geocoders.GoogleV3() try: results = g.geocode(place_name, exactly_one=False) formatted_results = [] @@ -403,7 +407,7 @@ def getNominatimResults(place_name): return [] def getConnection(layer_store=None): - dbname = settings.DATABASES[settings.GAZETTEER_DB_ALIAS]['NAME'] + dbname = settings.DATABASES['default']['NAME'] if layer_store: dbname = layer_store return psycopg2.connect( diff --git a/geonode/contrib/worldmap/gazetteer/views.py b/geonode/contrib/worldmap/gazetteer/views.py index 494d7c2779a..2b68560ed89 100755 --- a/geonode/contrib/worldmap/gazetteer/views.py +++ b/geonode/contrib/worldmap/gazetteer/views.py @@ -1,7 +1,19 @@ import json + from dicttoxml import dicttoxml + +from django.forms.models import inlineformset_factory +from django.contrib.auth.decorators import login_required +from django.forms.models import inlineformset_factory from django.http import HttpResponse -from geonode.gazetteer.utils import getGazetteerResults, getGazetteerEntry, getExternalServiceResults +from django.shortcuts import render_to_response +from django.template import RequestContext + +from geonode.layers.models import Layer, Attribute +from geonode.layers.views import _resolve_layer + +from .utils import add_to_gazetteer, getGazetteerResults, getGazetteerEntry, getExternalServiceResults +from .models import GazetteerAttribute def search(request, place_name, map=None, layer=None, start_date=None, end_date=None, project=None, services=None, user=None, format='json'): @@ -27,3 +39,71 @@ def search(request, place_name, map=None, layer=None, start_date=None, end_date= elif out_format == 'xml': return HttpResponse(dicttoxml([{'resource': post} for post in posts], attr_type=False, custom_root='response'), content_type="application/xml") + + +@login_required +def edit_layer_gazetteer( + request, + layername): + """ + Manage the layer in the gazetteer. + """ + + def set_none_if_empty(str): + if len(str) == 0: + return None + return str + + layer = _resolve_layer( + request, + layername, + 'base.change_resourcebase_metadata', + 'permissions message from grazetteer') + + if request.method == "POST": + gazetteer_name = set_none_if_empty(request.POST.get('gazetteer-name', '')) + start_attribute = set_none_if_empty(request.POST.get('start-attribute', '')) + end_attribute = set_none_if_empty(request.POST.get('end-attribute', '')) + sel_start_date_format = set_none_if_empty(request.POST.get('sel-start-date-format', '')) + sel_end_date_format = set_none_if_empty(request.POST.get('sel-end-date-format', '')) + attributes_list = request.POST.getlist('attributes') + for attribute in layer.attributes: + if attribute.attribute in attributes_list: + print 'Adding %s to gazetteer...' % attribute + gaz_att, created = GazetteerAttribute.objects.get_or_create(attribute=attribute) + gaz_att.in_gazetteer = True + if start_attribute == attribute.attribute: + gaz_att.is_start_date = True + gaz_att.date_format = sel_start_date_format + if end_attribute == attribute.attribute: + gaz_att.is_end_date = True + gaz_att.date_format = sel_end_date_format + gaz_att.save() + else: + print 'Removing %s from gazetteer...' % attribute + gaz_att, created = GazetteerAttribute.objects.get_or_create(attribute=attribute) + gaz_att.in_gazetteer = False + gaz_att.save() + # now update the gazetteer + # TODO use Celery for this + add_to_gazetteer(layer, + attributes_list, + start_attribute, + end_attribute, + gazetteer_name, + request.user.username) + + gazetteer_attributes = [] + gazetteer_attributes_date = [] + for attribute in layer.attributes: + if hasattr(attribute, 'gazetteerattribute'): + attribute.in_gazetteer = attribute.gazetteerattribute.in_gazetteer + else: + attribute.in_gazetteer = False + gazetteer_attributes.append(attribute) + template='gazetteer/edit_layer_gazetteer.html' + + return render_to_response(template, RequestContext(request, { + "layer": layer, + "gazetteer_attributes": gazetteer_attributes, + })) diff --git a/geonode/contrib/worldmap/wm_extra/__init__.py b/geonode/contrib/worldmap/wm_extra/__init__.py index adcc79e3743..8b137891791 100644 --- a/geonode/contrib/worldmap/wm_extra/__init__.py +++ b/geonode/contrib/worldmap/wm_extra/__init__.py @@ -1,3 +1 @@ -#import ipdb; ipdb.set_trace() -#print 'here' -#default_app_config = 'apps.WMExtraConfig' + diff --git a/geonode/contrib/worldmap/wm_extra/models.py b/geonode/contrib/worldmap/wm_extra/models.py index 1b2d19e17af..b514c382d70 100644 --- a/geonode/contrib/worldmap/wm_extra/models.py +++ b/geonode/contrib/worldmap/wm_extra/models.py @@ -117,31 +117,44 @@ class ExtLayer(models.Model): # else: # return service_layers[0].service # + # def queue_gazetteer_update(self): # from geonode.queue.models import GazetteerUpdateJob # if GazetteerUpdateJob.objects.filter(layer=self.id).exists() == 0: # newJob = GazetteerUpdateJob(layer=self) # newJob.save() - # - # def update_gazetteer(self): - # from geonode.gazetteer.utils import add_to_gazetteer, delete_from_gazetteer - # if not self.in_gazetteer: - # delete_from_gazetteer(self.name) - # else: - # includedAttributes = [] - # gazetteerAttributes = self.attribute_set.filter(in_gazetteer=True) - # for attribute in gazetteerAttributes: - # includedAttributes.append(attribute.attribute) - # - # startAttribute = self.attribute_set.filter(is_gaz_start_date=True)[0].attribute if self.attribute_set.filter(is_gaz_start_date=True).exists() > 0 else None - # endAttribute = self.attribute_set.filter(is_gaz_end_date=True)[0].attribute if self.attribute_set.filter(is_gaz_end_date=True).exists() > 0 else None - # - # add_to_gazetteer(self.name, - # includedAttributes, - # start_attribute=startAttribute, - # end_attribute=endAttribute, - # project=self.gazetteer_project, - # user=self.owner.username) + + def update_gazetteer(self): + from geonode.contrib.worldmap.gazetteer.utils import add_to_gazetteer, delete_from_gazetteer + if not self.in_gazetteer: + delete_from_gazetteer(self.name) + else: + includedAttributes = [] + #from geonode.contrib.worldmap.gazetteer.models import GazetteerAttribute + for attribute in self.layer.attribute_set.all(): + if hasattr(attribute, 'gazetteerattribute'): + if attribute.gazetteerattribute.in_gazetteer: + includedAttributes.append(attribute.attribute) + + print includedAttributes + + #includedAttributes = [] + #gazetteerAttributes = self.attribute_set.filter(in_gazetteer=True) + #for attribute in gazetteerAttributes: + # includedAttributes.append(attribute.attribute) + + # TODO implement this + startAttribute = None + endAttribute = None + #startAttribute = self.attribute_set.filter(is_gaz_start_date=True)[0].attribute if self.attribute_set.filter(is_gaz_start_date=True).exists() > 0 else None + #endAttribute = self.attribute_set.filter(is_gaz_end_date=True)[0].attribute if self.attribute_set.filter(is_gaz_end_date=True).exists() > 0 else None + + add_to_gazetteer(self.layer.name, + includedAttributes, + start_attribute=startAttribute, + end_attribute=endAttribute, + project=self.gazetteer_project, + user=self.layer.owner.username) # this must be added in a pre-delete signal # if settings.USE_GAZETTEER and instance.in_gazetteer: diff --git a/geonode/contrib/worldmap/wm_extra/urls.py b/geonode/contrib/worldmap/wm_extra/urls.py index f826f91704c..58d70d66235 100644 --- a/geonode/contrib/worldmap/wm_extra/urls.py +++ b/geonode/contrib/worldmap/wm_extra/urls.py @@ -5,7 +5,7 @@ from .views import (proxy, ajax_snapshot_history, ajax_layer_update, ajax_layer_edit_check, upload_layer, create_pg_layer, ajax_increment_layer_stats, add_layer_wm, new_map_wm, new_map_json_wm, - map_view_wm, map_json_wm, map_detail_wm, add_endpoint, printmap) + map_view_wm, map_json_wm, map_detail_wm, add_endpoint, printmap, ) from tastypie.api import Api from .api.resources import (LayerResource, TagResource, TopicCategoryResource, ActionAllResource, ActionLayerCreateResource, ActionLayerDeleteResource, diff --git a/geonode/layers/templates/layers/layer_detail.html b/geonode/layers/templates/layers/layer_detail.html index 0761ac6d925..3ef6b3802d8 100644 --- a/geonode/layers/templates/layers/layer_detail.html +++ b/geonode/layers/templates/layers/layer_detail.html @@ -445,6 +445,9 @@

{% trans "Metadata" %}

{% trans "Wizard" %} {% trans "Advanced Edit" %} {% trans "Upload Metadata" %} + {% if USE_WORLDMAP %} + {% trans "Gazetteer" %} + {% endif %} {% endif %} {% if GEOSERVER_BASE_URL and not resource.service %} diff --git a/geonode/urls.py b/geonode/urls.py index adf96f7324f..b0ff2f57e60 100644 --- a/geonode/urls.py +++ b/geonode/urls.py @@ -69,6 +69,7 @@ # WorldMap if settings.USE_WORLDMAP: urlpatterns += [url(r'', include('geonode.contrib.worldmap.wm_extra.urls', namespace='worldmap'))] + urlpatterns += [url(r'', include('geonode.contrib.worldmap.gazetteer.urls', namespace='gazetteer'))] urlpatterns += patterns('', # Layer views diff --git a/requirements.txt b/requirements.txt index 391f43fdb4b..b2a4ec30e26 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,6 +16,7 @@ certifi==2017.7.27.1 chardet==3.0.4 coverage==4.4.1 decorator==4.1.2 +dicttoxml==1.7.4 diff-match-patch==20121119 dj-database-url==0.4.2 django-allauth==0.34.0 @@ -63,6 +64,7 @@ Faker==0.8.4 flake8==2.5.4 funcsigs==1.0.2 geolinks==0.2.0 +geopy==1.11.0 geonode-agon-ratings==0.3.5 geonode-announcements==1.0.8 geonode-arcrest==10.2