Skip to content

Commit

Permalink
Merge pull request #3807 from geosolutions-it/master
Browse files Browse the repository at this point in the history 
 - Correct management of SLDs / Add GWC filterParameter to SLDs
  • Loading branch information
Alessio Fabiani authored May 22, 2018
2 parents df94109 + 6c0e8ca commit df12893
Show file tree
Hide file tree
Showing 28 changed files with 7,738 additions and 3,389 deletions.
60 changes: 31 additions & 29 deletions geonode/base/management/commands/updategeoip.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,6 @@
import logging
import gzip
import tarfile
import urllib2 as urllib
import traceback

from six import StringIO

from django.conf import settings
from django.core.management.base import BaseCommand
Expand Down Expand Up @@ -63,20 +59,34 @@ def handle(self, *args, **options):
if not options['overwrite'] and os.path.exists(fname):
logger.warning("File exists, won't overwrite %s", fname)
return
logger.info("Requesting %s", options['url'])

r = urllib.urlopen(options['url'], timeout=10.0)

if OLD_FORMAT:
self.handle_old_format(r, fname)
from tqdm import tqdm
import requests
import math
# Streaming, so we can iterate over the response.
r = requests.get(options['url'], stream=True, timeout=10)
# Total size in bytes.
total_size = int(r.headers.get('content-length', 0))
logger.info("Requesting %s", options['url'])
block_size = 1024
wrote = 0
with open('output.bin', 'wb') as f:
for data in tqdm(r.iter_content(block_size), total=math.ceil(total_size//block_size) , unit='KB', unit_scale=False):
wrote = wrote + len(data)
f.write(data)
logger.info(" total_size [%d] / wrote [%d] " % (total_size, wrote))
if total_size != 0 and wrote != total_size:
logger.info("ERROR, something went wrong")
else:
self.handle_new_format(r, fname)
if OLD_FORMAT:
self.handle_old_format(open('output.bin', 'r'), fname)
else:
self.handle_new_format(open('output.bin', 'r'), fname)


def handle_new_format(self, r, fname):
def handle_new_format(self, f, fname):
try:
data = StringIO(r.read())
with tarfile.open(fileobj=data) as zfile:
with tarfile.open(fileobj=f) as zfile:
members = zfile.getmembers()
for m in members:
if m.name.endswith('GeoLite2-City.mmdb'):
Expand All @@ -86,36 +96,28 @@ def handle_new_format(self, r, fname):
logger.info("Writing to %s", fname)
tofile.write(fromfile.read())
except Exception, err:
logger.error("Cannot extract %s and write to %s: %s", m, fname, err, exc_info=err)
logger.error("Cannot extract %s and write to %s: %s", m, fname, err, exc_info=err)
try:
os.remove(fname)
except OSError:
logger.debug("Could not delete file %s", fname)
return
except Exception, err:
logger.error("Cannot process %s: %s", r, err, exc_info=err)
logger.error("Cannot process %s: %s", f, err, exc_info=err)


def handle_old_format(self, r, fname):
def handle_old_format(self, f, fname):
try:
data = StringIO(r.read())
with gzip.GzipFile(fileobj=data) as zfile:
with gzip.GzipFile(fileobj=f) as zfile:
logger.info("Writing to %s", fname)
try:
os.remove(fname)
except OSError:
logger.debug('Could not delete file %s' % fname)
with open(fname, 'wb') as tofile:
try:
tofile.write(zfile.read())
except:
tb = traceback.format_exc()
logger.error(tb)

except Exception, err:
logger.error("Cannot extract %s and write to %s: %s", f, fname, err, exc_info=err)
try:
os.remove(fname)
except OSError:
logger.debug('Could not delete file %s' % fname)
except:
tb = traceback.format_exc()
logger.error(tb)
except Exception, err:
logger.error("Cannot process %s: %s", f, err, exc_info=err)
65 changes: 53 additions & 12 deletions geonode/geoserver/helpers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -304,10 +304,6 @@ def fixup_style(cat, resource, style):
lyr.default_style = style
logger.info("Saving changes to %s", lyr)
cat.save(lyr)

# Invalidate GeoWebCache for the updated resource
_invalidate_geowebcache_layer(resource)

logger.info("Successfully updated %s", lyr)


Expand Down Expand Up @@ -363,9 +359,6 @@ def set_layer_style(saved_layer, title, sld, base_file=None):
except Exception as e:
logger.exception(e)

# Invalidate GeoWebCache for the updated resource
_invalidate_geowebcache_layer(saved_layer.alternate)


def cascading_delete(cat, layer_name):
resource = None
Expand Down Expand Up @@ -924,22 +917,27 @@ def set_styles(layer, gs_catalog):

if not default_style:
try:
default_style = gs_catalog.get_style(layer.name, workspace=settings.DEFAULT_WORKSPACE) \
default_style = gs_catalog.get_style(layer.name, workspace=layer.workspace) \
or gs_catalog.get_style(layer.name)
gs_layer.default_style = default_style
gs_catalog.save(gs_layer)
except:
logger.exception("GeoServer Layer Default Style issues!")

if default_style:
layer.default_style = save_style(default_style)
# make sure we are not using a defaul SLD (which won't be editable)
if not default_style.workspace or default_style.workspace != layer.workspace:
sld_body = default_style.sld_body
gs_catalog.create_style(layer.name, sld_body, raw=True, workspace=layer.workspace)
style = gs_catalog.get_style(layer.name, workspace=layer.workspace)
else:
style = default_style
layer.default_style = save_style(style)
# FIXME: This should remove styles that are no longer valid
style_set.append(layer.default_style)

try:
if gs_layer.styles:
alt_styles = gs_layer.styles

for alt_style in alt_styles:
if alt_style:
style_set.append(save_style(alt_style))
Expand All @@ -955,7 +953,6 @@ def set_styles(layer, gs_catalog):

Layer.objects.filter(id=layer.id).update(**to_update)
layer.refresh_from_db()
return layer


def save_style(gs_style):
Expand Down Expand Up @@ -1472,6 +1469,49 @@ def wps_execute_layer_attribute_statistics(layer_name, field):
# exml = etree.fromstring(response)


def _stylefilterparams_geowebcache_layer(layer_name):
http = httplib2.Http()
username, password = ogc_server_settings.credentials
auth = base64.encodestring(username + ':' + password)
# http.add_credentials(username, password)
headers = {
"Content-Type": "text/xml",
"Authorization": "Basic " + auth
}
url = '%sgwc/rest/layers/%s.xml' % (ogc_server_settings.LOCATION, layer_name)

# read GWC configuration
method = "GET"
response, _ = http.request(url, method, headers=headers)
if response.status != 200:
line = "Error {0} reading Style Filter Params GeoWebCache at {1}".format(
response.status, url
)
logger.error(line)
return

# check/write GWC filter parameters
import xml.etree.ElementTree as ET
body = None
tree = ET.fromstring(_)
param_filters = tree.findall('parameterFilters')
if param_filters and len(param_filters) > 0:
if not param_filters[0].findall('styleParameterFilter'):
style_filters_xml = "<styleParameterFilter><key>STYLES</key>\
<defaultValue></defaultValue></styleParameterFilter>"
style_filters_elem = ET.fromstring(style_filters_xml)
param_filters[0].append(style_filters_elem)
body = ET.tostring(tree)
if body:
method = "POST"
response, _ = http.request(url, method, body=body, headers=headers)
if response.status != 200:
line = "Error {0} writing Style Filter Params GeoWebCache at {1}".format(
response.status, url
)
logger.error(line)


def _invalidate_geowebcache_layer(layer_name, url=None):
http = httplib2.Http()
username, password = ogc_server_settings.credentials
Expand Down Expand Up @@ -1549,6 +1589,7 @@ def style_update(request, url):

# Invalidate GeoWebCache so it doesn't retain old style in tiles
try:
_stylefilterparams_geowebcache_layer(layer_name)
_invalidate_geowebcache_layer(layer_name)
except:
pass
Expand Down
70 changes: 41 additions & 29 deletions geonode/geoserver/signals.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,17 @@
from geonode import GeoNodeException
from geonode.decorators import on_ogc_backend
from geonode.geoserver.ows import wcs_links, wfs_links, wms_links
from geonode.geoserver.helpers import cascading_delete, set_attributes_from_geoserver
from geonode.geoserver.helpers import set_styles, gs_catalog
from geonode.geoserver.helpers import ogc_server_settings
from geonode.geoserver.helpers import create_gs_thumbnail
from geonode.geoserver.upload import geoserver_upload
from geonode.base.models import ResourceBase
from geonode.base.models import Link
from geonode.geoserver.helpers import (cascading_delete,
set_attributes_from_geoserver,
set_styles,
set_layer_style,
gs_catalog,
ogc_server_settings,
create_gs_thumbnail,
_stylefilterparams_geowebcache_layer,
_invalidate_geowebcache_layer)
from geonode.base.models import ResourceBase, Link
from geonode.people.models import Profile
from geonode.layers.models import Layer
from geonode.social.signals import json_serializer_producer
Expand Down Expand Up @@ -172,30 +176,26 @@ def geoserver_post_save_local(instance, *args, **kwargs):
e.args = (msg,)
logger.exception(e)

gs_layer = gs_catalog.get_layer(instance.name)

if not gs_layer:
gs_layer = gs_catalog.get_layer(instance.alternate)

if gs_layer and instance.poc:
# Update Attribution link
if instance.poc:
# gsconfig now utilizes an attribution dictionary
gs_layer.attribution = {'title': str(instance.poc),
'width': None,
'height': None,
'href': None,
'url': None,
'type': None}
gs_resource.attribution = {'title': str(instance.poc),
'width': None,
'height': None,
'href': None,
'url': None,
'type': None}
profile = Profile.objects.get(username=instance.poc.username)
gs_layer.attribution_link = settings.SITEURL[
gs_resource.attribution_link = settings.SITEURL[
:-1] + profile.get_absolute_url()
# gs_layer should only be called if
# gs_resource should only be called if
# ogc_server_settings.BACKEND_WRITE_ENABLED == True
if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True):
try:
gs_catalog.save(gs_layer)
gs_catalog.save(gs_resource)
except geoserver.catalog.FailedRequestError as e:
msg = ('Error while trying to save layer named %s in GeoServer, '
'try to use: "%s"' % (gs_layer, str(e)))
'try to use: "%s"' % (gs_resource, str(e)))
e.args = (msg,)
logger.exception(e)

Expand All @@ -205,6 +205,24 @@ def geoserver_post_save_local(instance, *args, **kwargs):
else:
return

# Save layer attributes
set_attributes_from_geoserver(instance)

# Save layer styles
set_styles(instance, gs_catalog)

# set SLD
sld = instance.default_style.sld_body
if sld:
set_layer_style(instance, instance.alternate, sld)

# Invalidate GeoWebCache for the updated resource
try:
_stylefilterparams_geowebcache_layer(instance.alternate)
_invalidate_geowebcache_layer(instance.alternate)
except:
pass

if instance.storeType == "remoteStore":
# Save layer attributes
set_attributes_from_geoserver(instance)
Expand All @@ -219,7 +237,7 @@ def geoserver_post_save_local(instance, *args, **kwargs):
* Download links (WMS, WCS or WFS and KML)
* Styles (SLD)
"""
# instance.name = instance.name or gs_layer.name
# instance.name = instance.name or gs_resource.name
# instance.title = instance.title or gs_resource.title
instance.abstract = gs_resource.abstract or ''
instance.workspace = gs_resource.store.workspace.name
Expand Down Expand Up @@ -564,12 +582,6 @@ def command_url(command):
if created:
Link.objects.filter(pk=link.pk).update(url=tile_url)

# Save layer attributes
set_attributes_from_geoserver(instance)

# Save layer styles
set_styles(instance, gs_catalog)

# NOTTODO by simod: we should not do this!
# need to be removed when fixing #2015
catalogue_post_save(instance, Layer)
Expand Down
29 changes: 23 additions & 6 deletions geonode/geoserver/views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,9 @@
from .tasks import geoserver_update_layers
from geonode.utils import json_response, _get_basic_auth_info
from geoserver.catalog import FailedRequestError, ConflictingDataError
from .helpers import (get_stores, ogc_server_settings,
extract_name_from_sld, set_styles, style_update,
create_gs_thumbnail, _invalidate_geowebcache_layer)
from .helpers import (get_stores, ogc_server_settings, extract_name_from_sld, set_styles,
style_update, create_gs_thumbnail,
_stylefilterparams_geowebcache_layer, _invalidate_geowebcache_layer)

from django_basic_auth import logged_in_or_basicauth
from django.views.decorators.csrf import csrf_exempt
Expand Down Expand Up @@ -123,6 +123,13 @@ def layer_style(request, layername):
s for s in layer.styles if s.name != style_name] + [old_default]
layer.save()

# Invalidate GeoWebCache for the updated resource
try:
_stylefilterparams_geowebcache_layer(layer.alternate)
_invalidate_geowebcache_layer(layer.alternate)
except:
pass

return HttpResponse(
"Default style for %s changed to %s" %
(layer.name, style_name), status=200)
Expand Down Expand Up @@ -188,8 +195,14 @@ def respond(*args, **kw):
except ConflictingDataError:
return respond(errors="""A layer with this name exists. Select
the update option if you want to update.""")

# Invalidate GeoWebCache for the updated resource
_invalidate_geowebcache_layer(layer.alternate)
try:
_stylefilterparams_geowebcache_layer(layer.alternate)
_invalidate_geowebcache_layer(layer.alternate)
except:
pass

return respond(
body={
'success': True,
Expand Down Expand Up @@ -302,10 +315,14 @@ def layer_style_manage(request, layername):
cat.save(gs_layer)

# Save to Django
layer = set_styles(layer, cat)
set_styles(layer, cat)

# Invalidate GeoWebCache for the updated resource
_invalidate_geowebcache_layer(layer.alternate)
try:
_stylefilterparams_geowebcache_layer(layer.alternate)
_invalidate_geowebcache_layer(layer.alternate)
except:
pass

return HttpResponseRedirect(
reverse(
Expand Down
Loading

0 comments on commit df12893

Please sign in to comment.