diff --git a/geonode/base/models.py b/geonode/base/models.py index 596c40c8e7b..b6deb2ae1d4 100644 --- a/geonode/base/models.py +++ b/geonode/base/models.py @@ -799,7 +799,7 @@ def csw_crs(self): @property def group_name(self): if self.group: - return str(self.group) + return str(self.group).encode("utf-8", "replace") return None @property @@ -902,13 +902,13 @@ def metadata_completeness(self): return '{}%'.format(len(filled_fields) * 100 / len(required_fields)) def keyword_list(self): - return [kw.name for kw in self.keywords.all()] + return [kw.name.encode("utf-8", "replace") for kw in self.keywords.all()] def keyword_slug_list(self): - return [kw.slug for kw in self.keywords.all()] + return [kw.slug.encode("utf-8", "replace") for kw in self.keywords.all()] def region_name_list(self): - return [region.name for region in self.regions.all()] + return [region.name.encode("utf-8", "replace") for region in self.regions.all()] def spatial_representation_type_string(self): if hasattr(self.spatial_representation_type, 'identifier'): diff --git a/geonode/geoserver/signals.py b/geonode/geoserver/signals.py index e4297681b25..9db98eed857 100644 --- a/geonode/geoserver/signals.py +++ b/geonode/geoserver/signals.py @@ -243,14 +243,17 @@ def geoserver_post_save_local(instance, *args, **kwargs): instance.workspace = gs_resource.store.workspace.name instance.store = gs_resource.store.name - bbox = gs_resource.native_bbox - - # Set bounding box values - instance.bbox_x0 = bbox[0] - instance.bbox_x1 = bbox[1] - instance.bbox_y0 = bbox[2] - instance.bbox_y1 = bbox[3] - instance.srid = bbox[4] + try: + bbox = gs_resource.native_bbox + + # Set bounding box values + instance.bbox_x0 = bbox[0] + instance.bbox_x1 = bbox[1] + instance.bbox_y0 = bbox[2] + instance.bbox_y1 = bbox[3] + instance.srid = bbox[4] + except BaseException: + pass if instance.srid: instance.srid_url = "http://www.spatialreference.org/ref/" + \ @@ -271,14 +274,17 @@ def geoserver_post_save_local(instance, *args, **kwargs): gs_catalog.save(gs_resource) if not settings.FREETEXT_KEYWORDS_READONLY: - if len(instance.keyword_list()) == 0 and gs_resource.keywords: - for keyword in gs_resource.keywords: - if keyword not in instance.keyword_list(): - instance.keywords.add(keyword) + try: + if len(instance.keyword_list()) == 0 and gs_resource.keywords: + for keyword in gs_resource.keywords: + if keyword not in instance.keyword_list(): + instance.keywords.add(keyword) + except BaseException: + pass if any(instance.keyword_list()): keywords = instance.keyword_list() - gs_resource.keywords = list(set(keywords)) + gs_resource.keywords = [kw.decode("utf-8", "replace") for kw in list(set(keywords))] # gs_resource should only be called if # ogc_server_settings.BACKEND_WRITE_ENABLED == True @@ -321,7 +327,10 @@ def geoserver_post_save_local(instance, *args, **kwargs): # store the resource to avoid another geoserver call in the post_save instance.gs_resource = gs_resource - bbox = gs_resource.native_bbox + try: + bbox = gs_resource.native_bbox + except BaseException: + bbox = instance.bbox dx = float(bbox[1]) - float(bbox[0]) dy = float(bbox[3]) - float(bbox[2]) @@ -337,7 +346,10 @@ def geoserver_post_save_local(instance, *args, **kwargs): instance.bbox_x1, instance.bbox_y1]) # Create Raw Data download link - path = gs_resource.dom.findall('nativeName') + try: + path = gs_resource.dom.findall('nativeName') + except BaseException: + path = instance.alternate download_url = urljoin(settings.SITEURL, reverse('download', args=[instance.id])) Link.objects.get_or_create(resource=instance.resourcebase_ptr, @@ -397,7 +409,10 @@ def geoserver_post_save_local(instance, *args, **kwargs): url=ogc_server_settings.public_url, repo_name=geogig_repo_name) - path = gs_resource.dom.findall('nativeName') + try: + path = gs_resource.dom.findall('nativeName') + except BaseException: + path = instance.alternate if path: path = 'path={path}'.format(path=path[0].text) diff --git a/geonode/geoserver/upload.py b/geonode/geoserver/upload.py index ed74216d1dc..5e728543f76 100644 --- a/geonode/geoserver/upload.py +++ b/geonode/geoserver/upload.py @@ -180,8 +180,13 @@ def geoserver_upload( # Step 6. Make sure our data always has a valid projection # FIXME: Put this in gsconfig.py logger.info('>>> Step 6. Making sure [%s] has a valid projection' % name) - if gs_resource.native_bbox is None: - box = gs_resource.native_bbox[:4] + _native_bbox = None + try: + _native_bbox = gs_resource.native_bbox + except BaseException: + pass + if _native_bbox and len(_native_bbox) >= 5 and _native_bbox[4:5][0] == 'EPSG:4326': + box = _native_bbox[:4] minx, maxx, miny, maxy = [float(a) for a in box] if -180 <= minx <= 180 and -180 <= maxx <= 180 and \ - 90 <= miny <= 90 and -90 <= maxy <= 90: @@ -190,7 +195,7 @@ def geoserver_upload( # If GeoServer couldn't figure out the projection, we just # assume it's lat/lon to avoid a bad GeoServer configuration - gs_resource.latlon_bbox = gs_resource.native_bbox + gs_resource.latlon_bbox = _native_bbox gs_resource.projection = "EPSG:4326" cat.save(gs_resource) else: diff --git a/geonode/geoserver/views.py b/geonode/geoserver/views.py index 81c7b1f2e3e..cb2607b143e 100644 --- a/geonode/geoserver/views.py +++ b/geonode/geoserver/views.py @@ -839,7 +839,7 @@ def get_capabilities(request, layerid=None, user=None, 'catalogue_url': settings.CATALOGUE['default']['URL'], } gc_str = tpl.render(ctx) - gc_str = gc_str.encode("utf-8") + gc_str = gc_str.encode("utf-8", "replace") layerelem = etree.XML(gc_str) rootdoc = etree.ElementTree(layerelem) except Exception as e: diff --git a/geonode/layers/models.py b/geonode/layers/models.py index 5df5c480434..a89df0e6616 100644 --- a/geonode/layers/models.py +++ b/geonode/layers/models.py @@ -495,7 +495,7 @@ class Attribute(models.Model): def __unicode__(self): return "%s" % self.attribute_label.encode( - "utf-8") if self.attribute_label else self.attribute.encode("utf-8") + "utf-8", "replace") if self.attribute_label else self.attribute.encode("utf-8", "replace") def unique_values_as_list(self): return self.unique_values.split(',') diff --git a/geonode/layers/views.py b/geonode/layers/views.py index d99088a275f..d7b7fd961f8 100644 --- a/geonode/layers/views.py +++ b/geonode/layers/views.py @@ -27,7 +27,7 @@ import uuid import decimal import re - +import cPickle as pickle from django.db.models import Q from celery.exceptions import TimeoutError @@ -239,7 +239,7 @@ def layer_upload(request, template='upload/layer_upload.html'): user=request.user).order_by('-date') if latest_uploads.count() > 0: upload_session = latest_uploads[0] - upload_session.error = str(error) + upload_session.error = pickle.dumps(error).decode("utf-8", "replace") upload_session.traceback = traceback.format_exc(tb) upload_session.context = log_snippet(CONTEXT_LOG_FILE) upload_session.save() @@ -286,6 +286,16 @@ def layer_upload(request, template='upload/layer_upload.html'): layer_name = saved_layer.alternate if hasattr( saved_layer, 'alternate') else name request.add_resource('layer', layer_name) + _keys = ['info', 'errors'] + for _k in _keys: + if _k in out: + if isinstance(out[_k], unicode) or isinstance( + out[_k], str): + out[_k] = out[_k].decode(saved_layer.charset).encode("utf-8") + elif isinstance(out[_k], dict): + for key, value in out[_k].iteritems(): + out[_k][key] = out[_k][key].decode(saved_layer.charset).encode("utf-8") + out[_k][key.decode(saved_layer.charset).encode("utf-8")] = out[_k].pop(key) return HttpResponse( json.dumps(out), content_type='application/json', diff --git a/geonode/tests/data/ming_female_1.zip b/geonode/tests/data/ming_female_1.zip new file mode 100644 index 00000000000..14f1baa70a2 Binary files /dev/null and b/geonode/tests/data/ming_female_1.zip differ diff --git a/geonode/tests/data/ming_female_1/ming_female_1.cst b/geonode/tests/data/ming_female_1/ming_female_1.cst new file mode 100644 index 00000000000..3ad133c048f --- /dev/null +++ b/geonode/tests/data/ming_female_1/ming_female_1.cst @@ -0,0 +1 @@ +UTF-8 \ No newline at end of file diff --git a/geonode/tests/data/ming_female_1/ming_female_1.dbf b/geonode/tests/data/ming_female_1/ming_female_1.dbf new file mode 100644 index 00000000000..97fa67ed595 Binary files /dev/null and b/geonode/tests/data/ming_female_1/ming_female_1.dbf differ diff --git a/geonode/tests/data/ming_female_1/ming_female_1.prj b/geonode/tests/data/ming_female_1/ming_female_1.prj new file mode 100644 index 00000000000..ebb59326939 --- /dev/null +++ b/geonode/tests/data/ming_female_1/ming_female_1.prj @@ -0,0 +1 @@ +GEOGCS["Xian 1980", DATUM["Xian 1980", SPHEROID["IAG 1975", 6378140.0, 298.257, AUTHORITY["EPSG","7049"]], AUTHORITY["EPSG","6610"]], PRIMEM["Greenwich", 0.0, AUTHORITY["EPSG","8901"]], UNIT["degree", 0.017453292519943295], AXIS["Geodetic longitude", EAST], AXIS["Geodetic latitude", NORTH], AUTHORITY["EPSG","4610"]] \ No newline at end of file diff --git a/geonode/tests/data/ming_female_1/ming_female_1.shp b/geonode/tests/data/ming_female_1/ming_female_1.shp new file mode 100644 index 00000000000..4c35f66899e Binary files /dev/null and b/geonode/tests/data/ming_female_1/ming_female_1.shp differ diff --git a/geonode/tests/data/ming_female_1/ming_female_1.shx b/geonode/tests/data/ming_female_1/ming_female_1.shx new file mode 100644 index 00000000000..ad3c8db53fa Binary files /dev/null and b/geonode/tests/data/ming_female_1/ming_female_1.shx differ diff --git a/geonode/tests/data/ming_female_1/wfsrequest.txt b/geonode/tests/data/ming_female_1/wfsrequest.txt new file mode 100644 index 00000000000..066e912c7c0 --- /dev/null +++ b/geonode/tests/data/ming_female_1/wfsrequest.txt @@ -0,0 +1 @@ +http://amap.zju.edu.cn:8080/geoserver/wfs?access_token=vKa0jUDKSIJVS8mxn3QbeXlNKbxFlO?format_options=charset%3AUTF-8&typename=geonode%3Aming_female_1&outputFormat=SHAPE-ZIP&version=1.0.0&service=WFS&request=GetFeature&access_token=vKa0jUDKSIJVS8mxn3QbeXlNKbxFlO \ No newline at end of file diff --git a/geonode/tests/data/zhejiang_yangcan_yanyu.zip b/geonode/tests/data/zhejiang_yangcan_yanyu.zip new file mode 100644 index 00000000000..6251e319851 Binary files /dev/null and b/geonode/tests/data/zhejiang_yangcan_yanyu.zip differ diff --git a/geonode/tests/data/zhejiang_yangcan_yanyu/wfsrequest.txt b/geonode/tests/data/zhejiang_yangcan_yanyu/wfsrequest.txt new file mode 100644 index 00000000000..393c4117973 --- /dev/null +++ b/geonode/tests/data/zhejiang_yangcan_yanyu/wfsrequest.txt @@ -0,0 +1 @@ +http://128.31.22.46:8080/geoserver/wfs?access_token=TCSt2nAzYCZeeSF9jTDULPff6YUWrI?outputFormat=SHAPE-ZIP&service=WFS&srs=EPSG%3A4610&request=GetFeature&format_options=charset%3AUTF-8&typename=geonode%3Azhejiang_yangcan_yanyu&version=1.0.0&access_token=TCSt2nAzYCZeeSF9jTDULPff6YUWrI \ No newline at end of file diff --git a/geonode/tests/data/zhejiang_yangcan_yanyu/zhejiang_yangcan_yanyu.cst b/geonode/tests/data/zhejiang_yangcan_yanyu/zhejiang_yangcan_yanyu.cst new file mode 100644 index 00000000000..3ad133c048f --- /dev/null +++ b/geonode/tests/data/zhejiang_yangcan_yanyu/zhejiang_yangcan_yanyu.cst @@ -0,0 +1 @@ +UTF-8 \ No newline at end of file diff --git a/geonode/tests/data/zhejiang_yangcan_yanyu/zhejiang_yangcan_yanyu.dbf b/geonode/tests/data/zhejiang_yangcan_yanyu/zhejiang_yangcan_yanyu.dbf new file mode 100644 index 00000000000..9e2303913d3 Binary files /dev/null and b/geonode/tests/data/zhejiang_yangcan_yanyu/zhejiang_yangcan_yanyu.dbf differ diff --git a/geonode/tests/data/zhejiang_yangcan_yanyu/zhejiang_yangcan_yanyu.prj b/geonode/tests/data/zhejiang_yangcan_yanyu/zhejiang_yangcan_yanyu.prj new file mode 100644 index 00000000000..ebb59326939 --- /dev/null +++ b/geonode/tests/data/zhejiang_yangcan_yanyu/zhejiang_yangcan_yanyu.prj @@ -0,0 +1 @@ +GEOGCS["Xian 1980", DATUM["Xian 1980", SPHEROID["IAG 1975", 6378140.0, 298.257, AUTHORITY["EPSG","7049"]], AUTHORITY["EPSG","6610"]], PRIMEM["Greenwich", 0.0, AUTHORITY["EPSG","8901"]], UNIT["degree", 0.017453292519943295], AXIS["Geodetic longitude", EAST], AXIS["Geodetic latitude", NORTH], AUTHORITY["EPSG","4610"]] \ No newline at end of file diff --git a/geonode/tests/data/zhejiang_yangcan_yanyu/zhejiang_yangcan_yanyu.shp b/geonode/tests/data/zhejiang_yangcan_yanyu/zhejiang_yangcan_yanyu.shp new file mode 100644 index 00000000000..ef6831b45a7 Binary files /dev/null and b/geonode/tests/data/zhejiang_yangcan_yanyu/zhejiang_yangcan_yanyu.shp differ diff --git a/geonode/tests/data/zhejiang_yangcan_yanyu/zhejiang_yangcan_yanyu.shx b/geonode/tests/data/zhejiang_yangcan_yanyu/zhejiang_yangcan_yanyu.shx new file mode 100644 index 00000000000..a530bbfcd27 Binary files /dev/null and b/geonode/tests/data/zhejiang_yangcan_yanyu/zhejiang_yangcan_yanyu.shx differ diff --git a/geonode/tests/integration.py b/geonode/tests/integration.py index a782481f908..a4527700dfe 100644 --- a/geonode/tests/integration.py +++ b/geonode/tests/integration.py @@ -512,13 +512,8 @@ def test_layer_upload_metadata(self): # layer have projection file, but has no valid srid self.assertEqual( str(e), - "Invalid Layers. " - "Needs an authoritative SRID in its CRS to be accepted") - # except: - # # Sometimes failes with the message: - # # UploadError: Could not save the layer air_runways, - # # there was an upload error: Error occured unzipping file - # pass + "GeoServer failed to detect the projection for layer [air_runways]. " + "It doesn't look like EPSG:4326, so backing out the layer.") finally: # Clean up and completely delete the layer if uploaded: @@ -610,11 +605,57 @@ def test_layer_zip_upload_metadata(self): uploaded.metadata_xml = thelayer_metadata regions_resolved, regions_unresolved = resolve_regions(regions) self.assertIsNotNone(regions_resolved) - # except: - # # Sometimes failes with the message: - # # UploadError: Could not save the layer air_runways, - # # there was an upload error: Error occured unzipping file - # pass + finally: + # Clean up and completely delete the layer + if uploaded: + uploaded.delete() + + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + @timeout_decorator.timeout(LOCAL_TIMEOUT) + def test_layer_zip_upload_non_utf8(self): + """Test uploading a layer with non UTF-8 attributes names""" + uploaded = None + PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__)) + thelayer_path = os.path.join( + PROJECT_ROOT, + 'data/zhejiang_yangcan_yanyu') + thelayer_zip = os.path.join( + PROJECT_ROOT, + 'data/', + 'zhejiang_yangcan_yanyu.zip') + try: + if os.path.exists(thelayer_zip): + os.remove(thelayer_zip) + if os.path.exists(thelayer_path) and not os.path.exists(thelayer_zip): + zip_dir(thelayer_path, thelayer_zip) + if os.path.exists(thelayer_zip): + uploaded = file_upload(thelayer_zip, overwrite=True, charset='windows-1258') + self.assertEquals(uploaded.title, 'Zhejiang Yangcan Yanyu') + self.assertEquals(len(uploaded.keyword_list()), 2) + self.assertEquals(uploaded.constraints_other, None) + finally: + # Clean up and completely delete the layer + if uploaded: + uploaded.delete() + + uploaded = None + thelayer_path = os.path.join( + PROJECT_ROOT, + 'data/ming_female_1') + thelayer_zip = os.path.join( + PROJECT_ROOT, + 'data/', + 'ming_female_1.zip') + try: + if os.path.exists(thelayer_zip): + os.remove(thelayer_zip) + if os.path.exists(thelayer_path) and not os.path.exists(thelayer_zip): + zip_dir(thelayer_path, thelayer_zip) + if os.path.exists(thelayer_zip): + uploaded = file_upload(thelayer_zip, overwrite=True, charset='windows-1258') + self.assertEquals(uploaded.title, 'Ming Female 1') + self.assertEquals(len(uploaded.keyword_list()), 2) + self.assertEquals(uploaded.constraints_other, None) finally: # Clean up and completely delete the layer if uploaded: diff --git a/geonode/upload/files.py b/geonode/upload/files.py index 99d82ca71ae..9fdfbd44b4d 100644 --- a/geonode/upload/files.py +++ b/geonode/upload/files.py @@ -25,9 +25,9 @@ ''' import os.path -from geoserver.resource import FeatureType -from geoserver.resource import Coverage +from geonode.utils import fixup_shp_columnnames +from geoserver.resource import FeatureType, Coverage from django.utils.translation import ugettext as _ from UserList import UserList @@ -256,17 +256,24 @@ def get_scan_hint(valid_extensions): return result -def scan_file(file_name, scan_hint=None): +def scan_file(file_name, scan_hint=None, charset=None): '''get a list of SpatialFiles for the provided file''' if not os.path.exists(file_name): raise Exception(_("Could not access to uploaded data.")) dirname = os.path.dirname(file_name) if zipfile.is_zipfile(file_name): - paths, kept_zip = _process_zip(file_name, dirname, scan_hint=scan_hint) + paths, kept_zip = _process_zip(file_name, + dirname, + scan_hint=scan_hint, + charset=charset) archive = file_name if kept_zip else None else: - paths = [os.path.join(dirname, p) for p in os.listdir(dirname)] + paths = [] + for p in os.listdir(dirname): + _f = os.path.join(dirname, p) + fixup_shp_columnnames(_f, charset) + paths.append(_f) archive = None if paths is not None: safe_paths = _rename_files(paths) @@ -305,7 +312,7 @@ def scan_file(file_name, scan_hint=None): return SpatialFiles(dirname, found, archive=archive) -def _process_zip(zip_path, destination_dir, scan_hint=None): +def _process_zip(zip_path, destination_dir, scan_hint=None, charset=None): """Perform sanity checks on uploaded zip file This function will check if the zip file's contents have legal names. @@ -318,10 +325,10 @@ def _process_zip(zip_path, destination_dir, scan_hint=None): safe_zip_path = _rename_files([zip_path])[0] with zipfile.ZipFile(safe_zip_path, "r") as zip_handler: if scan_hint in _keep_original_data: - extracted_paths = _extract_zip(zip_handler, destination_dir) + extracted_paths = _extract_zip(zip_handler, destination_dir, charset) else: extracted_paths = _sanitize_zip_contents( - zip_handler, destination_dir) + zip_handler, destination_dir, charset) if extracted_paths is not None: all_paths = extracted_paths kept_zip = False @@ -333,16 +340,21 @@ def _process_zip(zip_path, destination_dir, scan_hint=None): return all_paths, kept_zip -def _sanitize_zip_contents(zip_handler, destination_dir): +def _sanitize_zip_contents(zip_handler, destination_dir, charset): clean_macosx_dir(zip_handler.namelist()) - result = _extract_zip(zip_handler, destination_dir) + result = _extract_zip(zip_handler, destination_dir, charset) return result -def _extract_zip(zip_handler, destination): +def _extract_zip(zip_handler, destination, charset): file_names = zip_handler.namelist() zip_handler.extractall(destination) - return [os.path.join(destination, p) for p in file_names] + paths = [] + for p in file_names: + _f = os.path.join(destination, p) + fixup_shp_columnnames(_f, charset) + paths.append(_f) + return paths def _probe_zip_for_sld(zip_handler, destination_dir): diff --git a/geonode/upload/forms.py b/geonode/upload/forms.py index dbd52d820f4..fcb8afa94ed 100644 --- a/geonode/upload/forms.py +++ b/geonode/upload/forms.py @@ -47,6 +47,7 @@ class LayerUploadForm(forms.Form): shx_file = forms.FileField(required=False) prj_file = forms.FileField(required=False) xml_file = forms.FileField(required=False) + charset = forms.CharField(required=False) if check_ogc_backend(geoserver.BACKEND_PACKAGE): sld_file = forms.FileField(required=False) diff --git a/geonode/upload/models.py b/geonode/upload/models.py index d9f096103ee..5fbd3738040 100644 --- a/geonode/upload/models.py +++ b/geonode/upload/models.py @@ -87,7 +87,7 @@ class Meta: def get_session(self): if self.session: - return pickle.loads(str(self.session)) + return pickle.loads(self.session.encode("utf-8", "replace")) def update_from_session(self, upload_session): self.state = upload_session.import_session.state @@ -97,10 +97,10 @@ def update_from_session(self, upload_session): self.session = None else: # Make sure we don't pickle UTF-8 chars - upload_session.user.first_name = u'{}'.format(upload_session.user.first_name).encode('ascii', 'ignore') - upload_session.user.last_name = u'{}'.format(upload_session.user.last_name).encode('ascii', 'ignore') + upload_session.user.first_name = u'{}'.format(upload_session.user.first_name).decode("utf-8", "replace") + upload_session.user.last_name = u'{}'.format(upload_session.user.last_name).decode("utf-8", "replace") unicode_session = pickle.dumps(upload_session) - self.session = unicode_session + self.session = unicode_session.decode("utf-8", "replace") if self.upload_dir is None: self.upload_dir = path.dirname(upload_session.base_file) self.name = upload_session.layer_title or upload_session.name diff --git a/geonode/upload/views.py b/geonode/upload/views.py index 12a97a16045..cfbbce5de4f 100644 --- a/geonode/upload/views.py +++ b/geonode/upload/views.py @@ -52,7 +52,7 @@ from django.shortcuts import get_object_or_404 from django.shortcuts import render from django.views.generic import CreateView, DeleteView -from geonode.utils import unzip_file +from geonode.utils import fixup_shp_columnnames from geonode.base.enumerations import CHARSETS from .forms import ( @@ -171,7 +171,8 @@ def save_step_view(req, session): scan_hint = get_scan_hint(form.cleaned_data["valid_extensions"]) spatial_files = scan_file( base_file, - scan_hint=scan_hint + scan_hint=scan_hint, + charset=form.cleaned_data["charset"] ) logger.info("spatial_files: {}".format(spatial_files)) import_session = save_step( @@ -191,14 +192,15 @@ def save_step_view(req, session): ) sld = None - if spatial_files[0].sld_files: sld = spatial_files[0].sld_files[0] if not os.path.isfile(os.path.join(tempdir, spatial_files[0].base_file)): tmp_files = [f for f in os.listdir(tempdir) if os.path.isfile(os.path.join(tempdir, f))] for f in tmp_files: if zipfile.is_zipfile(os.path.join(tempdir, f)): - unzip_file(os.path.join(tempdir, f), '.shp', tempdir=tempdir) + fixup_shp_columnnames(os.path.join(tempdir, f), + form.cleaned_data["charset"], + tempdir=tempdir) _log('provided sld is %s' % sld) # upload_type = get_upload_type(base_file) diff --git a/geonode/utils.py b/geonode/utils.py index 17af3c6a7a7..738657c7891 100755 --- a/geonode/utils.py +++ b/geonode/utils.py @@ -61,7 +61,7 @@ from django.core.serializers.json import DjangoJSONEncoder from django.utils import timezone -from geonode import geoserver, qgis_server # noqa +from geonode import geoserver, qgis_server, GeoNodeException # noqa try: import json @@ -965,14 +965,21 @@ def check_shp_columnnames(layer): """ Check if shapefile for a given layer has valid column names. If not, try to fix column names and warn the user """ - # TODO we may add in a better location this method inShapefile = '' for f in layer.upload_session.layerfile_set.all(): if os.path.splitext(f.file.name)[1] == '.shp': inShapefile = f.file.path + if inShapefile: + return fixup_shp_columnnames(inShapefile, layer.charset) + + +def fixup_shp_columnnames(inShapefile, charset, tempdir=None): + """ Try to fix column names and warn the user + """ - tempdir = tempfile.mkdtemp() + if not tempdir: + tempdir = tempfile.mkdtemp() if is_zipfile(inShapefile): inShapefile = unzip_file(inShapefile, '.shp', tempdir=tempdir) @@ -1004,46 +1011,49 @@ def check_shp_columnnames(layer): if a.match(field_name): list_col_original.append(field_name) - try: - for i in range(0, inLayerDefn.GetFieldCount()): - charset = layer.charset if layer.charset and 'undefined' not in layer.charset \ - else 'UTF-8' - field_name = unicode( - inLayerDefn.GetFieldDefn(i).GetName(), - charset) - - if not a.match(field_name): - # once the field_name contains Chinese, to use slugify_zh - has_ch = False - for ch in field_name: - if u'\u4e00' <= ch <= u'\u9fff': + + for i in range(0, inLayerDefn.GetFieldCount()): + charset = charset if charset and 'undefined' not in charset \ + else 'UTF-8' + + field_name = inLayerDefn.GetFieldDefn(i).GetName() + if not a.match(field_name): + # once the field_name contains Chinese, to use slugify_zh + has_ch = False + for ch in field_name: + try: + if u'\u4e00' <= ch.decode("utf-8", "replace") <= u'\u9fff': has_ch = True break - if has_ch: - new_field_name = slugify_zh(field_name, separator='_') - else: - new_field_name = custom_slugify(field_name) - if not b.match(new_field_name): - new_field_name = '_' + new_field_name - j = 0 - while new_field_name in list_col_original or new_field_name in list_col.values(): - if j == 0: - new_field_name += '_0' - if new_field_name.endswith('_' + str(j)): - j += 1 - new_field_name = new_field_name[:-2] + '_' + str(j) - list_col.update({field_name: new_field_name}) - except UnicodeDecodeError as e: - logger.error(str(e)) - return False, None, None + except UnicodeDecodeError: + has_ch = True + break + if has_ch: + new_field_name = slugify_zh(field_name, separator='_') + else: + new_field_name = custom_slugify(field_name) + if not b.match(new_field_name): + new_field_name = '_' + new_field_name + j = 0 + while new_field_name in list_col_original or new_field_name in list_col.values(): + if j == 0: + new_field_name += '_0' + if new_field_name.endswith('_' + str(j)): + j += 1 + new_field_name = new_field_name[:-2] + '_' + str(j) + list_col.update({field_name: new_field_name}) if len(list_col) == 0: return True, None, None else: - for key in list_col.keys(): - qry = u"ALTER TABLE {0} RENAME COLUMN \"{1}\" TO \"{2}\"".format( - inLayer.GetName(), key, list_col[key]) - inDataSource.ExecuteSQL(qry.encode(layer.charset)) + try: + for key in list_col.keys(): + qry = u"ALTER TABLE {} RENAME COLUMN \"".format(inLayer.GetName()) + qry = qry + key.decode(charset) + u"\" TO \"{}\"".format(list_col[key]) + inDataSource.ExecuteSQL(qry.encode(charset)) + except UnicodeDecodeError: + raise GeoNodeException( + "Could not decode SHAPEFILE attributes by using the specified charset '{}'.".format(charset)) return True, None, list_col