Use xpath as key for valueLabels dict

onadata/libs/tests/utils/test_export_tools.py

@@ -1,7 +1,9 @@
 import os
 from datetime import date, datetime
+from django.conf import settings
 from django.core.files.storage import default_storage
 from django.contrib.sites.models import Site
+from pyxform.builder import create_survey_from_xls
 from pyxform.tests_v1.pyxform_test_case import PyxformTestCase
 from django.core.files.temp import NamedTemporaryFile
 
@@ -22,6 +24,11 @@
 from onadata.apps.api import tests as api_tests
 
 
+def _logger_fixture_path(*args):
+    return os.path.join(settings.PROJECT_ROOT, 'libs', 'tests', 'fixtures',
+                        *args)
+
+
 class TestExportTools(PyxformTestCase, TestBase):
 
     def _create_old_export(self, xform, export_type, options):
@@ -378,3 +385,18 @@ def test_sav_duplicate_columns(self):
             sav_file = NamedTemporaryFile(suffix=".sav")
             # No exception is raised
             SavWriter(sav_file.name, **sav_options)
+
+    def test_sav_special_char_columns(self):
+        survey = create_survey_from_xls(_logger_fixture_path(
+            'grains/grains.xls'))
+        export_builder = ExportBuilder()
+        export_builder.TRUNCATE_GROUP_TITLE = True
+        export_builder.set_survey(survey)
+        export_builder.INCLUDE_LABELS = True
+        export_builder.set_survey(survey)
+
+        for sec in export_builder.sections:
+            sav_options = export_builder._get_sav_options(sec['elements'])
+            sav_file = NamedTemporaryFile(suffix=".sav")
+            # No exception is raised
+            SavWriter(sav_file.name, **sav_options)

onadata/libs/utils/export_builder.py

@@ -711,8 +711,8 @@ def get_default_language(self, languages):
 
         return language
 
-    def _get_sav_value_labels(self):
-        """GET/SET SPSS `VALUE LABELS`. It takes the dictionay of the form
+    def _get_sav_value_labels(self, xpath_var_names=None):
+        """GET/SET SPSS `VALUE LABELS`. It takes the dictionary of the form
         `{varName: {value: valueLabel}}`:
 
         .. code-block: python
@@ -727,6 +727,8 @@ def _get_sav_value_labels(self):
             self._sav_value_labels = {}
 
             for q in choice_questions:
+                var_name = xpath_var_names.get(q.get_abbreviated_xpath()) if \
+                    xpath_var_names else q['name']
                 choices = q.to_json_dict().get('children')
                 if choices is None:
                     choices = self.survey.get('choices')
@@ -737,10 +739,25 @@ def _get_sav_value_labels(self):
                     name = choice['name'].strip()
                     label = self.get_choice_label_from_dict(choice['label'])
                     _value_labels[name] = label.strip()
-                self._sav_value_labels[q['name']] = _value_labels
+                self._sav_value_labels[var_name or q['name']] = _value_labels
 
         return self._sav_value_labels
 
+    def _get_var_name(self, title, var_names):
+        """GET valid SPSS varName.
+                @param title - survey element title/name
+                @param var_names - list of existing var_names
+
+                @return valid varName and list of var_names with new var name
+                 appended
+
+                """
+        var_name = title.replace('/', '.').replace('-', '_')
+        var_name = self._check_sav_column(var_name, var_names)
+        var_name = '@' + var_name if var_name.startswith('_') else var_name
+        var_names.append(var_name)
+        return var_name, var_names
+
     def _get_sav_options(self, elements):
         """GET/SET SPSS options.
         @param elements - a list of survey elements
@@ -757,27 +774,31 @@ def _get_sav_options(self, elements):
                 'ioUtf8': True
             }
         """
-        all_value_labels = self._get_sav_value_labels()
         _var_types = {}
         value_labels = {}
         var_labels = {}
         var_names = []
-
-        fields_and_labels = [
-            (element['title'], element['label'], element['xpath'])
-            for element in elements
-        ] + zip(self.EXTRA_FIELDS, self.EXTRA_FIELDS, self.EXTRA_FIELDS)
-
-        for field, label, xpath in fields_and_labels:
-            var_name = field.replace('/', '.')
-            var_name = self._check_sav_column(var_name, var_names)
-            var_name = '@' + var_name \
-                if var_name.startswith('_') else var_name
+        fields_and_labels = []
+        elements += [{'title': f, "label": f, "xpath": f, 'type': f}
+                     for f in self.EXTRA_FIELDS]
+
+        for element in elements:
+            title = element['title']
+            _var_name, _var_names = self._get_var_name(title, var_names)
+            var_names = _var_names
+            fields_and_labels.append((element['title'], element['label'],
+                                      element['xpath'], _var_name))
+
+        xpath_var_names = dict([(xpath, var_name)
+                                for field, label, xpath, var_name
+                                in fields_and_labels])
+        all_value_labels = self._get_sav_value_labels(xpath_var_names)
+
+        for field, label, xpath, var_name in fields_and_labels:
             var_labels[var_name] = label
-            var_names.append(var_name)
             _var_types[xpath] = var_name
-            if field in all_value_labels:
-                value_labels[field] = all_value_labels.get(field)
+            if var_name in all_value_labels:
+                value_labels[var_name] = all_value_labels.get(var_name)
 
         var_types = dict(
             [(_var_types[element['xpath']],
@@ -826,7 +847,8 @@ def write_row(row, csv_writer, fields):
         for section in self.sections:
             sav_options = self._get_sav_options(section['elements'])
             sav_file = NamedTemporaryFile(suffix=".sav")
-            sav_writer = SavWriter(sav_file.name, **sav_options)
+            sav_writer = SavWriter(sav_file.name, ioLocale="en_US.UTF-8",
+                                   **sav_options)
             sav_defs[section['name']] = {
                 'sav_file': sav_file, 'sav_writer': sav_writer}
 
@@ -856,7 +878,7 @@ def write_row(row, csv_writer, fields):
                 sav_def = sav_defs[section_name]
                 fields = [
                     element['xpath'] for element in
-                    section['elements']] + self.EXTRA_FIELDS
+                    section['elements']]
                 sav_writer = sav_def['sav_writer']
                 row = output.get(section_name, None)
                 if type(row) == dict: