Merge pull request #1884 from onaio/v2.4.0-rc

Tag release v2.4.0

CHANGES.rst

@@ -3,6 +3,25 @@ Changelog for Onadata
 
 ``* represents releases that introduce new migrations``
 
+v2.4.0(2020-09-01)
+------------------
+
+- Initial support for tracking submissions with Segment
+  `PR #1872 <https://github.com/onaio/onadata/pull/1872>`_
+  [@DavisRayM]
+- Add caching to the organization profile viewset
+  `PR #1876 <https://github.com/onaio/onadata/pull/1876>`_
+  [@FrankApiyo]
+- Include support for repeat groups in the Tableau-Onadata integration
+  `PR #1845 <https://github.com/onaio/onadata/pull/1845>`_
+  [@WinnyTroy]
+- Enketo intergration updates
+  `PR #1857 <https://github.com/onaio/onadata/pull/1845>`_
+  [@WinnyTroy]
+- Unpack GPS data into separate columns for altitude, precision, latitude and longitude
+  `PR #1880 <https://github.com/onaio/onadata/pull/1880>`_
+  [@WinnyTroy]
+
 v2.3.8(2020-08-25)
 ------------------
 

onadata/__init__.py

@@ -6,7 +6,7 @@
 """
 from __future__ import absolute_import, unicode_literals
 
-__version__ = "2.3.8"
+__version__ = "2.4.0"
 
 
 # This will make sure the app is always imported when

onadata/apps/api/viewsets/open_data_viewset.py

@@ -32,6 +32,7 @@
     NOTES,
     GEOLOCATION,
     MULTIPLE_SELECT_TYPE,
+    REPEAT_SELECT_TYPE,
     NA_REP)
 
 BaseViewset = get_baseviewset_class()
@@ -73,6 +74,15 @@ def get_updated_data_dict(key, value, data_dict):
             for choice in choices:
                 xpaths = f'{key}/{choice}'
                 data_dict[xpaths] = choice
+        elif isinstance(value, list):
+            try:
+                for item in value:
+                    for (nested_key, nested_val) in item.items():
+                        xpath = get_xpath(key, nested_key)
+                        data_dict[xpath] = nested_val
+            except AttributeError:
+                data_dict[key] = value
+
         return data_dict
 
     def get_ordered_repeat_value(key, item, index):
@@ -98,6 +108,9 @@ def get_ordered_repeat_value(key, item, index):
                     if qstn_type == MULTIPLE_SELECT_TYPE:
                         data = get_updated_data_dict(
                             xpaths, nested_val, data)
+                    elif qstn_type == REPEAT_SELECT_TYPE:
+                        data = get_updated_data_dict(
+                            xpaths, nested_val, data)
                     else:
                         data[xpaths] = nested_val
         return data

onadata/libs/tests/utils/test_analytics.py

@@ -14,7 +14,7 @@
     TestAbstractViewSet
 from onadata.apps.api.viewsets.xform_submission_viewset import \
     XFormSubmissionViewSet
-from onadata.libs.utils.analytics import get_user_id
+from onadata.libs.utils.analytics import get_user_id, sanitize_metric_values
 
 
 class TestAnalytics(TestAbstractViewSet):
@@ -48,6 +48,34 @@ def test_track(self):
             {'value': 1},
             {'source': 'test-server'})
 
+    def test_sanitize_metric_values(self):
+        """Test sanitize_metric_values()"""
+        expected_output = {
+            'action_from': 'XML_Submissions',
+            'event_by': 'bob',
+            'ip': '18.203.134.101',
+            'path': '/enketo/1814/submission',
+            'source': 'onadata-ona-stage',
+            'url': 'https://stage-api.ona.io/enketo/1814/submission',
+            'userId': 1,
+            'xform_id': 1}
+        context = {
+            'action_from': 'XML Submissions',
+            'event_by': 'bob',
+            'ip': '18.203.134.101',
+            'library': {
+                'name': 'analytics-python',
+                'version': '1.2.9'
+                },
+            'organization': '',
+            'path': '/enketo/1814/submission',
+            'source': 'onadata-ona-stage',
+            'url': 'https://stage-api.ona.io/enketo/1814/submission',
+            'userId': 1,
+            'xform_id': 1}
+
+        self.assertEqual(sanitize_metric_values(context), expected_output)
+
     @override_settings(
             SEGMENT_WRITE_KEY='123', HOSTNAME='test-server',
             APPOPTICS_API_TOKEN='123')
@@ -112,7 +140,6 @@ def test_submission_tracking(self):
                 'event_by': 'anonymous',
                 'action_from': 'XML Submissions',
                 'xform_id': self.xform.pk,
-                'userAgent': '',
                 'path': f'/{username}/submission',
                 'url': f'http://testserver/{username}/submission',
                 'ip': '127.0.0.1',
@@ -128,7 +155,6 @@ def test_submission_tracking(self):
                 'organization': 'Bob_Inc.',
                 'action_from': 'XML_Submissions',
                 'xform_id': self.xform.pk,
-                'userAgent': '',
                 'path': f'/{username}/submission',
                 'url': f'http://testserver/{username}/submission',
                 'ip': '127.0.0.1',

onadata/libs/utils/analytics.py

@@ -1,11 +1,12 @@
 # -*- coding: utf-8 -*-
 # Analytics package for tracking and measuring with services like Segment.
 # Heavily borrowed from RapidPro's temba.utils.analytics
+import sys
 
 import analytics as segment_analytics
 from typing import Dict, List, Optional, Any
 import appoptics_metrics
-from appoptics_metrics import sanitize_metric_name
+from appoptics_metrics import sanitize_metric_name, exceptions
 
 from django.conf import settings
 from django.contrib.auth.models import User
@@ -14,6 +15,7 @@
 from onadata.apps.logger.models import Instance
 from onadata.libs.utils.common_tags import (
     INSTANCE_CREATE_EVENT, INSTANCE_UPDATE_EVENT)
+from onadata.libs.utils.common_tools import report_exception
 
 
 appoptics_api = None
@@ -41,16 +43,19 @@ def sanitize_metric_values(data: Dict[str, Any]) -> Dict[str, Any]:
     for AppOptics
     """
     sanitized_data = data.copy()
-    for key, value in sanitized_data.items():
-        if isinstance(value, str):
-            new_value = value
-            if ' ' in new_value:
-                new_value = new_value.replace(' ', '_')
+    for key, value in data.items():
+        new_value = value
+        if new_value and not isinstance(new_value, dict):
+            if isinstance(new_value, str):
+                if ' ' in new_value:
+                    new_value = new_value.replace(' ', '_')
 
-            if '(' in value or ')' in new_value:
-                new_value = new_value.replace(')', "").replace('(', "")
+                if '(' in value or ')' in new_value:
+                    new_value = new_value.replace(')', "").replace('(', "")
 
             sanitized_data.update({key: new_value})
+        else:
+            sanitized_data.pop(key)
     return sanitized_data
 
 
@@ -196,7 +201,6 @@ def track(user, event_name, properties=None, context=None, request=None):
             context['xform_id'] = properties['xform_id']
 
         if request:
-            context['userAgent'] = request.META.get('HTTP_USER_AGENT', '')
             context['path'] = request.path
             context['url'] = request.build_absolute_uri()
             context['ip'] = request.META.get('REMOTE_ADDR', '')
@@ -207,5 +211,10 @@ def track(user, event_name, properties=None, context=None, request=None):
 
         if appoptics_api:
             tags = sanitize_metric_values(context)
-            appoptics_api.submit_measurement(
-                event_name, properties['value'], tags=tags)
+            try:
+                appoptics_api.submit_measurement(
+                    event_name,
+                    properties['value'],
+                    tags=tags)
+            except exceptions.BadRequest as e:
+                report_exception("Bad AppOptics Request", e, sys.exc_info())

onadata/libs/utils/common_tags.py

@@ -85,6 +85,7 @@
 OSM = 'osm'
 SELECT_BIND_TYPE = 'string'
 MULTIPLE_SELECT_TYPE = 'select all that apply'
+REPEAT_SELECT_TYPE = 'repeat'
 GROUPNAME_REMOVED_FLAG = 'group-name-removed'
 DATAVIEW_EXPORT = U'dataview'
 OWNER_TEAM_NAME = "Owners"