diff --git a/onadata/apps/api/tests/viewsets/test_xform_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_viewset.py index 04f268f137..cc934444ea 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_viewset.py @@ -3298,7 +3298,7 @@ def test_csv_export_with_and_without_removed_group_name(self): with open(test_file_path, 'r') as test_file: self.assertEqual(content, test_file.read()) - def test__csv_export__no_new_generated(self): + def test_csv_export_no_new_generated(self): with HTTMock(enketo_mock): self._publish_xls_form_to_project() survey = self.surveys[0] @@ -4103,3 +4103,57 @@ def test_csv_export_with_meta_perms(self): request = self.factory.get('/', **alices_extra) response = view(request, pk=self.xform.pk, format='csv') self.assertEqual(response.status_code, 403) + + def test_csv_export_cache(self): + with HTTMock(enketo_mock): + self._publish_xls_form_to_project() + self._make_submissions() + + count = Export.objects.all().count() + + view = XFormViewSet.as_view({ + 'get': 'retrieve' + }) + + data = { + "export_type": "csv", + "win_excel_utf8": False + } + + request = self.factory.get('/', data=data, **self.extra) + response = view(request, pk=self.xform.pk, format='csv') + self.assertEqual(response.status_code, 200) + + # should generate new + self.assertEquals(count + 1, Export.objects.all().count()) + + survey = self.surveys[0] + self._make_submission( + os.path.join( + settings.PROJECT_ROOT, 'apps', + 'main', 'tests', 'fixtures', 'transportation', + 'instances', survey, survey + '.xml')) + + data = { + "export_type": "csv", + "win_excel_utf8": True + } + + request = self.factory.get('/', data=data, **self.extra) + response = view(request, pk=self.xform.pk, format='csv') + self.assertEqual(response.status_code, 200) + + # changed options, should generate new + self.assertEquals(count + 2, Export.objects.all().count()) + + data = { + "export_type": "csv", + "win_excel_utf8": False + } + + request = self.factory.get('/', data=data, **self.extra) + response = view(request, pk=self.xform.pk, format='csv') + self.assertEqual(response.status_code, 200) + + # reused options, should generate new with new submission + self.assertEquals(count + 3, Export.objects.all().count()) diff --git a/onadata/apps/viewer/models/export.py b/onadata/apps/viewer/models/export.py index 70ff51ddd1..791f14491a 100644 --- a/onadata/apps/viewer/models/export.py +++ b/onadata/apps/viewer/models/export.py @@ -1,4 +1,5 @@ import os +import hashlib from tempfile import NamedTemporaryFile from django.core.files.storage import get_storage_class @@ -11,6 +12,8 @@ from onadata.libs.utils.common_tags import OSM from onadata.libs.utils import async_status +EXPORT_QUERY_KEY = 'query' + def export_delete_callback(sender, **kwargs): export = kwargs['instance'] @@ -19,6 +22,30 @@ def export_delete_callback(sender, **kwargs): storage.delete(export.filepath) +def md5hash(string): + return hashlib.md5(string).hexdigest() + + +def get_export_options_query_kwargs(options): + """ + Get dict with options JSONField lookups for export options field + """ + options_kwargs = {} + for field in Export.EXPORT_OPTION_FIELDS: + if field in options: + field_value = options.get(field) + + if field == EXPORT_QUERY_KEY: + query_str = str(format(field_value)) + + field_value = md5hash(query_str) + + key = 'options__{}'.format(field) + options_kwargs[key] = field_value + + return options_kwargs + + class Export(models.Model): """ Class representing a data export from an XForm @@ -196,13 +223,14 @@ def full_filepath(self): return None @classmethod - def exports_outdated(cls, xform, export_type): + def exports_outdated(cls, xform, export_type, options={}): # get newest export for xform try: + export_options = get_export_options_query_kwargs(options) latest_export = Export.objects.filter( xform=xform, export_type=export_type, - internal_status__in=[Export.SUCCESSFUL, Export.PENDING])\ - .latest('created_on') + internal_status__in=[Export.SUCCESSFUL, Export.PENDING], + **export_options).latest('created_on') except cls.DoesNotExist: return True else: diff --git a/onadata/libs/utils/export_tools.py b/onadata/libs/utils/export_tools.py index a8e0d70377..030e32ab53 100644 --- a/onadata/libs/utils/export_tools.py +++ b/onadata/libs/utils/export_tools.py @@ -20,7 +20,8 @@ from onadata.apps.logger.models import XForm from onadata.apps.logger.models.data_view import DataView from onadata.apps.main.models.meta_data import MetaData -from onadata.apps.viewer.models.export import Export +from onadata.apps.viewer.models.export import Export,\ + get_export_options_query_kwargs from onadata.apps.viewer.models.parsed_instance import query_data from onadata.libs.exceptions import J2XException, NoRecordsFoundError from onadata.libs.utils.viewer_tools import create_attachments_zipfile,\ @@ -59,26 +60,6 @@ def get_export_options(options): return export_options -def get_export_options_query_kwargs(options): - """ - Get dict with options JSONField lookups for export options field - """ - options_kwargs = {} - for field in Export.EXPORT_OPTION_FIELDS: - if field in options: - field_value = options.get(field) - - if field == EXPORT_QUERY_KEY: - query_str = str(format(field_value)) - - field_value = md5hash(query_str) - - key = 'options__{}'.format(field) - options_kwargs[key] = field_value - - return options_kwargs - - def get_or_create_export(export_id, xform, export_type, options): if export_id: try: @@ -271,7 +252,7 @@ def should_create_new_export(xform, ) if export_query.count() == 0 or\ - Export.exports_outdated(xform, export_type): + Export.exports_outdated(xform, export_type, options=options): return True return False