Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

include export options in outdated export function #850

Merged
merged 1 commit into from
Nov 28, 2016
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 55 additions & 1 deletion onadata/apps/api/tests/viewsets/test_xform_viewset.py
Original file line number Diff line number Diff line change
Expand Up @@ -3298,7 +3298,7 @@ def test_csv_export_with_and_without_removed_group_name(self):
with open(test_file_path, 'r') as test_file:
self.assertEqual(content, test_file.read())

def test__csv_export__no_new_generated(self):
def test_csv_export_no_new_generated(self):
with HTTMock(enketo_mock):
self._publish_xls_form_to_project()
survey = self.surveys[0]
Expand Down Expand Up @@ -4103,3 +4103,57 @@ def test_csv_export_with_meta_perms(self):
request = self.factory.get('/', **alices_extra)
response = view(request, pk=self.xform.pk, format='csv')
self.assertEqual(response.status_code, 403)

def test_csv_export_cache(self):
with HTTMock(enketo_mock):
self._publish_xls_form_to_project()
self._make_submissions()

count = Export.objects.all().count()

view = XFormViewSet.as_view({
'get': 'retrieve'
})

data = {
"export_type": "csv",
"win_excel_utf8": False
}

request = self.factory.get('/', data=data, **self.extra)
response = view(request, pk=self.xform.pk, format='csv')
self.assertEqual(response.status_code, 200)

# should generate new
self.assertEquals(count + 1, Export.objects.all().count())

survey = self.surveys[0]
self._make_submission(
os.path.join(
settings.PROJECT_ROOT, 'apps',
'main', 'tests', 'fixtures', 'transportation',
'instances', survey, survey + '.xml'))

data = {
"export_type": "csv",
"win_excel_utf8": True
}

request = self.factory.get('/', data=data, **self.extra)
response = view(request, pk=self.xform.pk, format='csv')
self.assertEqual(response.status_code, 200)

# changed options, should generate new
self.assertEquals(count + 2, Export.objects.all().count())

data = {
"export_type": "csv",
"win_excel_utf8": False
}

request = self.factory.get('/', data=data, **self.extra)
response = view(request, pk=self.xform.pk, format='csv')
self.assertEqual(response.status_code, 200)

# reused options, should generate new with new submission
self.assertEquals(count + 3, Export.objects.all().count())
34 changes: 31 additions & 3 deletions onadata/apps/viewer/models/export.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import os
import hashlib
from tempfile import NamedTemporaryFile

from django.core.files.storage import get_storage_class
Expand All @@ -11,6 +12,8 @@
from onadata.libs.utils.common_tags import OSM
from onadata.libs.utils import async_status

EXPORT_QUERY_KEY = 'query'


def export_delete_callback(sender, **kwargs):
export = kwargs['instance']
Expand All @@ -19,6 +22,30 @@ def export_delete_callback(sender, **kwargs):
storage.delete(export.filepath)


def md5hash(string):
return hashlib.md5(string).hexdigest()


def get_export_options_query_kwargs(options):
"""
Get dict with options JSONField lookups for export options field
"""
options_kwargs = {}
for field in Export.EXPORT_OPTION_FIELDS:
if field in options:
field_value = options.get(field)

if field == EXPORT_QUERY_KEY:
query_str = str(format(field_value))

field_value = md5hash(query_str)

key = 'options__{}'.format(field)
options_kwargs[key] = field_value

return options_kwargs


class Export(models.Model):
"""
Class representing a data export from an XForm
Expand Down Expand Up @@ -196,13 +223,14 @@ def full_filepath(self):
return None

@classmethod
def exports_outdated(cls, xform, export_type):
def exports_outdated(cls, xform, export_type, options={}):
# get newest export for xform
try:
export_options = get_export_options_query_kwargs(options)
latest_export = Export.objects.filter(
xform=xform, export_type=export_type,
internal_status__in=[Export.SUCCESSFUL, Export.PENDING])\
.latest('created_on')
internal_status__in=[Export.SUCCESSFUL, Export.PENDING],
**export_options).latest('created_on')
except cls.DoesNotExist:
return True
else:
Expand Down
25 changes: 3 additions & 22 deletions onadata/libs/utils/export_tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@
from onadata.apps.logger.models import XForm
from onadata.apps.logger.models.data_view import DataView
from onadata.apps.main.models.meta_data import MetaData
from onadata.apps.viewer.models.export import Export
from onadata.apps.viewer.models.export import Export,\
get_export_options_query_kwargs
from onadata.apps.viewer.models.parsed_instance import query_data
from onadata.libs.exceptions import J2XException, NoRecordsFoundError
from onadata.libs.utils.viewer_tools import create_attachments_zipfile,\
Expand Down Expand Up @@ -59,26 +60,6 @@ def get_export_options(options):
return export_options


def get_export_options_query_kwargs(options):
"""
Get dict with options JSONField lookups for export options field
"""
options_kwargs = {}
for field in Export.EXPORT_OPTION_FIELDS:
if field in options:
field_value = options.get(field)

if field == EXPORT_QUERY_KEY:
query_str = str(format(field_value))

field_value = md5hash(query_str)

key = 'options__{}'.format(field)
options_kwargs[key] = field_value

return options_kwargs


def get_or_create_export(export_id, xform, export_type, options):
if export_id:
try:
Expand Down Expand Up @@ -271,7 +252,7 @@ def should_create_new_export(xform,
)

if export_query.count() == 0 or\
Export.exports_outdated(xform, export_type):
Export.exports_outdated(xform, export_type, options=options):
return True

return False
Expand Down