Keep track of whether all attachments have been submitted

Fixes: #1049

Add new fields to the model instance to keep track of:

- How many attachments, if any are expected in a submission
- How many attachments have been actually received
- Whether all attachments have been received

onadata/apps/logger/migrations/0034_auto_20170814_0432.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.1 on 2017-08-14 08:32
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('logger', '0033_auto_20170705_0159'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='instance',
+            name='media_all_received',
+            field=models.BooleanField(
+                default=True,
+                verbose_name='Received All Media Attachemts'
+            ),
+        ),
+        migrations.AddField(
+            model_name='instance',
+            name='media_count',
+            field=models.PositiveIntegerField(
+                default=0,
+                verbose_name='Received Media Attachments'
+            ),
+        ),
+        migrations.AddField(
+            model_name='instance',
+            name='total_media',
+            field=models.PositiveIntegerField(
+                default=0,
+                verbose_name='Total Media Attachments'
+            ),
+        ),
+    ]

onadata/apps/logger/models/instance.py

@@ -25,7 +25,8 @@
 from onadata.libs.utils.common_tags import ATTACHMENTS, BAMBOO_DATASET_ID,\
     DELETEDAT, EDITED, GEOLOCATION, ID, MONGO_STRFTIME, NOTES, \
     SUBMISSION_TIME, TAGS, UUID, XFORM_ID_STRING, SUBMITTED_BY, VERSION, \
-    STATUS, DURATION, START, END, LAST_EDITED, XFORM_ID
+    STATUS, DURATION, START, END, LAST_EDITED, MEDIA_ALL_RECEIVED, \
+    TOTAL_MEDIA, MEDIA_COUNT, XFORM_ID
 from onadata.libs.utils.model_tools import set_uuid
 from onadata.libs.data.query import get_numeric_fields
 from onadata.libs.utils.cache_tools import safe_delete
@@ -301,7 +302,6 @@ def get_full_dict(self, load_existing=True):
         doc = self.json or {} if load_existing else {}
         # Get latest dict
         doc = self.get_dict()
-
         if self.id:
             doc.update({
                 UUID: self.uuid,
@@ -331,6 +331,10 @@ def get_full_dict(self, load_existing=True):
 
             doc[SUBMISSION_TIME] = self.date_created.strftime(MONGO_STRFTIME)
 
+            doc[TOTAL_MEDIA] = self.total_media
+            doc[MEDIA_COUNT] = self.media_count
+            doc[MEDIA_ALL_RECEIVED] = self.media_all_received
+
             edited = False
             if hasattr(self, 'last_edited'):
                 edited = self.last_edited is not None
@@ -339,7 +343,6 @@ def get_full_dict(self, load_existing=True):
             edited and doc.update({
                 LAST_EDITED: convert_to_serializable_date(self.last_edited)
             })
-
         return doc
 
     def _set_parser(self):
@@ -423,6 +426,15 @@ class Instance(models.Model, InstanceBaseClass):
     # store a geographic objects associated with this instance
     geom = models.GeometryCollectionField(null=True)
 
+    # Keep track of whether all media attachments have been received
+    media_all_received = models.BooleanField(
+        _("Received All Media Attachemts"),
+        default=True)
+    total_media = models.PositiveIntegerField(_("Total Media Attachments"),
+                                              default=0)
+    media_count = models.PositiveIntegerField(_("Received Media Attachments"),
+                                              default=0)
+
     tags = TaggableManager()
 
     class Meta:

onadata/apps/logger/tests/models/test_instance.py

@@ -12,7 +12,7 @@
 from onadata.apps.viewer.models.parsed_instance import (
     ParsedInstance, query_data)
 from onadata.libs.utils.common_tags import MONGO_STRFTIME, SUBMISSION_TIME,\
-    XFORM_ID_STRING, SUBMITTED_BY
+                                           XFORM_ID_STRING, SUBMITTED_BY
 
 
 class TestInstance(TestBase):

onadata/libs/tests/utils/test_logger_tools.py

@@ -1,10 +1,22 @@
 import re
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+from pyxform.tests_v1.pyxform_test_case import PyxformTestCase
+from django.conf import settings
 
 from onadata.libs.utils.logger_tools import generate_content_disposition_header
+from onadata.libs.utils.logger_tools import create_instance
+from onadata.apps.logger.import_tools import django_file
 from onadata.apps.main.tests.test_base import TestBase
+from onadata.apps.logger.models import Instance
+from onadata.libs.utils.common_tags import MEDIA_ALL_RECEIVED, TOTAL_MEDIA,\
+                                           MEDIA_COUNT
 
 
-class TestLoggerTools(TestBase):
+class TestLoggerTools(PyxformTestCase, TestBase):
     def test_generate_content_disposition_header(self):
         file_name = "export"
         extension = "ext"
@@ -24,6 +36,68 @@ def test_generate_content_disposition_header(self):
 
         return_value_with_name_and_false_show_date = \
             generate_content_disposition_header(file_name, extension, False)
-        print return_value_with_name_and_false_show_date
         self.assertTrue(re.search(file_name_pattern,
                                   return_value_with_name_and_false_show_date))
+
+    def test_attachment_tracking(self):
+        """
+        Test that when a submission with many attachments is made,
+        we keep track of the total number of attachments expected
+        and if we have received all of them
+        """
+        md = """
+        | survey |       |        |       |
+        |        | type  | name   | label |
+        |        | image | image1 | Photo |
+        |        | image | image2 | Photo |
+        """
+        self._create_user_and_login()
+        self.xform = self._publish_md(md, self.user)
+
+        xml_string = """
+        <data id="{}">
+            <image1>1300221157303.jpg</image1>
+            <image2>1300375832136.jpg</image2>
+        </data>
+        """.format(self.xform.id_string)
+        file_path = "{}/apps/logger/tests/Health_2011_03_13."\
+                    "xml_2011-03-15_20-30-28/1300221157303"\
+                    ".jpg".format(settings.PROJECT_ROOT)
+        media_file = django_file(path=file_path,
+                                 field_name="image1",
+                                 content_type="image/jpeg")
+        instance = create_instance(self.user.username,
+                                   StringIO(xml_string.strip()),
+                                   media_files=[media_file])
+        self.assertFalse(instance.json[MEDIA_ALL_RECEIVED])
+        self.assertEquals(instance.json[TOTAL_MEDIA], 2)
+        self.assertEquals(instance.json[MEDIA_COUNT], 1)
+        self.assertEquals(instance.json[TOTAL_MEDIA], instance.total_media)
+        self.assertEquals(instance.json[MEDIA_COUNT], instance.media_count)
+        self.assertEquals(instance.json[MEDIA_ALL_RECEIVED],
+                          instance.media_all_received)
+        xml2_string = """
+        <data id="{}">
+            <meta>
+                <instanceID>uuid:{}</instanceID>
+            </meta>
+            <image1>1300221157303.jpg</image1>
+            <image2>1300375832136.jpg</image2>
+        </data>
+        """.format(self.xform.id_string, instance.uuid)
+        file2_path = "{}/apps/logger/tests/Water_2011_03_17_2011-03-17_16-29"\
+                     "-59/1300375832136.jpg".format(settings.PROJECT_ROOT)
+        media2_file = django_file(path=file2_path,
+                                  field_name="image2",
+                                  content_type="image/jpeg")
+        create_instance(self.user.username,
+                        StringIO(xml2_string.strip()),
+                        media_files=[media2_file])
+        instance2 = Instance.objects.get(pk=instance.pk)
+        self.assertTrue(instance2.json[MEDIA_ALL_RECEIVED])
+        self.assertEquals(instance2.json[TOTAL_MEDIA], 2)
+        self.assertEquals(instance2.json[MEDIA_COUNT], 2)
+        self.assertEquals(instance2.json[TOTAL_MEDIA], instance2.total_media)
+        self.assertEquals(instance2.json[MEDIA_COUNT], instance2.media_count)
+        self.assertEquals(instance2.json[MEDIA_ALL_RECEIVED],
+                          instance2.media_all_received)

onadata/libs/utils/common_tags.py

@@ -39,6 +39,12 @@
 VERSION = u"_version"
 DURATION = u"_duration"
 
+# fields to deal with media attachments and keep track of how many
+# have been received
+MEDIA_ALL_RECEIVED = u"_media_all_received"
+TOTAL_MEDIA = u"_total_media"
+MEDIA_COUNT = u"_media_count"
+
 INSTANCE_ID = u"instanceID"
 META_INSTANCE_ID = u"meta/instanceID"
 INDEX = u"_index"

onadata/libs/utils/logger_tools.py

@@ -88,7 +88,6 @@ def _get_instance(xml, new_uuid, submitted_by, status, xform):
         # new submission
         instance = Instance.objects.create(
             xml=xml, user=submitted_by, status=status, xform=xform)
-
     return instance
 
 
@@ -197,6 +196,15 @@ def save_attachments(xform, instance, media_files):
             mimetype=content_type,
             extension=extension)
 
+    instance.total_media = len([m for m in instance.get_dict().keys() if m in
+                                xform.get_media_survey_xpaths()])
+    instance.media_count = instance.attachments.count()
+    instance.media_all_received = instance.media_count == \
+        instance.total_media
+    instance.save(update_fields=['total_media',
+                                 'media_count',
+                                 'media_all_received'])
+
 
 def save_submission(xform, xml, media_files, new_uuid, submitted_by, status,
                     date_created_override):