Merge pull request #1214 from onaio/attachments-count-fix

Attachments count fix

onadata/apps/api/tests/fixtures/Transportation Form.xml

@@ -41,6 +41,7 @@
               </taxi>
             </loop_over_transport_types_frequency>
           </transport>
+          <image1/>
           <meta>
             <instanceID/>
           </meta>
@@ -57,6 +58,7 @@
       <bind nodeset="/transportation/transport/loop_over_transport_types_frequency/lorry/frequency_to_referral_facility" relevant="selected( /transportation/transport/available_transportation_types_to_referral_facility , 'lorry')" type="select1"/>
       <bind nodeset="/transportation/transport/loop_over_transport_types_frequency/motorbike/frequency_to_referral_facility" relevant="selected( /transportation/transport/available_transportation_types_to_referral_facility , 'motorbike')" type="select1"/>
       <bind nodeset="/transportation/transport/loop_over_transport_types_frequency/taxi/frequency_to_referral_facility" relevant="selected( /transportation/transport/available_transportation_types_to_referral_facility , 'taxi')" type="select1"/>
+      <bind nodeset="/transportation/image1" type="binary"/>
       <bind calculate="concat('uuid:', uuid())" nodeset="/transportation/meta/instanceID" readonly="true()" type="string"/>
       <bind calculate="'%(form_uuid)s'" nodeset="/transportation/formhub/uuid" type="string"/>
     </model>
@@ -311,5 +313,8 @@
         </group>
       </group>
     </group>
+    <upload mediatype="image/*" ref="/transportation/image1">
+      <label>Photo</label>
+    </upload>
   </h:body>
 </h:html>

onadata/apps/api/tests/fixtures/osm/osm.csv

@@ -1,2 +1,2 @@
 photo,osm_road,osm_building,fav_color,form_completed,meta/instanceID,_uuid,_submission_time,_tags,_notes,_version,_duration,_submitted_by,_total_media,_media_count,_media_all_received,osm_road:ctr:lat,osm_road:ctr:lon,osm_road:highway,osm_road:lanes,osm_road:name,osm_road:way:id,osm_building:building,osm_building:building:levels,osm_building:ctr:lat,osm_building:ctr:lon,osm_building:name,osm_building:way:id
-1424308569120.jpg,OSMWay234134797.osm,OSMWay34298972.osm,red,2015-02-19T04:18:21.427+03,uuid:d3ef929e-e3e7-456c-9f27-7679c0074f4f,d3ef929e-e3e7-456c-9f27-7679c0074f4f,2013-02-18T15:54:01,,,201511091147,,bob,1,2,False,23.708174238006087,90.40946505581161,tertiary,2,Patuatuli Road,234134797,yes,4,23.707316084046038,90.40849938337506,kol,34298972
+1424308569120.jpg,OSMWay234134797.osm,OSMWay34298972.osm,red,2015-02-19T04:18:21.427+03,uuid:d3ef929e-e3e7-456c-9f27-7679c0074f4f,d3ef929e-e3e7-456c-9f27-7679c0074f4f,2013-02-18T15:54:01,,,201511091147,,bob,3,2,False,23.708174238006087,90.40946505581161,tertiary,2,Patuatuli Road,234134797,yes,4,23.707316084046038,90.40849938337506,kol,34298972

onadata/apps/api/tests/viewsets/test_attachment_viewset.py

@@ -1,12 +1,12 @@
-from os import path
+import os
 from django.utils import timezone
 
 from onadata.apps.api.tests.viewsets.test_abstract_viewset import \
     TestAbstractViewSet
 from onadata.apps.api.viewsets.attachment_viewset import AttachmentViewSet
 from onadata.apps.logger.models.attachment import Attachment
 from onadata.apps.logger.models.instance import get_attachment_url
-from onadata.apps.logger.models.instance import Instance
+from onadata.apps.logger.import_tools import django_file
 
 
 def attachment_url(attachment, suffix=None):
@@ -67,29 +67,22 @@ def test_retrieve_view(self):
         response = self.retrieve_view(request, pk=pk)
         self.assertEqual(response.status_code, 404)
 
-    def test_attachments_added_on_duplicate_submission_has_start_time(self):
-        self.xform.has_start_time = True
-        self.xform.save()
-        count = Attachment.objects.count()
-        instance_count = Instance.objects.count()
-        self._submit_transport_instance_w_attachment()
-        self.assertEqual(self.response.status_code, 201)
-        self.assertEqual(Attachment.objects.count(), count + 1)
-        self.assertEqual(Instance.objects.count(), instance_count + 1)
-        self._submit_transport_instance_w_attachment(
-            media_file="1335783522564.JPG")
-        self.assertEqual(self.response.status_code, 202)
-        # change in number of attachments
-        self.assertEqual(Attachment.objects.count(), count + 2)
-        # no change in number of submissions
-        self.assertEqual(Instance.objects.count(), instance_count + 1)
-
     def test_attachment_pagination(self):
+        """
+        Test attachments endpoint pagination support.
+        """
         self._submit_transport_instance_w_attachment()
         self.assertEqual(self.response.status_code, 201)
-        self._submit_transport_instance_w_attachment(
-            media_file="1335783522564.JPG")
-        self.assertEqual(self.response.status_code, 202)
+        filename = "1335783522564.JPG"
+        path = os.path.join(self.main_directory, 'fixtures', 'transportation',
+                            'instances', self.surveys[0], filename)
+        media_file = django_file(path, 'image2', 'image/jpeg')
+        Attachment.objects.create(
+            instance=self.xform.instances.first(),
+            mimetype='image/jpeg',
+            extension='JPG',
+            name=filename,
+            media_file=media_file)
 
         # not using pagination params
         request = self.factory.get('/', **self.extra)
@@ -276,11 +269,16 @@ def test_direct_image_link(self):
         self.assertEqual(response.status_code, 404)
 
     def test_direct_image_link_uppercase(self):
-        self._submit_transport_instance_w_attachment(
-            media_file="1335783522564.JPG")
+        self._submit_transport_instance_w_attachment()
+        filename = "1335783522564.JPG"
+        path = os.path.join(self.main_directory, 'fixtures', 'transportation',
+                            'instances', self.surveys[0], filename)
+        self.attachment.media_file = django_file(path, 'image2', 'image/jpeg')
+        self.attachment.name = filename
+        self.attachment.save()
 
         filename = self.attachment.media_file.name
-        file_base, file_extension = path.splitext(filename)
+        file_base, file_extension = os.path.splitext(filename)
         data = {
             'filename': file_base + file_extension.upper()
         }

onadata/apps/api/tests/viewsets/test_data_viewset.py

@@ -11,6 +11,7 @@
 from django_digest.test import DigestAuth
 from django_digest.test import Client as DigestClient
 from httmock import urlmatch, HTTMock
+from tempfile import NamedTemporaryFile
 
 from onadata.apps.api.viewsets.data_viewset import DataViewSet
 from onadata.apps.main import tests as main_tests
@@ -1183,73 +1184,102 @@ def test_get_form_public_data(self):
         self.assertDictContainsSubset(data, sorted(response.data)[0])
 
     def test_same_submission_with_different_attachments(self):
-        self._submit_transport_instance_w_attachment()
-
+        """
+        Test same submission with different attachments on each request.
+        """
+        images_md = """
+        | survey |
+        |        | type  | name   | label |
+        |        | photo | image1 | Pic 1 |
+        |        | photo | image2 | Pic 2 |
+        """
+        xform = self._publish_markdown(images_md, self.user)
+        submission_file = NamedTemporaryFile(delete=False)
+        with open(submission_file.name, 'w') as xml_file:
+            xml_file.write(
+                "<?xml version='1.0'?><data id=\"%s\">"
+                "<image1>1335783522563.jpg</image1>"
+                "<image2>1442323232322.jpg</image2>"
+                "<meta><instanceID>uuid:729f173c688e482486a48661700455ff"
+                "</instanceID></meta></data>" %
+                (xform.id_string))
+        media_file = "1335783522563.jpg"
+        self._make_submission_w_attachment(
+            submission_file.name,
+            os.path.join(self.this_directory, 'fixtures', 'transportation',
+                         'instances', self.surveys[0], media_file))
         view = DataViewSet.as_view({'get': 'list'})
         request = self.factory.get('/', **self.extra)
         response = view(request)
         self.assertEqual(response.status_code, 200)
-        formid = self.xform.pk
-        data = _data_list(formid)
-        self.assertEqual(response.data, data)
+        formid = xform.pk
+        data = {
+            u'id': formid,
+            u'id_string': xform.id_string,
+            u'title': xform.title,
+            u'description': '',
+            u'url': u'http://testserver/api/v1/data/%s' % formid
+        }
+        self.assertEqual(response.data[1], data)
         response = view(request, pk=formid)
         self.assertEqual(response.status_code, 200)
 
-        dataid = self.xform.instances.all().order_by('id')[0].pk
+        instance = xform.instances.all().order_by('id')[0]
+        dataid = instance.pk
+        attachment = instance.attachments.all().first()
 
         data = {
             u'_bamboo_dataset_id': u'',
             u'_attachments': [{
-                'download_url': get_attachment_url(self.attachment),
-                'small_download_url':
-                get_attachment_url(self.attachment, 'small'),
-                'medium_download_url':
-                get_attachment_url(self.attachment, 'medium'),
-                u'mimetype': self.attachment.mimetype,
-                u'instance': self.attachment.instance.pk,
-                u'filename': self.attachment.media_file.name,
-                u'id': self.attachment.pk,
-                u'xform': self.xform.id}
+                u'download_url': get_attachment_url(attachment),
+                u'small_download_url':
+                get_attachment_url(attachment, 'small'),
+                u'medium_download_url':
+                get_attachment_url(attachment, 'medium'),
+                u'mimetype': attachment.mimetype,
+                u'instance': attachment.instance.pk,
+                u'filename': attachment.media_file.name,
+                u'id': attachment.pk,
+                u'xform': xform.id}
             ],
             u'_geolocation': [None, None],
-            u'_xform_id_string': u'transportation_2011_07_25',
-            u'transport/available_transportation_types_to_referral_facility':
-            u'none',
+            u'_xform_id_string': xform.id_string,
             u'_status': u'submitted_via_web',
-            u'_id': dataid
+            u'_id': dataid,
+            u'image1': u'1335783522563.jpg'
         }
         self.assertDictContainsSubset(data, sorted(response.data)[0])
 
         patch_value = 'onadata.libs.utils.logger_tools.get_filtered_instances'
         with patch(patch_value) as get_filtered_instances:
             get_filtered_instances.return_value = Instance.objects.filter(
                 uuid='#doesnotexist')
-            s = self.surveys[0]
             media_file = "1442323232322.jpg"
-            self._make_submission_w_attachment(os.path.join(
-                self.this_directory, 'fixtures',
-                'transportation', 'instances', s, s + '.xml'),
-                os.path.join(self.this_directory, 'fixtures',
-                             'transportation', 'instances', s, media_file))
-            self.attachment = Attachment.objects.last()
-            self.attachment_media_file = self.attachment.media_file
+            self._make_submission_w_attachment(
+                submission_file.name,
+                os.path.join(self.this_directory, 'fixtures', 'transportation',
+                             'instances', self.surveys[0], media_file))
+            attachment = Attachment.objects.get(name=media_file)
 
             data['_attachments'] = data.get('_attachments') + [{
-                'download_url': get_attachment_url(self.attachment),
-                'small_download_url':
-                get_attachment_url(self.attachment, 'small'),
-                'medium_download_url':
-                get_attachment_url(self.attachment, 'medium'),
-                u'mimetype': self.attachment.mimetype,
-                u'instance': self.attachment.instance.pk,
-                u'filename': self.attachment.media_file.name,
-                u'id': self.attachment.pk,
-                u'xform': self.xform.id
+                u'download_url': get_attachment_url(attachment),
+                u'small_download_url':
+                get_attachment_url(attachment, 'small'),
+                u'medium_download_url':
+                get_attachment_url(attachment, 'medium'),
+                u'mimetype': attachment.mimetype,
+                u'instance': attachment.instance.pk,
+                u'filename': attachment.media_file.name,
+                u'id': attachment.pk,
+                u'xform': xform.id
             }]
+            self.maxDiff = None
             response = view(request, pk=formid)
-
-            self.assertDictContainsSubset(data, sorted(response.data)[0])
+            self.assertDictContainsSubset(sorted([data])[0],
+                                          sorted(response.data)[0])
             self.assertEqual(response.status_code, 200)
+        submission_file.close()
+        os.unlink(submission_file.name)
 
     def test_data_w_attachment(self):
         self._submit_transport_instance_w_attachment()
@@ -2130,7 +2160,7 @@ def test_filterset(self):
                                    **self.extra)
         view = DataViewSet.as_view({'get': 'list'})
         response = view(request, pk=formid, format='json')
-        self.assertEqual(len(response.data), 0)
+        self.assertEqual(len(response.data), 1)
         # we set one to False and filter for it
         instance.media_all_received = False
         instance.save()
@@ -2139,7 +2169,7 @@ def test_filterset(self):
                                    **self.extra)
         view = DataViewSet.as_view({'get': 'list'})
         response = view(request, pk=formid, format='json')
-        self.assertEqual(len(response.data), 1)
+        self.assertEqual(len(response.data), 2)
 
     def test_floip_format(self):
         """

onadata/apps/api/tests/viewsets/test_xform_submission_viewset.py

@@ -226,40 +226,35 @@ def test_post_submission_authenticated_bad_json(self):
                              'http://testserver/submission')
 
     def test_post_submission_require_auth(self):
+        """
+        Test require_auth on submission post.
+        """
         self.user.profile.require_auth = True
         self.user.profile.save()
-        count = Attachment.objects.count()
-        s = self.surveys[0]
-        media_file = "1335783522563.jpg"
-        path = os.path.join(self.main_directory, 'fixtures',
-                            'transportation', 'instances', s, media_file)
-        with open(path) as f:
-            f = InMemoryUploadedFile(f, 'media_file', media_file, 'image/jpg',
-                                     os.path.getsize(path), None)
-            submission_path = os.path.join(
-                self.main_directory, 'fixtures',
-                'transportation', 'instances', s, s + '.xml')
-            with open(submission_path) as sf:
-                data = {'xml_submission_file': sf, 'media_file': f}
-                request = self.factory.post('/submission', data)
-                response = self.view(request)
-                self.assertEqual(response.status_code, 401)
-                response = self.view(request, username=self.user.username)
-                self.assertEqual(response.status_code, 401)
-                auth = DigestAuth('bob', 'bobbob')
-                request.META.update(auth(request.META, response))
-                response = self.view(request, username=self.user.username)
-                self.assertContains(response, 'Successful submission',
-                                    status_code=201)
-                self.assertEqual(count + 1, Attachment.objects.count())
-                self.assertTrue(response.has_header('X-OpenRosa-Version'))
-                self.assertTrue(
-                    response.has_header('X-OpenRosa-Accept-Content-Length'))
-                self.assertTrue(response.has_header('Date'))
-                self.assertEqual(response['Content-Type'],
-                                 'text/xml; charset=utf-8')
-                self.assertEqual(response['Location'],
-                                 'http://testserver/submission')
+        submission = self.surveys[0]
+        submission_path = os.path.join(
+            self.main_directory, 'fixtures',
+            'transportation', 'instances', submission, submission + '.xml')
+        with open(submission_path) as submission_file:
+            data = {'xml_submission_file': submission_file}
+            request = self.factory.post('/submission', data)
+            response = self.view(request)
+            self.assertEqual(response.status_code, 401)
+            response = self.view(request, username=self.user.username)
+            self.assertEqual(response.status_code, 401)
+            auth = DigestAuth('bob', 'bobbob')
+            request.META.update(auth(request.META, response))
+            response = self.view(request, username=self.user.username)
+            self.assertContains(response, 'Successful submission',
+                                status_code=201)
+            self.assertTrue(response.has_header('X-OpenRosa-Version'))
+            self.assertTrue(
+                response.has_header('X-OpenRosa-Accept-Content-Length'))
+            self.assertTrue(response.has_header('Date'))
+            self.assertEqual(response['Content-Type'],
+                             'text/xml; charset=utf-8')
+            self.assertEqual(response['Location'],
+                             'http://testserver/submission')
 
     def test_post_submission_require_auth_anonymous_user(self):
         self.user.profile.require_auth = True