From d54222e77e2669a9f9013f5998d551efc3aca822 Mon Sep 17 00:00:00 2001 From: Kelvin Muchiri Date: Mon, 3 Jun 2024 14:26:08 +0300 Subject: [PATCH 01/29] add merged dataset geojson format on endpoint /api/v1/data/ --- .../api/tests/viewsets/test_data_viewset.py | 34 ++++++++++++++ onadata/apps/api/viewsets/data_viewset.py | 15 +++--- .../logger/models/tests/test_merged_xform.py | 37 ++------------- onadata/apps/main/tests/test_base.py | 46 ++++++++++++++++++- .../libs/tests/models/test_share_project.py | 26 +---------- .../libs/tests/utils/test_project_utils.py | 37 ++------------- 6 files changed, 96 insertions(+), 99 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_data_viewset.py b/onadata/apps/api/tests/viewsets/test_data_viewset.py index d30b708283..f3d54d2b91 100644 --- a/onadata/apps/api/tests/viewsets/test_data_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_data_viewset.py @@ -3707,6 +3707,40 @@ def test_data_paginated_past_threshold(self): '; rel="last"', ) + def test_merged_dataset(self): + """Data for merged dataset is returned""" + merged_xf = self._create_merged_dataset(make_submissions=True) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) + response = view(request, pk=merged_xf.pk) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 2) + + def test_merged_dataset_geojson(self): + """Merged dataset geojson works""" + merged_xf = self._create_merged_dataset(make_submissions=True) + view = DataViewSet.as_view({"get": "list"}) + request = self.factory.get("/", **self.extra) + response = view(request, pk=merged_xf.pk, format="geojson") + self.assertEqual(response.status_code, 200) + # we get correct content type + headers = dict(response.items()) + self.assertEqual(headers["Content-Type"], "application/geo+json") + del response.data["features"][0]["properties"]["xform"] + del response.data["features"][1]["properties"]["xform"] + del response.data["features"][0]["properties"]["id"] + del response.data["features"][1]["properties"]["id"] + self.assertEqual( + { + "type": "FeatureCollection", + "features": [ + {"type": "Feature", "geometry": None, "properties": {}}, + {"type": "Feature", "geometry": None, "properties": {}}, + ], + }, + response.data, + ) + class TestOSM(TestAbstractViewSet): """ diff --git a/onadata/apps/api/viewsets/data_viewset.py b/onadata/apps/api/viewsets/data_viewset.py index 693f9ef86a..99617d5407 100644 --- a/onadata/apps/api/viewsets/data_viewset.py +++ b/onadata/apps/api/viewsets/data_viewset.py @@ -648,17 +648,20 @@ def list(self, request, *args, **kwargs): return super().list(request, *args, **kwargs) if export_type == "geojson": - # raise 404 if all instances dont have geoms - if not xform.instances_with_geopoints and not ( - xform.polygon_xpaths() or xform.geotrace_xpaths() - ): - raise Http404(_("Not Found")) + if not is_merged_dataset: + # raise 404 if all instances dont have geoms + if not xform.instances_with_geopoints and not ( + xform.polygon_xpaths() or xform.geotrace_xpaths() + ): + raise Http404(_("Not Found")) # add pagination when fetching geojson features page = self.paginate_queryset(self.object_list) serializer = self.get_serializer(page, many=True) - return Response(serializer.data) + return Response( + serializer.data, headers={"Content-Type": "application/geo+json"} + ) return custom_response_handler(request, xform, query, export_type) diff --git a/onadata/apps/logger/models/tests/test_merged_xform.py b/onadata/apps/logger/models/tests/test_merged_xform.py index 3a206c2347..1ac9db9cc1 100644 --- a/onadata/apps/logger/models/tests/test_merged_xform.py +++ b/onadata/apps/logger/models/tests/test_merged_xform.py @@ -1,48 +1,17 @@ """Tests for module onadata.apps.logger.models.merged_xform""" -from pyxform.builder import create_survey_element_from_dict from unittest.mock import call, patch from onadata.apps.main.tests.test_base import TestBase -from onadata.apps.logger.models.merged_xform import MergedXForm -from onadata.apps.logger.models.xform import XForm class MergedXFormTestCase(TestBase): @patch("onadata.libs.utils.project_utils.set_project_perms_to_xform_async.delay") def test_perms_applied_async_on_create(self, mock_set_perms): """Permissions are applied asynchronously on create""" - md = """ - | survey | - | | type | name | label | - | | select one fruits | fruit | Fruit | - - | choices | - | | list name | name | label | - | | fruits | orange | Orange | - | | fruits | mango | Mango | - """ - self._publish_markdown(md, self.user, id_string="a") - self._publish_markdown(md, self.user, id_string="b") - xf1 = XForm.objects.get(id_string="a") - xf2 = XForm.objects.get(id_string="b") - survey = create_survey_element_from_dict(xf1.json_dict()) - survey["id_string"] = "c" - survey["sms_keyword"] = survey["id_string"] - survey["title"] = "Merged XForm" - merged_xf = MergedXForm.objects.create( - id_string=survey["id_string"], - sms_id_string=survey["id_string"], - title=survey["title"], - user=self.user, - created_by=self.user, - is_merged_dataset=True, - project=self.project, - xml=survey.to_xml(), - json=survey.to_json(), - ) - merged_xf.xforms.add(xf1) - merged_xf.xforms.add(xf2) + merged_xf = self._create_merged_dataset() + xf1 = merged_xf.xforms.get(id_string="a") + xf2 = merged_xf.xforms.get(id_string="b") calls = [ call(xf1.pk, self.project.pk), call(xf2.pk, self.project.pk), diff --git a/onadata/apps/main/tests/test_base.py b/onadata/apps/main/tests/test_base.py index 7eac49240d..a24c38e622 100644 --- a/onadata/apps/main/tests/test_base.py +++ b/onadata/apps/main/tests/test_base.py @@ -13,6 +13,8 @@ from io import StringIO from tempfile import NamedTemporaryFile +from pyxform.builder import create_survey_element_from_dict + from django.conf import settings from django.contrib.auth import authenticate, get_user_model from django.core.files.uploadedfile import InMemoryUploadedFile @@ -28,7 +30,7 @@ from six.moves.urllib.request import urlopen from onadata.apps.api.viewsets.xform_viewset import XFormViewSet -from onadata.apps.logger.models import Instance, XForm, XFormVersion +from onadata.apps.logger.models import Instance, MergedXForm, XForm, XFormVersion from onadata.apps.logger.views import submission from onadata.apps.logger.xform_instance_parser import clean_and_parse_xml from onadata.apps.main.models import UserProfile @@ -570,3 +572,45 @@ def _publish_follow_up_form(self, user, project=None): latest_form = XForm.objects.all().order_by("-pk").first() return latest_form + + def _create_merged_dataset(self, make_submissions=False): + md = """ + | survey | + | | type | name | label | + | | select one fruits | fruit | Fruit | + | choices | + | | list name | name | label | + | | fruits | orange | Orange | + | | fruits | mango | Mango | + """ + self._publish_markdown(md, self.user, id_string="a") + self._publish_markdown(md, self.user, id_string="b") + xf1 = XForm.objects.get(id_string="a") + xf2 = XForm.objects.get(id_string="b") + survey = create_survey_element_from_dict(xf1.json_dict()) + survey["id_string"] = "c" + survey["sms_keyword"] = survey["id_string"] + survey["title"] = "Merged XForm" + merged_xf = MergedXForm.objects.create( + id_string=survey["id_string"], + sms_id_string=survey["id_string"], + title=survey["title"], + user=self.user, + created_by=self.user, + is_merged_dataset=True, + project=self.project, + xml=survey.to_xml(), + json=survey.to_json(), + ) + merged_xf.xforms.add(xf1) + merged_xf.xforms.add(xf2) + + if make_submissions: + # Make submission for form a + xml = 'orange' + Instance(xform=xf1, xml=xml).save() + # Make submission for form b + xml = 'mango' + Instance(xform=xf2, xml=xml).save() + + return merged_xf diff --git a/onadata/libs/tests/models/test_share_project.py b/onadata/libs/tests/models/test_share_project.py index f60d299a24..5a29e06839 100644 --- a/onadata/libs/tests/models/test_share_project.py +++ b/onadata/libs/tests/models/test_share_project.py @@ -1,11 +1,9 @@ """Tests for module onadata.libs.models.share_project""" from unittest.mock import patch, call -from pyxform.builder import create_survey_element_from_dict from onadata.apps.logger.models.data_view import DataView from onadata.apps.logger.models.project import Project -from onadata.apps.logger.models.merged_xform import MergedXForm from onadata.apps.logger.models.xform import XForm from onadata.apps.main.tests.test_base import TestBase from onadata.libs.models.share_project import ShareProject @@ -37,7 +35,7 @@ def setUp(self): project = Project.objects.create( name="Demo", organization=self.user, created_by=self.user ) - self._publish_markdown(md_xform, self.user, project, id_string="a") + self._publish_markdown(md_xform, self.user, project) self.dataview_form = XForm.objects.all().order_by("-pk")[0] DataView.objects.create( name="Demo", @@ -46,27 +44,7 @@ def setUp(self): matches_parent=True, columns=[], ) - # MergedXForm - self._publish_markdown(md_xform, self.user, project, id_string="b") - xf1 = XForm.objects.get(id_string="a") - xf2 = XForm.objects.get(id_string="b") - survey = create_survey_element_from_dict(xf1.json_dict()) - survey["id_string"] = "c" - survey["sms_keyword"] = survey["id_string"] - survey["title"] = "Merged XForm" - self.merged_xf = MergedXForm.objects.create( - id_string=survey["id_string"], - sms_id_string=survey["id_string"], - title=survey["title"], - user=self.user, - created_by=self.user, - is_merged_dataset=True, - project=self.project, - xml=survey.to_xml(), - json=survey.to_json(), - ) - self.merged_xf.xforms.add(xf1) - self.merged_xf.xforms.add(xf2) + self.merged_xf = self._create_merged_dataset() self.alice = self._create_user("alice", "Yuao8(-)") @patch("onadata.libs.models.share_project.safe_delete") diff --git a/onadata/libs/tests/utils/test_project_utils.py b/onadata/libs/tests/utils/test_project_utils.py index 9711c69675..133698367b 100644 --- a/onadata/libs/tests/utils/test_project_utils.py +++ b/onadata/libs/tests/utils/test_project_utils.py @@ -5,11 +5,10 @@ from unittest.mock import call, MagicMock, patch from django.test.utils import override_settings -from pyxform.builder import create_survey_element_from_dict from kombu.exceptions import OperationalError from requests import Response -from onadata.apps.logger.models import MergedXForm, Project, XForm +from onadata.apps.logger.models import Project from onadata.apps.main.tests.test_base import TestBase from onadata.libs.permissions import DataEntryRole from onadata.libs.utils.project_utils import ( @@ -75,38 +74,8 @@ def test_set_project_perms_to_xform_async(self, mock): @patch("onadata.libs.utils.project_utils.set_project_perms_to_xform") def test_set_project_perms_to_xform_async_mergedxform(self, mock): - """Permissions for a MergedXForm are set""" - md = """ - | survey | - | | type | name | label | - | | select one fruits | fruit | Fruit | - - | choices | - | | list name | name | label | - | | fruits | orange | Orange | - | | fruits | mango | Mango | - """ - self._publish_markdown(md, self.user, id_string="a") - self._publish_markdown(md, self.user, id_string="b") - xf1 = XForm.objects.get(id_string="a") - xf2 = XForm.objects.get(id_string="b") - survey = create_survey_element_from_dict(xf1.json_dict()) - survey["id_string"] = "c" - survey["sms_keyword"] = survey["id_string"] - survey["title"] = "Merged XForm" - merged_xf = MergedXForm.objects.create( - id_string=survey["id_string"], - sms_id_string=survey["id_string"], - title=survey["title"], - user=self.user, - created_by=self.user, - is_merged_dataset=True, - project=self.project, - xml=survey.to_xml(), - json=survey.to_json(), - ) - merged_xf.xforms.add(xf1) - merged_xf.xforms.add(xf2) + """set_project_perms_to_xform_async sets permissions for a MergedXForm""" + merged_xf = self._create_merged_dataset() set_project_perms_to_xform_async.delay(merged_xf.pk, self.project.pk) expected_calls = [ call(merged_xf.xform_ptr, self.project), From 743650460ad13383846b52b643c7463ba5e847a8 Mon Sep 17 00:00:00 2001 From: Kelvin Muchiri Date: Mon, 3 Jun 2024 14:40:14 +0300 Subject: [PATCH 02/29] update docs --- docs/merged-datasets.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/merged-datasets.rst b/docs/merged-datasets.rst index 50efd3b4e1..3299c8719d 100644 --- a/docs/merged-datasets.rst +++ b/docs/merged-datasets.rst @@ -138,7 +138,8 @@ Response Retrieving Data from a Merged Dataset -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + Returns the data from both forms. The key `_xform_id_string` can be used to differentiate data from linked forms. From 8c0f62f899162c85358a05057772cfeb29f5cba1 Mon Sep 17 00:00:00 2001 From: Kelvin Muchiri Date: Mon, 3 Jun 2024 14:42:57 +0300 Subject: [PATCH 03/29] update docs --- docs/merged-datasets.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/merged-datasets.rst b/docs/merged-datasets.rst index 3299c8719d..bea5119281 100644 --- a/docs/merged-datasets.rst +++ b/docs/merged-datasets.rst @@ -140,8 +140,7 @@ Response Retrieving Data from a Merged Dataset ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Returns the data from both forms. The key `_xform_id_string` can be used to -differentiate data from linked forms. +Returns the data from all linked forms .. raw:: html From c0310b3ba95ad525d75fea9f40be4b4fd6e58ed6 Mon Sep 17 00:00:00 2001 From: Kelvin Muchiri Date: Mon, 3 Jun 2024 14:57:38 +0300 Subject: [PATCH 04/29] update docs --- docs/merged-datasets.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/merged-datasets.rst b/docs/merged-datasets.rst index bea5119281..ff82ccea8c 100644 --- a/docs/merged-datasets.rst +++ b/docs/merged-datasets.rst @@ -140,7 +140,7 @@ Response Retrieving Data from a Merged Dataset ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Returns the data from all linked forms +Returns the data from all linked forms. .. raw:: html @@ -155,6 +155,8 @@ Returns the data from all linked forms curl -X GET "https://api.ona.io/api/v1/data/1" +Endpoint `/api/v1/data/{pk} `_ supports pagination +and advanced filtering options. Example Response ---------------- From 2728bec1dcaee12b0c6719f1c69c113266165771 Mon Sep 17 00:00:00 2001 From: Kelvin Muchiri Date: Mon, 3 Jun 2024 15:01:32 +0300 Subject: [PATCH 05/29] update docs --- docs/merged-datasets.rst | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/merged-datasets.rst b/docs/merged-datasets.rst index ff82ccea8c..27b782a408 100644 --- a/docs/merged-datasets.rst +++ b/docs/merged-datasets.rst @@ -143,17 +143,15 @@ Retrieving Data from a Merged Dataset Returns the data from all linked forms. .. raw:: html + 
GET /api/v1/merged-datasets/{pk}/data
- 
-    GET /api/v1/merged-datasets/{pk}/data
-    GET /api/v1/data/{pk}
-
+ +Example +^^^^^^^ :: curl -X GET "https://api.ona.io/api/v1/merged-datasets/1/data" - curl -X GET "https://api.ona.io/api/v1/data/1" - Endpoint `/api/v1/data/{pk} `_ supports pagination and advanced filtering options. From cf11c4c7f0bff98e5d35e2be5272358bf9713b79 Mon Sep 17 00:00:00 2001 From: Kelvin Muchiri Date: Mon, 3 Jun 2024 15:06:01 +0300 Subject: [PATCH 06/29] update docs --- docs/merged-datasets.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/merged-datasets.rst b/docs/merged-datasets.rst index 27b782a408..9697a11578 100644 --- a/docs/merged-datasets.rst +++ b/docs/merged-datasets.rst @@ -153,10 +153,7 @@ Example curl -X GET "https://api.ona.io/api/v1/merged-datasets/1/data" -Endpoint `/api/v1/data/{pk} `_ supports pagination -and advanced filtering options. - -Example Response +Response ---------------- :: @@ -167,6 +164,9 @@ Example Response {"date": "2015-05-19", "gender": "male", "age": 21, "name": "Tom", "_xform_id_string": "form_c"} ] + +For data pagination and advanced filteriing options, use endpoint `/api/v1/data/{pk} `_ + How data in parent forms differs from and affects the merged xform ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -174,6 +174,6 @@ A merged dataset combines data from multiple forms into one form. It creates a n A merged dataset: - Does not allow submissions or data edits, this can only be done on the individual forms. - - Data deleted from the individual forms will also not be present in the mereged dataset. + - Data deleted from the individual forms will also not be present in the merged dataset. - Form replacement is not supported. - It has it's own form structure, which is not replaceable the same way you could replace an individual form when changing certain aspects of a form. From c76164a510078278f9b0046c4c21184c325c8c31 Mon Sep 17 00:00:00 2001 From: Kelvin Muchiri Date: Mon, 3 Jun 2024 15:08:01 +0300 Subject: [PATCH 07/29] update docs --- docs/merged-datasets.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/merged-datasets.rst b/docs/merged-datasets.rst index 9697a11578..7ff6032da4 100644 --- a/docs/merged-datasets.rst +++ b/docs/merged-datasets.rst @@ -143,6 +143,7 @@ Retrieving Data from a Merged Dataset Returns the data from all linked forms. .. raw:: html + 
GET /api/v1/merged-datasets/{pk}/data
@@ -151,10 +152,10 @@ Example :: - curl -X GET "https://api.ona.io/api/v1/merged-datasets/1/data" + curl -X GET "https://api.ona.io/api/v1/merged-datasets/1/data" Response ----------------- +-------- :: From cd23c3c3530ca8de91efce149bc240566952159a Mon Sep 17 00:00:00 2001 From: Kelvin Muchiri Date: Mon, 3 Jun 2024 15:10:33 +0300 Subject: [PATCH 08/29] update docs --- docs/merged-datasets.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/merged-datasets.rst b/docs/merged-datasets.rst index 7ff6032da4..748d58d29e 100644 --- a/docs/merged-datasets.rst +++ b/docs/merged-datasets.rst @@ -148,7 +148,7 @@ Returns the data from all linked forms. Example -^^^^^^^ +------- :: @@ -156,6 +156,7 @@ Example Response -------- + :: From 36b086b2ab122e345616cf4279bcf01e03ab4d1e Mon Sep 17 00:00:00 2001 From: Kelvin Muchiri Date: Mon, 3 Jun 2024 15:12:27 +0300 Subject: [PATCH 09/29] update docs --- docs/merged-datasets.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/merged-datasets.rst b/docs/merged-datasets.rst index 748d58d29e..432d010665 100644 --- a/docs/merged-datasets.rst +++ b/docs/merged-datasets.rst @@ -159,7 +159,6 @@ Response :: - [ {"date": "2015-05-19", "gender": "male", "age": 32, "name": "Kendy", "_xform_id_string": "form_a"}, {"date": "2015-05-19", "gender": "female", "age": 41, "name": "Maasai", "_xform_id_string": "form_b"}, From 4bd44f107e15835ac0398c801823ecd0f6df8763 Mon Sep 17 00:00:00 2001 From: Kelvin Muchiri Date: Mon, 3 Jun 2024 15:39:51 +0300 Subject: [PATCH 10/29] mark flaky test --- onadata/apps/api/tests/viewsets/test_xform_viewset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/onadata/apps/api/tests/viewsets/test_xform_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_viewset.py index a5969e631c..08300f830c 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_viewset.py @@ -3567,6 +3567,7 @@ def test_failed_form_publishing_after_maximum_retries( self.assertEqual(response.status_code, 202) self.assertEqual(response.data, error_message) + @flaky(max_runs=3) def test_survey_preview_endpoint(self): view = XFormViewSet.as_view({"post": "survey_preview", "get": "survey_preview"}) From 0762efcf02e88dc94414c05c3e8ae5f476de2435 Mon Sep 17 00:00:00 2001 From: Kelvin Muchiri Date: Mon, 3 Jun 2024 15:59:15 +0300 Subject: [PATCH 11/29] set max_runs for flaky test --- onadata/apps/api/tests/viewsets/test_xform_viewset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onadata/apps/api/tests/viewsets/test_xform_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_viewset.py index 08300f830c..3e1b033fc4 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_viewset.py @@ -2284,7 +2284,7 @@ def test_form_clone_shared_forms(self): self.assertEqual(response.status_code, 201) self.assertEqual(count + 1, XForm.objects.count()) - @flaky + @flaky(max_runs=3) def test_return_error_on_clone_duplicate(self): with HTTMock(enketo_mock): self._publish_xls_form_to_project() From 2f27e672962770fa6d6cb02663a22d45f4b5e9f3 Mon Sep 17 00:00:00 2001 From: Kelvin Muchiri Date: Mon, 3 Jun 2024 17:03:42 +0300 Subject: [PATCH 12/29] update flaky test max run --- onadata/apps/api/tests/viewsets/test_xform_viewset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onadata/apps/api/tests/viewsets/test_xform_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_viewset.py index 3e1b033fc4..65d4a5dce7 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_viewset.py @@ -2284,7 +2284,7 @@ def test_form_clone_shared_forms(self): self.assertEqual(response.status_code, 201) self.assertEqual(count + 1, XForm.objects.count()) - @flaky(max_runs=3) + @flaky(max_runs=8) def test_return_error_on_clone_duplicate(self): with HTTMock(enketo_mock): self._publish_xls_form_to_project() From 945bca845868e0cd307714ed61a7cf8a6af39b17 Mon Sep 17 00:00:00 2001 From: Kelvin Muchiri Date: Tue, 4 Jun 2024 11:45:06 +0300 Subject: [PATCH 13/29] fix typo --- docs/merged-datasets.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/merged-datasets.rst b/docs/merged-datasets.rst index 432d010665..26c90316d8 100644 --- a/docs/merged-datasets.rst +++ b/docs/merged-datasets.rst @@ -166,7 +166,7 @@ Response ] -For data pagination and advanced filteriing options, use endpoint `/api/v1/data/{pk} `_ +For data pagination and advanced filtering options, use endpoint `/api/v1/data/{pk} `_ How data in parent forms differs from and affects the merged xform ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From 2a7f3b7c0e1de96e016a138e1fc135176137e9c1 Mon Sep 17 00:00:00 2001 From: Kelvin Muchiri Date: Tue, 4 Jun 2024 13:00:36 +0300 Subject: [PATCH 14/29] enhance test case --- .../api/tests/viewsets/test_data_viewset.py | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_data_viewset.py b/onadata/apps/api/tests/viewsets/test_data_viewset.py index f3d54d2b91..6652bcead7 100644 --- a/onadata/apps/api/tests/viewsets/test_data_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_data_viewset.py @@ -3726,16 +3726,27 @@ def test_merged_dataset_geojson(self): # we get correct content type headers = dict(response.items()) self.assertEqual(headers["Content-Type"], "application/geo+json") - del response.data["features"][0]["properties"]["xform"] - del response.data["features"][1]["properties"]["xform"] - del response.data["features"][0]["properties"]["id"] - del response.data["features"][1]["properties"]["id"] + instance_qs = Instance.objects.all().order_by("pk") self.assertEqual( { "type": "FeatureCollection", "features": [ - {"type": "Feature", "geometry": None, "properties": {}}, - {"type": "Feature", "geometry": None, "properties": {}}, + { + "type": "Feature", + "geometry": None, + "properties": { + "id": instance_qs[0].pk, + "xform": instance_qs[0].xform.pk, + }, + }, + { + "type": "Feature", + "geometry": None, + "properties": { + "id": instance_qs[1].pk, + "xform": instance_qs[1].xform.pk, + }, + }, ], }, response.data, From bbc82331192e8e5d2ead70d0f8325e1ffd1bcde6 Mon Sep 17 00:00:00 2001 From: Kelvin Muchiri Date: Tue, 4 Jun 2024 17:04:37 +0300 Subject: [PATCH 15/29] add disclaimer for merged datasets docs --- docs/merged-datasets.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/merged-datasets.rst b/docs/merged-datasets.rst index 26c90316d8..8d4df7f160 100644 --- a/docs/merged-datasets.rst +++ b/docs/merged-datasets.rst @@ -1,6 +1,12 @@ Merged Datasets *************** +**This feature is currently in beta and is still undergoing testing. As a result, users may encounter bugs, glitches, or unexpected behavior. +While we have taken steps to ensure a stable experience, some functionality may not work as intended. Your feedback is invaluable in helping us improve this feature. +Please report any issues or provide suggestions to help us enhance the final version. Use this feature at your +own discretion and be prepared for potential interruptions or performance inconsistencies.** + + This endpoint provides access to data from multiple forms. Merged datasets should have the same functionality as the forms endpoint with the difference being: - They do not accept submissions directly, submissions to individual forms will be reflected in merged datasets.. From 44ac4bff15490344fbbfcce8ff1cc03e18731c05 Mon Sep 17 00:00:00 2001 From: Kelvin Muchiri Date: Tue, 4 Jun 2024 17:10:27 +0300 Subject: [PATCH 16/29] update docs --- docs/merged-datasets.rst | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/merged-datasets.rst b/docs/merged-datasets.rst index 8d4df7f160..68615731c4 100644 --- a/docs/merged-datasets.rst +++ b/docs/merged-datasets.rst @@ -1,10 +1,14 @@ Merged Datasets *************** -**This feature is currently in beta and is still undergoing testing. As a result, users may encounter bugs, glitches, or unexpected behavior. -While we have taken steps to ensure a stable experience, some functionality may not work as intended. Your feedback is invaluable in helping us improve this feature. -Please report any issues or provide suggestions to help us enhance the final version. Use this feature at your -own discretion and be prepared for potential interruptions or performance inconsistencies.** +.. warning:: + + **Disclaimer: Beta Feature** + + This feature is currently in beta and is still undergoing testing. As a result, users may encounter bugs, glitches, or unexpected behavior. While we have + taken steps to ensure a stable experience, some functionality may not work as intended. Your feedback is invaluable in helping us improve this feature. + Please report any issues or provide suggestions to help us enhance the final version. Use this feature at your own discretion and be prepared for potential + interruptions or performance inconsistencies. This endpoint provides access to data from multiple forms. Merged datasets should have the same functionality as the forms endpoint with the difference being: From daaaaa33765653ba8d427184005a10531c627565 Mon Sep 17 00:00:00 2001 From: Kelvin Muchiri Date: Tue, 4 Jun 2024 17:17:08 +0300 Subject: [PATCH 17/29] update docs --- docs/merged-datasets.rst | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/merged-datasets.rst b/docs/merged-datasets.rst index 68615731c4..0824e7dc0d 100644 --- a/docs/merged-datasets.rst +++ b/docs/merged-datasets.rst @@ -2,13 +2,12 @@ Merged Datasets *************** .. warning:: + **Disclaimer: Beta Feature** - **Disclaimer: Beta Feature** - - This feature is currently in beta and is still undergoing testing. As a result, users may encounter bugs, glitches, or unexpected behavior. While we have - taken steps to ensure a stable experience, some functionality may not work as intended. Your feedback is invaluable in helping us improve this feature. - Please report any issues or provide suggestions to help us enhance the final version. Use this feature at your own discretion and be prepared for potential - interruptions or performance inconsistencies. + This feature is currently in beta and is still undergoing testing. As a result, users may encounter bugs, glitches, or unexpected behavior. While we have + taken steps to ensure a stable experience, some functionality may not work as intended. Your feedback is invaluable in helping us improve this feature. + Please report any issues or provide suggestions to help us enhance the final version. Use this feature at your own discretion and be prepared for potential + interruptions or performance inconsistencies. This endpoint provides access to data from multiple forms. Merged datasets should have the same functionality as the forms endpoint with the difference being: From f73ab48a0a03b8c9cfd9dad6d2b3c661f6ef6a35 Mon Sep 17 00:00:00 2001 From: Kelvin Muchiri Date: Tue, 4 Jun 2024 17:19:36 +0300 Subject: [PATCH 18/29] update docs --- docs/merged-datasets.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/merged-datasets.rst b/docs/merged-datasets.rst index 0824e7dc0d..2d276c5623 100644 --- a/docs/merged-datasets.rst +++ b/docs/merged-datasets.rst @@ -2,6 +2,7 @@ Merged Datasets *************** .. warning:: + **Disclaimer: Beta Feature** This feature is currently in beta and is still undergoing testing. As a result, users may encounter bugs, glitches, or unexpected behavior. While we have From d3c4e4dd0cd084a6c4db24c2e1ebdb43ad373858 Mon Sep 17 00:00:00 2001 From: Kelvin Muchiri Date: Tue, 4 Jun 2024 17:20:50 +0300 Subject: [PATCH 19/29] update docs --- docs/merged-datasets.rst | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/merged-datasets.rst b/docs/merged-datasets.rst index 2d276c5623..61463ff21b 100644 --- a/docs/merged-datasets.rst +++ b/docs/merged-datasets.rst @@ -2,13 +2,12 @@ Merged Datasets *************** .. warning:: + **Disclaimer: Beta Feature** - **Disclaimer: Beta Feature** - - This feature is currently in beta and is still undergoing testing. As a result, users may encounter bugs, glitches, or unexpected behavior. While we have - taken steps to ensure a stable experience, some functionality may not work as intended. Your feedback is invaluable in helping us improve this feature. - Please report any issues or provide suggestions to help us enhance the final version. Use this feature at your own discretion and be prepared for potential - interruptions or performance inconsistencies. + This feature is currently in beta and is still undergoing testing. As a result, users may encounter bugs, glitches, or unexpected behavior. While we have + taken steps to ensure a stable experience, some functionality may not work as intended. Your feedback is invaluable in helping us improve this feature. + Please report any issues or provide suggestions to help us enhance the final version. Use this feature at your own discretion and be prepared for potential + interruptions or performance inconsistencies. This endpoint provides access to data from multiple forms. Merged datasets should have the same functionality as the forms endpoint with the difference being: From b8b34bdedd9ecc11805a631ac464d5a1db761520 Mon Sep 17 00:00:00 2001 From: Kelvin Muchiri Date: Tue, 4 Jun 2024 17:23:19 +0300 Subject: [PATCH 20/29] update docs --- docs/merged-datasets.rst | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/merged-datasets.rst b/docs/merged-datasets.rst index 61463ff21b..81788db974 100644 --- a/docs/merged-datasets.rst +++ b/docs/merged-datasets.rst @@ -5,9 +5,19 @@ Merged Datasets **Disclaimer: Beta Feature** This feature is currently in beta and is still undergoing testing. As a result, users may encounter bugs, glitches, or unexpected behavior. While we have - taken steps to ensure a stable experience, some functionality may not work as intended. Your feedback is invaluable in helping us improve this feature. - Please report any issues or provide suggestions to help us enhance the final version. Use this feature at your own discretion and be prepared for potential - interruptions or performance inconsistencies. + taken steps to ensure a stable experience, some functionality may not work as intended. + + Your feedback is invaluable in helping us improve this feature. Please report any issues or provide suggestions to help us enhance the final version. + + Use this feature at your own discretion and be prepared for potential interruptions or performance inconsistencies. + + +.. warning:: + This is warning text. Use a warning for information the user must + understand to avoid negative consequences. + + Warnings are formatted in the same way as notes. In the same way, + lines must be broken and indented under the warning tag. This endpoint provides access to data from multiple forms. Merged datasets should have the same functionality as the forms endpoint with the difference being: From 31149da349a0f057ea1ccf1db917d3dacc3c71e3 Mon Sep 17 00:00:00 2001 From: Kelvin Muchiri Date: Tue, 4 Jun 2024 17:32:28 +0300 Subject: [PATCH 21/29] update docs --- docs/merged-datasets.rst | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/docs/merged-datasets.rst b/docs/merged-datasets.rst index 81788db974..7480377862 100644 --- a/docs/merged-datasets.rst +++ b/docs/merged-datasets.rst @@ -2,6 +2,7 @@ Merged Datasets *************** .. warning:: + **Disclaimer: Beta Feature** This feature is currently in beta and is still undergoing testing. As a result, users may encounter bugs, glitches, or unexpected behavior. While we have @@ -12,14 +13,6 @@ Merged Datasets Use this feature at your own discretion and be prepared for potential interruptions or performance inconsistencies. -.. warning:: - This is warning text. Use a warning for information the user must - understand to avoid negative consequences. - - Warnings are formatted in the same way as notes. In the same way, - lines must be broken and indented under the warning tag. - - This endpoint provides access to data from multiple forms. Merged datasets should have the same functionality as the forms endpoint with the difference being: - They do not accept submissions directly, submissions to individual forms will be reflected in merged datasets.. From 528cb68e59c705a3f3c235aeded9c9fd1b333b44 Mon Sep 17 00:00:00 2001 From: Kelvin Muchiri Date: Tue, 4 Jun 2024 17:34:27 +0300 Subject: [PATCH 22/29] update docs --- docs/merged-datasets.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/merged-datasets.rst b/docs/merged-datasets.rst index 7480377862..6d1108326f 100644 --- a/docs/merged-datasets.rst +++ b/docs/merged-datasets.rst @@ -3,8 +3,6 @@ Merged Datasets .. warning:: - **Disclaimer: Beta Feature** - This feature is currently in beta and is still undergoing testing. As a result, users may encounter bugs, glitches, or unexpected behavior. While we have taken steps to ensure a stable experience, some functionality may not work as intended. From b1c78106cbd54eeeb43b6880b8c28c40b7341d1e Mon Sep 17 00:00:00 2001 From: Kelvin Muchiri Date: Tue, 4 Jun 2024 17:42:10 +0300 Subject: [PATCH 23/29] update docs --- docs/merged-datasets.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/merged-datasets.rst b/docs/merged-datasets.rst index 6d1108326f..8b71a430d7 100644 --- a/docs/merged-datasets.rst +++ b/docs/merged-datasets.rst @@ -1,8 +1,8 @@ Merged Datasets *************** -.. warning:: - +.. warning:: Disclaimer: Beta Feature + This feature is currently in beta and is still undergoing testing. As a result, users may encounter bugs, glitches, or unexpected behavior. While we have taken steps to ensure a stable experience, some functionality may not work as intended. From f50368e9dec28cc30cf6e516460a23876378bd3a Mon Sep 17 00:00:00 2001 From: Kelvin Muchiri Date: Tue, 4 Jun 2024 17:44:39 +0300 Subject: [PATCH 24/29] update docs --- docs/merged-datasets.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/merged-datasets.rst b/docs/merged-datasets.rst index 8b71a430d7..d5ff639c50 100644 --- a/docs/merged-datasets.rst +++ b/docs/merged-datasets.rst @@ -1,6 +1,12 @@ Merged Datasets *************** ++------------------------+ +| !DANGER! | +| | +| Beware killer rabbits! | ++------------------------+ + .. warning:: Disclaimer: Beta Feature This feature is currently in beta and is still undergoing testing. As a result, users may encounter bugs, glitches, or unexpected behavior. While we have From 121fdc675b1c6c8ff41b92f6b3c68b26330ea932 Mon Sep 17 00:00:00 2001 From: Kelvin Muchiri Date: Tue, 4 Jun 2024 17:56:52 +0300 Subject: [PATCH 25/29] update docs --- docs/merged-datasets.rst | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/docs/merged-datasets.rst b/docs/merged-datasets.rst index d5ff639c50..43e4cd4db5 100644 --- a/docs/merged-datasets.rst +++ b/docs/merged-datasets.rst @@ -1,16 +1,9 @@ Merged Datasets *************** -+------------------------+ -| !DANGER! | -| | -| Beware killer rabbits! | -+------------------------+ - .. warning:: Disclaimer: Beta Feature - This feature is currently in beta and is still undergoing testing. As a result, users may encounter bugs, glitches, or unexpected behavior. While we have - taken steps to ensure a stable experience, some functionality may not work as intended. + This feature is currently in beta and is still undergoing testing. As a result, users may encounter bugs, glitches, or unexpected behavior. While we have taken steps to ensure a stable experience, some functionality may not work as intended. Your feedback is invaluable in helping us improve this feature. Please report any issues or provide suggestions to help us enhance the final version. From d9c688156a9b2eb84d5e1aa9c5767cc56adfab85 Mon Sep 17 00:00:00 2001 From: Kelvin Muchiri Date: Wed, 5 Jun 2024 09:29:50 +0300 Subject: [PATCH 26/29] update docs --- docs/merged-datasets.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/merged-datasets.rst b/docs/merged-datasets.rst index 43e4cd4db5..6589fc5e11 100644 --- a/docs/merged-datasets.rst +++ b/docs/merged-datasets.rst @@ -1,7 +1,7 @@ Merged Datasets *************** -.. warning:: Disclaimer: Beta Feature +.. warning:: **Disclaimer: Beta Feature** This feature is currently in beta and is still undergoing testing. As a result, users may encounter bugs, glitches, or unexpected behavior. While we have taken steps to ensure a stable experience, some functionality may not work as intended. From 7d2a44a66348b3d1ed25474969044ebd3a10b88d Mon Sep 17 00:00:00 2001 From: Kelvin Muchiri Date: Wed, 5 Jun 2024 10:37:33 +0300 Subject: [PATCH 27/29] update docs --- docs/merged-datasets.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/merged-datasets.rst b/docs/merged-datasets.rst index 6589fc5e11..b0eb279d19 100644 --- a/docs/merged-datasets.rst +++ b/docs/merged-datasets.rst @@ -1,9 +1,9 @@ Merged Datasets *************** -.. warning:: **Disclaimer: Beta Feature** +.. warning:: **Disclaimer: Experimental Feature** - This feature is currently in beta and is still undergoing testing. As a result, users may encounter bugs, glitches, or unexpected behavior. While we have taken steps to ensure a stable experience, some functionality may not work as intended. + This feature is experimental. As a result, users may encounter bugs, glitches, or unexpected behavior. While we have taken steps to ensure a stable experience, some functionality may not work as intended. Your feedback is invaluable in helping us improve this feature. Please report any issues or provide suggestions to help us enhance the final version. From eb0594800d2f7d90c79bb4f1343e89aaae4fea30 Mon Sep 17 00:00:00 2001 From: Kelvin Muchiri Date: Wed, 5 Jun 2024 11:16:46 +0300 Subject: [PATCH 28/29] update flaky test max_runs --- onadata/apps/logger/tests/test_briefcase_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onadata/apps/logger/tests/test_briefcase_client.py b/onadata/apps/logger/tests/test_briefcase_client.py index ead683b0d9..1e80ea8272 100644 --- a/onadata/apps/logger/tests/test_briefcase_client.py +++ b/onadata/apps/logger/tests/test_briefcase_client.py @@ -169,7 +169,7 @@ def _download_submissions(self): mocker.head(requests_mock.ANY, content=submission_list) self.briefcase_client.download_instances(self.xform.id_string) - @flaky(max_runs=8) + @flaky(max_runs=10) def test_download_xform_xml(self): """ Download xform via briefcase api From 23cca3bc16e7d6c4c673a9bb47ea2c235db9ca37 Mon Sep 17 00:00:00 2001 From: Kelvin Muchiri Date: Wed, 5 Jun 2024 11:32:10 +0300 Subject: [PATCH 29/29] update flaky test max_runs --- onadata/apps/api/tests/viewsets/test_data_viewset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onadata/apps/api/tests/viewsets/test_data_viewset.py b/onadata/apps/api/tests/viewsets/test_data_viewset.py index 6652bcead7..096e59c111 100644 --- a/onadata/apps/api/tests/viewsets/test_data_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_data_viewset.py @@ -3766,7 +3766,7 @@ def setUp(self): self.logger = logging.getLogger("console_logger") # pylint: disable=invalid-name,too-many-locals - @flaky(max_runs=8) + @flaky(max_runs=10) def test_data_retrieve_instance_osm_format(self): """Test /data endpoint OSM format.""" filenames = [