Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Release/v1.2.0 #693

Merged
merged 24 commits into from
Apr 30, 2020
Merged
Changes from 6 commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
29d8d5e
Fetch data from FTS Google sheet
thenav56 Mar 17, 2020
da512c1
Log cronlog on error and success in Databank ingest
thenav56 Mar 19, 2020
1977c57
Add is_private field for Project
thenav56 Apr 1, 2020
fa2d310
Add Programme Type Domestic
thenav56 Apr 8, 2020
f6dda67
Add Project Bulk Import
thenav56 Apr 8, 2020
f520e59
Add reversion for Project
thenav56 Apr 8, 2020
7cb03ca
Private project only accessable to ifrc users.
thenav56 Apr 9, 2020
ae57909
Add dtype to Mini Event View
thenav56 Apr 9, 2020
1dab1fe
Replace is_private by visibility choices
thenav56 Apr 9, 2020
3d3bb66
Clean up data for project import
thenav56 Apr 14, 2020
d8dbcc4
Change Sector Health
thenav56 Apr 22, 2020
fae7ee6
Fix Public/Clinical Health Project Import Error
thenav56 Apr 28, 2020
ca1666b
Merge pull request #640 from IFRCGo/feature/databank-fts-google-sheet
GergiH Apr 28, 2020
588499a
Add Region Project Vizualization API
thenav56 Apr 21, 2020
153a596
Add Sankey API for project viz (region)
thenav56 Apr 24, 2020
52b9386
Add visibilty filter for Region 3W API
thenav56 Apr 27, 2020
2041f0a
Fix Project Test
thenav56 Apr 27, 2020
ba5d3b0
Add Secontary tag COVID
thenav56 Apr 29, 2020
4114d1e
Merge pull request #688 from IFRCGo/feature/region-project-viz-api
GergiH Apr 29, 2020
53d0503
Add Test For New Project API
thenav56 Apr 29, 2020
2f8c968
Merge pull request #691 from IFRCGo/feature/3W-few-changes
GergiH Apr 30, 2020
71b8c4f
object_name CharField -> TextField
GergiH Apr 30, 2020
86cb4d4
Merge pull request #692 from IFRCGo/fix/500-error
GergiH Apr 30, 2020
111f366
Update changelog for 3W Changes
thenav56 Apr 30, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
225 changes: 186 additions & 39 deletions deployments/drf_views.py
Original file line number Diff line number Diff line change
@@ -1,32 +1,36 @@
import json, datetime, pytz
from collections import defaultdict
from rest_framework.authentication import (
TokenAuthentication,
BasicAuthentication,
SessionAuthentication,
)
from rest_framework.permissions import IsAuthenticated
from rest_framework.decorators import action
from rest_framework.response import Response
from django_filters import rest_framework as filters
from django.shortcuts import render
from django.db.models import Q
from django.contrib.postgres.aggregates.general import ArrayAgg
from django.db.models import Q, Sum, Count, F, Subquery, OuterRef, IntegerField
from django.db.models.functions import Coalesce
from django.http import JsonResponse
from django.shortcuts import get_object_or_404
from rest_framework import viewsets
from reversion.views import RevisionMixin

from .filters import ProjectFilter
from .models import (
ERUOwner,
ERU,
PersonnelDeployment,
Personnel,
PartnerSocietyDeployment,
ProgrammeTypes,
Sectors,
OperationTypes,
Statuses,
RegionalProject,
Project,
)
from api.models import Country
from api.models import Country, Region
from api.view_filters import ListFilter
from .serializers import (
ERUOwnerSerializer,
@@ -131,35 +135,6 @@ class RegionalProjectViewset(viewsets.ReadOnlyModelViewSet):
search_fields = ('name',)


class ProjectFilter(filters.FilterSet):
budget_amount = filters.NumberFilter(field_name='budget_amount', lookup_expr='exact')
country = filters.CharFilter(field_name='country', method='filter_country')

def filter_country(self, queryset, name, value):
return queryset.filter(
# ISO2
Q(project_country__iso__iexact=value) |
Q(project_district__country__iso__iexact=value) |
# ISO3
Q(project_country__iso3__iexact=value) |
Q(project_district__country__iso3__iexact=value)
)

class Meta:
model = Project
fields = [
'country',
'budget_amount',
'start_date',
'end_date',
'project_district',
'reporting_ns',
'programme_type',
'status',
'primary_sector',
'operation_type',
]


class ProjectViewset(RevisionMixin, viewsets.ModelViewSet):
queryset = Project.objects.prefetch_related(
@@ -187,9 +162,181 @@ def get_permissions(self):
def get_queryset(self):
# Public Project are viewable to unauthenticated or non-ifrc users
qs = super().get_queryset()
user = self.request.user
if user.is_authenticated:
if user.email and user.email.endswith('@ifrc.org'):
return qs
return qs.exclude(visibility=Project.IFRC_ONLY)
return qs.filter(visibility=Project.PUBLIC)
return Project.get_for(self.request.user, queryset=qs)


class RegionProjectViewset(viewsets.ViewSet):
def get_region(self):
if not hasattr(self, '_region'):
self._region = get_object_or_404(Region, pk=self.kwargs['pk'])
return self._region

def get_projects(self):
region = self.get_region()
# Filter by region
qs = Project.objects.filter(
Q(project_country__region=region) |
Q(project_district__country__region=region)
).distinct()
# Filter by GET params
qs = ProjectFilter(self.request.query_params, queryset=qs).qs
# Filter by visibility
return Project.get_for(self.request.user, queryset=qs)

@action(detail=True, url_path='overview', methods=('get',))
def overview(self, request, pk=None):
projects = self.get_projects()
aggregate_data = projects.aggregate(
total_budget=Sum('budget_amount'),
target_total=Sum('target_total'),
reached_total=Sum('reached_total'),
)
return Response({
'total_projects': projects.count(),
'ns_with_ongoing_activities': projects.filter(
status=Statuses.ONGOING).order_by('reporting_ns').values('reporting_ns').distinct().count(),
'total_budget': aggregate_data['total_budget'],
'target_total': aggregate_data['target_total'],
'reached_total': aggregate_data['reached_total'],
'projects_by_status': projects.order_by().values('status').annotate(count=Count('id')).values('status', 'count')
})

@action(detail=True, url_path='movement-activities', methods=('get',))
def movement_activities(self, request, pk=None):
projects = self.get_projects()

def _get_country_ns_sector_count():
agg = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(int))))
fields = ('project_country', 'reporting_ns', 'primary_sector')
qs = projects.order_by().values(*fields).annotate(count=Count('id')).values_list(
*fields, 'project_country__name', 'reporting_ns__name', 'count')
for country, ns, sector, country_name, ns_name, count in qs:
agg[country][ns][sector] = count
agg[country]['name'] = country_name
agg[country][ns]['name'] = ns_name
return [
{
'id': cid,
'name': country.pop('name'),
'reporting_national_societies': [
{
'id': nsid,
'name': ns.pop('name'),
'sectors': [
{
'id': sector,
'sector': Sectors(sector).label,
'count': count,
} for sector, count in ns.items()
],
}
for nsid, ns in country.items()
],
}
for cid, country in agg.items()
]

region = self.get_region()
country_projects = projects.filter(project_country=OuterRef('pk'))
countries = Country.objects.filter(region=region)
country_annotate = {
f'{status_label.lower()}_projects_count': Coalesce(Subquery(
country_projects.filter(status=status).values('project_country').annotate(
count=Count('id')).values('count')[:1],
output_field=IntegerField(),
), 0) for status, status_label in Statuses.choices()
}

return Response({
'total_projects': projects.count(),
'countries_count': countries.annotate(
projects_count=Coalesce(Subquery(
projects.filter(project_country=OuterRef('pk')).values('project_country').annotate(
count=Count('*')).values('count')[:1],
output_field=IntegerField(),
), 0),
**country_annotate,
).values('id', 'name', 'iso', 'iso3', 'projects_count', *country_annotate.keys()),
'country_ns_sector_count': _get_country_ns_sector_count(),
'supporting_ns': projects.order_by().values('reporting_ns').annotate(count=Count('id')).values(
'count', id=F('reporting_ns'), name=F('reporting_ns__name')),
})

@action(detail=True, url_path='national-society-activities', methods=('get',))
def national_society_activities(self, request, pk=None):
projects = self.get_projects()

def _get_distinct(field, *args, **kwargs):
return list(
projects.order_by().values(field).annotate(count=Count('*')).values(field, *args, **kwargs).distinct()
)

def _get_count(*fields):
return list(
projects.order_by().values(*fields).annotate(count=Count('*')).values_list(*fields, 'count')
)

# Raw nodes
supporting_ns_list = _get_distinct(
'reporting_ns',
iso3=F('reporting_ns__iso3'),
iso=F('reporting_ns__iso'),
name=F('reporting_ns__society_name')
)
receiving_ns_list = _get_distinct(
'project_country',
iso3=F('project_country__iso3'),
iso=F('project_country__iso'),
name=F('project_country__name')
)
sector_list = _get_distinct('primary_sector')

# Raw links
supporting_ns_and_sector_group = _get_count('reporting_ns', 'primary_sector')
sector_and_receiving_ns_group = _get_count('primary_sector', 'project_country')

# Node Types
SUPPORTING_NS = 'supporting_ns'
RECEIVING_NS = 'receiving_ns'
SECTOR = 'sector'

nodes = [
{
'id': node[id_selector],
'type': gtype,
**(
{
'name': node['name'],
'iso': node['iso'],
'iso3': node['iso3'],
} if gtype != SECTOR else {
'name': Sectors(node[id_selector]).label,
}
)
}
for group, gtype, id_selector in [
(supporting_ns_list, SUPPORTING_NS, 'reporting_ns'),
(sector_list, SECTOR, 'primary_sector'),
(receiving_ns_list, RECEIVING_NS, 'project_country'),
]
for node in group
]

node_id_map = {
f"{node['type']}-{node['id']}": index
for index, node in enumerate(nodes)
}

links = [
{
'source': node_id_map[f"{source_type}-{source}"],
'target': node_id_map[f"{target_type}-{target}"],
'value': value,
}
for group, source_type, target_type in [
(supporting_ns_and_sector_group, SUPPORTING_NS, SECTOR),
(sector_and_receiving_ns_group, SECTOR, RECEIVING_NS),
]
for source, target, value in group
]
return Response({'nodes': nodes, 'links': links})
64 changes: 64 additions & 0 deletions deployments/filters.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
import django_filters as filters
from django.db.models import Q

from api.models import Region, Country
from .models import (
OperationTypes,
ProgrammeTypes,
Sectors,
Statuses,
Project,
)


class ProjectFilter(filters.FilterSet):
budget_amount = filters.NumberFilter(field_name='budget_amount', lookup_expr='exact')
country = filters.CharFilter(label='Country ISO/ISO3', field_name='country', method='filter_country')
region = filters.ModelChoiceFilter(label='Region', queryset=Region.objects.all(), method='filter_region')
operation_type = filters.MultipleChoiceFilter(choices=OperationTypes.choices(), widget=filters.widgets.CSVWidget)
programme_type = filters.MultipleChoiceFilter(choices=ProgrammeTypes.choices(), widget=filters.widgets.CSVWidget)
primary_sector = filters.MultipleChoiceFilter(choices=Sectors.choices(), widget=filters.widgets.CSVWidget)
status = filters.MultipleChoiceFilter(choices=Statuses.choices(), widget=filters.widgets.CSVWidget)

# Supporting/Receiving NS Filters (Multiselect)
project_country = filters.ModelMultipleChoiceFilter(queryset=Country.objects.all(), widget=filters.widgets.CSVWidget)
reporting_ns = filters.ModelMultipleChoiceFilter(queryset=Country.objects.all(), widget=filters.widgets.CSVWidget)

def filter_country(self, queryset, name, value):
if value:
return queryset.filter(
# ISO2
Q(project_country__iso__iexact=value) |
Q(project_district__country__iso__iexact=value) |
# ISO3
Q(project_country__iso3__iexact=value) |
Q(project_district__country__iso3__iexact=value)
)
return queryset

def filter_region(self, queryset, name, region):
if region:
return queryset.filter(
# ISO2
Q(project_country__region=region) |
Q(project_district__country__region=region) |
# ISO3
Q(project_country__region=region) |
Q(project_district__country__region=region)
)
return queryset

class Meta:
model = Project
fields = [
'country',
'budget_amount',
'start_date',
'end_date',
'project_district',
'reporting_ns',
'programme_type',
'status',
'primary_sector',
'operation_type',
]
10 changes: 10 additions & 0 deletions deployments/models.py
Original file line number Diff line number Diff line change
@@ -172,6 +172,7 @@ class SectorTags(IntEnum):
INTERNAL_DISPLACEMENT = 11
HEALTH_PUBLIC = 4
HEALTH_CLINICAL = 12
COVID = 13


class Statuses(IntEnum):
@@ -264,6 +265,15 @@ def __str__(self):
postfix = self.reporting_ns.society_name
return '%s (%s)' % (self.name, postfix)

@classmethod
def get_for(cls, user, queryset=None):
qs = cls.objects.all() if queryset is None else queryset
if user.is_authenticated:
if user.email and user.email.endswith('@ifrc.org'):
return qs
return qs.exclude(visibility=Project.IFRC_ONLY)
return qs.filter(visibility=Project.PUBLIC)


class ProjectImport(models.Model):
"""
4 changes: 3 additions & 1 deletion deployments/serializers.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
from rest_framework import serializers
from enumfields.drf.serializers import EnumSupportSerializerMixin

from .models import (
ERUOwner,
ERU,
@@ -98,7 +100,7 @@ class Meta:
fields = '__all__'


class ProjectSerializer(serializers.ModelSerializer):
class ProjectSerializer(EnumSupportSerializerMixin, serializers.ModelSerializer):
project_country_detail = MiniCountrySerializer(source='project_country', read_only=True)
project_district_detail = MiniDistrictSerializer(source='project_district', read_only=True)
reporting_ns_detail = MiniCountrySerializer(source='reporting_ns', read_only=True)
18 changes: 9 additions & 9 deletions deployments/test_views.py
Original file line number Diff line number Diff line change
@@ -94,15 +94,15 @@ def test_2(self):
'project_country': district2.country.id,
'project_district': district2.id,
'name': 'CreateMePls',
'programme_type': ProgrammeTypes.BILATERAL,
'primary_sector': Sectors.WASH,
'secondary_sectors': [Sectors.CEA, Sectors.PGI],
'operation_type': OperationTypes.EMERGENCY_OPERATION,
'programme_type': ProgrammeTypes.BILATERAL.value,
'primary_sector': Sectors.WASH.value,
'secondary_sectors': [Sectors.CEA.value, Sectors.PGI.value],
'operation_type': OperationTypes.EMERGENCY_OPERATION.value,
'start_date': '2012-11-12',
'end_date': '2013-11-13',
'budget_amount': 7000,
'target_total': 100,
'status': Statuses.PLANNED,
'status': Statuses.PLANNED.value,
}
self.client.credentials(HTTP_AUTHORIZATION='Token ' + token)
self.client.force_authenticate(user=user, token=token)
@@ -111,20 +111,20 @@ def test_2(self):

# Validation Tests
# Reached total should be provided if status is completed
body['status'] = Statuses.COMPLETED
body['status'] = Statuses.COMPLETED.value
body['reached_total'] = '' # The new framework does not allow None to be sent.
resp = self.client.post('/api/v2/project/', body)
self.assertEqual(resp.status_code, 400, resp.content)

# Disaster Type should be provided if operation type is Long Term Operation
body['operation_type'] = OperationTypes.PROGRAMME
body['operation_type'] = OperationTypes.PROGRAMME.value
body['dtype'] = ''
resp = self.client.post('/api/v2/project/', body)
self.assertEqual(resp.status_code, 400, resp.content)

# Event should be provided if operation type is Emergency Operation and programme type is Multilateral
body['operation_type'] = OperationTypes.PROGRAMME
body['programme_type'] = ProgrammeTypes.MULTILATERAL
body['operation_type'] = OperationTypes.PROGRAMME.value
body['programme_type'] = ProgrammeTypes.MULTILATERAL.value
body['event'] = ''
resp = self.client.post('/api/v2/project/', body)
self.assertEqual(resp.status_code, 400, resp.content)
1 change: 1 addition & 0 deletions main/urls.py
Original file line number Diff line number Diff line change
@@ -105,6 +105,7 @@
router.register(r'regional-project', deployment_views.RegionalProjectViewset)
router.register(r'project', deployment_views.ProjectViewset)
router.register(r'data-bank/country-overview', CountryOverviewViewSet)
router.register(r'region-project', deployment_views.RegionProjectViewset, base_name='region-project')


admin.site.site_header = 'IFRC Go administration'