Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Release/v1.2.0 #693

Merged
merged 24 commits into from
Apr 30, 2020
Merged
Changes from 1 commit
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
29d8d5e
Fetch data from FTS Google sheet
thenav56 Mar 17, 2020
da512c1
Log cronlog on error and success in Databank ingest
thenav56 Mar 19, 2020
1977c57
Add is_private field for Project
thenav56 Apr 1, 2020
fa2d310
Add Programme Type Domestic
thenav56 Apr 8, 2020
f6dda67
Add Project Bulk Import
thenav56 Apr 8, 2020
f520e59
Add reversion for Project
thenav56 Apr 8, 2020
7cb03ca
Private project only accessable to ifrc users.
thenav56 Apr 9, 2020
ae57909
Add dtype to Mini Event View
thenav56 Apr 9, 2020
1dab1fe
Replace is_private by visibility choices
thenav56 Apr 9, 2020
3d3bb66
Clean up data for project import
thenav56 Apr 14, 2020
d8dbcc4
Change Sector Health
thenav56 Apr 22, 2020
fae7ee6
Fix Public/Clinical Health Project Import Error
thenav56 Apr 28, 2020
ca1666b
Merge pull request #640 from IFRCGo/feature/databank-fts-google-sheet
GergiH Apr 28, 2020
588499a
Add Region Project Vizualization API
thenav56 Apr 21, 2020
153a596
Add Sankey API for project viz (region)
thenav56 Apr 24, 2020
52b9386
Add visibilty filter for Region 3W API
thenav56 Apr 27, 2020
2041f0a
Fix Project Test
thenav56 Apr 27, 2020
ba5d3b0
Add Secontary tag COVID
thenav56 Apr 29, 2020
4114d1e
Merge pull request #688 from IFRCGo/feature/region-project-viz-api
GergiH Apr 29, 2020
53d0503
Add Test For New Project API
thenav56 Apr 29, 2020
2f8c968
Merge pull request #691 from IFRCGo/feature/3W-few-changes
GergiH Apr 30, 2020
71b8c4f
object_name CharField -> TextField
GergiH Apr 30, 2020
86cb4d4
Merge pull request #692 from IFRCGo/fix/500-error
GergiH Apr 30, 2020
111f366
Update changelog for 3W Changes
thenav56 Apr 30, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Clean up data for project import
thenav56 committed Apr 28, 2020
commit 3d3bb66c348ac5520bb529b463c233c01b6368b6
69 changes: 46 additions & 23 deletions deployments/forms.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import io
import dateutil.parser
import traceback
import datetime
import csv

from django import forms
@@ -42,8 +42,19 @@ class ProjectImportForm(forms.Form):
file = forms.FileField(widget=forms.FileInput(attrs={'accept': '.csv'}))

def _handle_bulk_upload(self, user, file):
def strip_date(date):
return datetime.datetime.strptime(date, "%Y-%m-%d").date()
def key_clean(string):
return string.lower().strip()

def parse_date(date):
return dateutil.parser.parse(date)

def parse_integer(integer):
try:
if isinstance(integer, str):
return int(integer.replace(',', ''))
return integer
except ValueError:
return None

file.seek(0)
reader = csv.DictReader(io.StringIO(file.read().decode('utf-8', errors='ignore')), skipinitialspace=True)
@@ -58,38 +69,50 @@ def strip_date(date):

for row in reader:
# TODO: Try to do this in a single query
reporting_ns = Country.objects.filter(name__iexact=row['Supporting NS'].strip()).first()
project_district = District.objects.filter(
country__name__iexact=row['Country'].strip(),
name__iexact=row['Region'].strip(),
).first()
district_name = row['Region'].strip()
reporting_ns_name = row['Supporting NS'].strip()
country_name = row['Country'].strip()

reporting_ns = Country.objects.filter(name__iexact=reporting_ns_name).first()
if district_name.lower() in ['countrywide', '']:
project_country = Country.objects.filter(name__iexact=country_name).first()
project_district = None
else:
project_district = District.objects.filter(
country__name__iexact=country_name,
name__iexact=district_name,
).first()
project_country = project_district.country

projects.append(Project(
user=user,
reporting_ns=reporting_ns,
project_district=project_district,
project_country=project_country,

# Enum fields
operation_type=operation_types[row['Operation type'].lower().strip()],
programme_type=programme_types[row['Programme Type'].lower().strip()],
primary_sector=sectors[row['Primary Sector'].lower().strip()],
secondary_sectors=[sector_tags[tag.lower().strip()] for tag in row['Tags'].split(',')],
status=statuses[row['Status'].lower().strip()],
operation_type=operation_types[key_clean(row['Operation type'])],
programme_type=programme_types[key_clean(row['Programme Type'])],
primary_sector=sectors[key_clean(row['Primary Sector'])],
secondary_sectors=[
sector_tags[key_clean(tag)] for tag in row['Tags'].split(',') if key_clean(tag) in sector_tags
],
status=statuses[key_clean(row['Status'])],

name=row['Project Name'],
start_date=strip_date(row['Start Date']),
end_date=strip_date(row['End Date']),
start_date=parse_date(row['Start Date']),
end_date=parse_date(row['End Date']),
budget_amount=row['Budget(CHF)'],

# Optional fields
target_male=row['Targeted Males'],
target_female=row['Targeted Females'],
target_other=row['Targeted Others'],
target_total=row['Targeted Total'],
reached_male=row['Reached Males'],
reached_female=row['Reached Females'],
reached_other=row['Reached Others'],
reached_total=row['Reached Total'],
target_male=parse_integer(row['Targeted Males']),
target_female=parse_integer(row['Targeted Females']),
target_other=parse_integer(row['Targeted Others']),
target_total=parse_integer(row['Targeted Total']),
reached_male=parse_integer(row['Reached Males']),
reached_female=parse_integer(row['Reached Females']),
reached_other=parse_integer(row['Reached Others']),
reached_total=parse_integer(row['Reached Total']),
))
return Project.objects.bulk_create(projects)