From f504bddb85c96b3ef15b8bfe347f510acb9e80da Mon Sep 17 00:00:00 2001 From: Kasper Brandt Date: Thu, 17 Jul 2014 10:07:42 +0200 Subject: [PATCH 1/3] [#680] Hacky solutions for Cordaid script --- akvo/rsr/models.py | 6 +++++- akvo/scripts/cordaid/pre_import.py | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/akvo/rsr/models.py b/akvo/rsr/models.py index 0e80787d1b..d2a0fa4848 100644 --- a/akvo/rsr/models.py +++ b/akvo/rsr/models.py @@ -145,7 +145,11 @@ def save(self, *args, **kwargs): # Set location as primary location if it is the first location location_target = self.location_target - if location_target.primary_location is None or location_target.primary_location.pk > self.pk: + try: + if location_target.primary_location is None or location_target.primary_location.pk > self.pk: + location_target.primary_location = self + location_target.save() + except: location_target.primary_location = self location_target.save() diff --git a/akvo/scripts/cordaid/pre_import.py b/akvo/scripts/cordaid/pre_import.py index adc43fbb7c..ecec05864a 100644 --- a/akvo/scripts/cordaid/pre_import.py +++ b/akvo/scripts/cordaid/pre_import.py @@ -96,7 +96,10 @@ def create_cats_and_benches(business_units): log(u"Created bench: {id}, {name}", dict(id=new_bench.id, name=name)) else: log(u"Found existing bench: {id}, {name}", dict(id=new_bench.id, name=name)) - data['category'].benchmarknames.add(new_bench) + try: + data['category'].benchmarknames.add(new_bench) + except: + log(u"Did not find bench: {id}, {name}", dict(id=new_bench.id, name=name)) def import_cordaid_benchmarks(csv_file): From 1437207c618a925ed782af9b7258bb9991dc943d Mon Sep 17 00:00:00 2001 From: Gabriel von Heijne Date: Thu, 17 Jul 2014 17:22:42 +0200 Subject: [PATCH 2/3] [#680] Fix pre_import.py when data contains unknown business units Replace create_cordaid_business_units() with find_cordaid_business_units() turning the process inside-out so that we loop over the data and lookup the business units instead of the other way around. Change create_cats_and_benches() so it logs to file properly --- akvo/scripts/cordaid/__init__.py | 12 ++-- akvo/scripts/cordaid/pre_import.py | 108 ++++++++++++++++++----------- 2 files changed, 75 insertions(+), 45 deletions(-) diff --git a/akvo/scripts/cordaid/__init__.py b/akvo/scripts/cordaid/__init__.py index 48c7e27a37..7c153a6a98 100644 --- a/akvo/scripts/cordaid/__init__.py +++ b/akvo/scripts/cordaid/__init__.py @@ -24,11 +24,11 @@ AKVO_NS='{http://akvo.org/iati-activities}', CORDAID_ROOT_DIR = '/var/tmp/cordaid', - CORDAID_PROJECT_IMAGES_SUBDIR = 'project_images', - CORDAID_LOGOS_SUBDIR = 'logos', + CORDAID_PROJECT_IMAGES_SUBDIR = 'out_proj', + CORDAID_LOGOS_SUBDIR = 'out_rltn', - CORDAID_IATI_ACTIVITES_FILENAME = 'iati_export0.xml', - CORDAID_ORGANISATIONS_FILENAME = 'organisations.xml', + CORDAID_IATI_ACTIVITES_FILENAME = 'iati-activities.xml', + CORDAID_ORGANISATIONS_FILENAME = 'akvo-organizations.xml', CORDAID_INDICATORS_CSV_FILE_NAME = '20130711_indicators.csv', CORDAID_LOG_FILENAME = 'cordaid_import.log', CORDAID_ORG_CSV_FILENAME = 'cordaid_organisations_import_{datetime}.csv', @@ -127,6 +127,8 @@ def print_log(log_file, column_names, to_console=False): ACTION_CREATE_ORG = 'organisation created' ACTION_UPDATE_ORG = 'organisation updated' ACTION_PUBLISHING_SET = 'set publishing status' +ACTION_BENCH_CREATE = 'benchmarkname created' +ACTION_BENCH_FOUND = 'benchmarkname found' ERROR_COUNTRY_CODE = 'invalid country code' ERROR_MULTIPLE_OBJECTS = 'multiple objects' @@ -139,3 +141,5 @@ def print_log(log_file, column_names, to_console=False): ERROR_CREATE_ORG = 'organisation create error' ERROR_UPDATE_ORG = 'organisation update error' ERROR_UPLOAD_ORG = "organisation upload error" +ERROR_BUSINESS_UNIT_MISSING = "business unit organisation missing error" +ERROR_CATEGORY_MISSING = "category missing for business unit error" \ No newline at end of file diff --git a/akvo/scripts/cordaid/pre_import.py b/akvo/scripts/cordaid/pre_import.py index ecec05864a..f055cc8917 100644 --- a/akvo/scripts/cordaid/pre_import.py +++ b/akvo/scripts/cordaid/pre_import.py @@ -23,48 +23,51 @@ ) from akvo.utils import model_and_instance_based_filename, custom_get_or_create_country, who_am_i from akvo.scripts.cordaid import ( - CORDAID_ORG_ID, CORDAID_IATI_ID, DGIS_ORG_ID, DGIS_IATI_ID, CORDAID_INDICATORS_CSV, - CORDAID_LOGOS_DIR, CORDAID_ORGANISATIONS_XML, + CORDAID_ORG_ID, CORDAID_IATI_ID, DGIS_ORG_ID, DGIS_IATI_ID, CORDAID_LOGOS_DIR, CORDAID_ORGANISATIONS_XML, print_log, log, LOG_ORGANISATIONS, ACTION_FOUND, ERROR_MULTIPLE_OBJECTS, ACTION_LOCATION_SET, ERROR_COUNTRY_CODE, ACTION_CREATE_ORG, ERROR_EXCEPTION, ACTION_LOCATION_FOUND, ACTION_SET_IMAGE, - CORDAID_ORG_CSV_FILE, - init_log, - outsys, CORDAID_IATI_ACTIVITIES_XML, ACTION_UPDATE_ORG, ACTION_CREATE_IOI) - + CORDAID_ORG_CSV_FILE, init_log, outsys, CORDAID_IATI_ACTIVITIES_XML, ACTION_UPDATE_ORG, ACTION_CREATE_IOI, + ERROR_BUSINESS_UNIT_MISSING, ACTION_BENCH_FOUND, ACTION_BENCH_CREATE, ERROR_CATEGORY_MISSING +) -def create_cordaid_business_units(business_units): +def find_cordaid_business_units(business_units): outsys("\nRunning {}() ".format(who_am_i())) - business_units_info = [ - dict(pk=CORDAID_ORG_ID, internal_id="27239"), - dict(pk=959, internal_id="K6020", cat_name="Children & Education", fa="Education"), - dict(pk=962, internal_id="K6090", cat_name="Domestic", fa="Economic development"), - dict(pk=961, internal_id="K6030", cat_name="Disaster Recovery", fa="Economic development"), - dict(pk=950, internal_id="K6070", cat_name="Entrepreneurship", fa="Economic development"), - dict(pk=1099, internal_id="K6110", cat_name="Food Security", fa="Healthcare"), - dict(pk=953, internal_id="K6100", cat_name="Investments", fa="Economic development"), - dict(pk=949, internal_id="K6010", cat_name="Healthcare", fa="Healthcare"), - dict(pk=1241, internal_id="K6060", cat_name="Security & Justice", fa="Economic development"), - dict(pk=946, internal_id="K6080", cat_name="Urban Matters", fa="Economic development"), - dict(pk=955, internal_id="K6040", cat_name="Women's leadership", fa="Economic development"), - dict(pk=960, internal_id="K6050", cat_name="Extractives", fa="Economic development"), - ] + known_business_units = { + "27239": dict(pk=CORDAID_ORG_ID), + "K6020": dict(pk=959, cat_name="Children & Education", fa="Education"), + "K6090": dict(pk=962, cat_name="Domestic", fa="Economic development"), + "K6030": dict(pk=961, cat_name="Disaster Recovery", fa="Economic development"), + "K6070": dict(pk=950, cat_name="Entrepreneurship", fa="Economic development"), + "K6110": dict(pk=1099, cat_name="Food Security", fa="Healthcare"), + "K6100": dict(pk=953, cat_name="Investments", fa="Economic development"), + "K6010": dict(pk=949, cat_name="Healthcare", fa="Healthcare"), + "K6060": dict(pk=1241, cat_name="Security & Justice", fa="Economic development"), + "K6080": dict(pk=946, cat_name="Urban Matters", fa="Economic development"), + "K6040": dict(pk=955, cat_name="Women's leadership", fa="Economic development"), + "K6050": dict(pk=960, cat_name="Extractives", fa="Economic development"), + } cordaid = Organisation.objects.get(pk=CORDAID_ORG_ID) - for data in business_units_info: - outsys('.') - pk, identifier = data['pk'], data['internal_id'] - cat_name, fa_name = data.get('cat_name'), data.get('fa') + + for internal_id in business_units.keys(): + cbu = known_business_units.get(internal_id, {'pk': -1}) + pk, cat_name, fa_name = cbu['pk'], cbu.get('cat_name'), cbu.get('fa'), try: organisation = Organisation.objects.get(pk=pk) + outsys('.') except: + outsys('*') log( - u"No business unit with id {pk}, internal ID {identifier}", - dict(pk=pk, identifier=identifier), + u"No business unit with internal ID {internal_id}", + dict( + internal_id=internal_id, + event=ERROR_BUSINESS_UNIT_MISSING + ) ) continue internal_org, created = InternalOrganisationID.objects.get_or_create( recording_org=cordaid, referenced_org=organisation, - identifier= identifier + identifier= internal_id ) if cat_name: new_cat, created = Category.objects.get_or_create(name=cat_name) @@ -73,7 +76,7 @@ def create_cordaid_business_units(business_units): new_cat.focus_area.add(FocusArea.objects.get(name=fa_name)) else: log(u"Found existing cat: {id}, {cat_name}", dict(id=new_cat.id, cat_name=cat_name)) - business_units.setdefault(identifier, {'category': None, 'benchmarknames': []})['category'] = new_cat + business_units.setdefault(internal_id, {'category': None, 'benchmarknames': []})['category'] = new_cat cordaid.iati_org_id = CORDAID_IATI_ID cordaid.save() @@ -89,17 +92,40 @@ def create_cordaid_business_units(business_units): def create_cats_and_benches(business_units): outsys("\nRunning {}() ".format(who_am_i())) for internal_id, data in business_units.items(): - for name in data['benchmarknames']: - outsys('.') - new_bench, created = Benchmarkname.objects.get_or_create(name=name) - if created: - log(u"Created bench: {id}, {name}", dict(id=new_bench.id, name=name)) - else: - log(u"Found existing bench: {id}, {name}", dict(id=new_bench.id, name=name)) - try: - data['category'].benchmarknames.add(new_bench) - except: - log(u"Did not find bench: {id}, {name}", dict(id=new_bench.id, name=name)) + if data.get('category'): + for name in data['benchmarknames']: + outsys('.') + new_bench, created = Benchmarkname.objects.get_or_create(name=name) + if created: + log(u"Created bench: {pk}, {label}", + dict( + label=name, + pk=new_bench.id, + event=ACTION_BENCH_CREATE + ) + ) + else: + log(u"Found existing bench: {pk}, {label}", + dict( + label=name, + pk=new_bench.id, + event=ACTION_BENCH_FOUND + ) + ) + try: + data['category'].benchmarknames.add(new_bench) + except: + # we shouldn't end up here since we're testing for the existance of the category above + pass + else: + outsys('*') + log( + u"No category set of business unit with internal ID {internal_id}", + dict( + internal_id=internal_id, + event=ERROR_CATEGORY_MISSING + ) + ) def import_cordaid_benchmarks(csv_file): @@ -395,7 +421,7 @@ def organisation_logo(org_etree, internal_id, org): if __name__ == '__main__': #business_units = import_cordaid_benchmarks(CORDAID_INDICATORS_CSV) business_units = find_benchmarknames_and_BUs() - business_units = create_cordaid_business_units(business_units) + business_units = find_cordaid_business_units(business_units) create_cats_and_benches(business_units) import_orgs(CORDAID_ORGANISATIONS_XML) log_file = init_log(CORDAID_ORG_CSV_FILE) From efcebe552260587f4d0cd3128037fbd4cd303159 Mon Sep 17 00:00:00 2001 From: Gabriel von Heijne Date: Fri, 18 Jul 2014 12:15:59 +0200 Subject: [PATCH 3/3] [#680] Make sure Project.primary_location is none for updated projects When updating Cordaid projects the IATIProjectResource.obj_update method deletes all related objects so they can be re-created. This leads to Project.primary_location pointing to a deleted object so we have to set it to None since QuerySet.delete() does not call Model.delete() for the individual objects. (For reasons not understood a solution iterating over the ProjectLocation objects and calling delete() didn't work.) --- akvo/api/resources/project.py | 3 +++ akvo/rsr/models.py | 7 +------ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/akvo/api/resources/project.py b/akvo/api/resources/project.py index 5f7a1a3ecc..d96528fb20 100644 --- a/akvo/api/resources/project.py +++ b/akvo/api/resources/project.py @@ -103,6 +103,9 @@ def obj_update(self, bundle, skip_errors=False, **kwargs): Goal.objects.filter(project=bundle.obj).delete() BudgetItem.objects.filter(project=bundle.obj).delete() ProjectLocation.objects.filter(location_target=bundle.obj).delete() + # Since all locations for the project are deleted we need to make sure Project.primary_location is set to None + bundle.obj.primary_location = None + bundle.obj.save() Partnership.objects.filter(project=bundle.obj).delete() Benchmark.objects.filter(project=bundle.obj).delete() bundle.obj.categories.clear() diff --git a/akvo/rsr/models.py b/akvo/rsr/models.py index d2a0fa4848..48e03881e5 100644 --- a/akvo/rsr/models.py +++ b/akvo/rsr/models.py @@ -144,12 +144,7 @@ def save(self, *args, **kwargs): # Set location as primary location if it is the first location location_target = self.location_target - - try: - if location_target.primary_location is None or location_target.primary_location.pk > self.pk: - location_target.primary_location = self - location_target.save() - except: + if location_target.primary_location is None or location_target.primary_location.pk > self.pk: location_target.primary_location = self location_target.save()