Merge branch 'develop'

cookbook/admin.py

@@ -10,12 +10,13 @@
 
 from cookbook.managers import DICTIONARY
 
-from .models import (BookmarkletImport, Comment, CookLog, Food, FoodInheritField, ImportLog,
-                     Ingredient, InviteLink, Keyword, MealPlan, MealType, NutritionInformation,
-                     Recipe, RecipeBook, RecipeBookEntry, RecipeImport, SearchPreference, ShareLink,
-                     ShoppingList, ShoppingListEntry, ShoppingListRecipe, Space, Step, Storage,
-                     Supermarket, SupermarketCategory, SupermarketCategoryRelation, Sync, SyncLog,
-                     TelegramBot, Unit, UserFile, UserPreference, ViewLog, Automation, UserSpace)
+from .models import (Automation, BookmarkletImport, Comment, CookLog, Food, FoodInheritField,
+                     ImportLog, Ingredient, InviteLink, Keyword, MealPlan, MealType,
+                     NutritionInformation, Property, PropertyType, Recipe, RecipeBook,
+                     RecipeBookEntry, RecipeImport, SearchPreference, ShareLink, ShoppingList,
+                     ShoppingListEntry, ShoppingListRecipe, Space, Step, Storage, Supermarket,
+                     SupermarketCategory, SupermarketCategoryRelation, Sync, SyncLog, TelegramBot,
+                     Unit, UnitConversion, UserFile, UserPreference, UserSpace, ViewLog)
 
 
 class CustomUserAdmin(UserAdmin):
@@ -150,9 +151,16 @@ class KeywordAdmin(TreeAdmin):
 admin.site.register(Keyword, KeywordAdmin)
 
 
+@admin.action(description='Delete Steps not part of a Recipe.')
+def delete_unattached_steps(modeladmin, request, queryset):
+    with scopes_disabled():
+        Step.objects.filter(recipe=None).delete()
+
+
 class StepAdmin(admin.ModelAdmin):
     list_display = ('name', 'order',)
     search_fields = ('name',)
+    actions = [delete_unattached_steps]
 
 
 admin.site.register(Step, StepAdmin)
@@ -201,9 +209,24 @@ class FoodAdmin(TreeAdmin):
 admin.site.register(Food, FoodAdmin)
 
 
+class UnitConversionAdmin(admin.ModelAdmin):
+    list_display = ('base_amount', 'base_unit', 'food', 'converted_amount', 'converted_unit')
+    search_fields = ('food__name', 'unit__name')
+
+
+admin.site.register(UnitConversion, UnitConversionAdmin)
+
+
+@admin.action(description='Delete Ingredients not part of a Recipe.')
+def delete_unattached_ingredients(modeladmin, request, queryset):
+    with scopes_disabled():
+        Ingredient.objects.filter(step__recipe=None).delete()
+
+
 class IngredientAdmin(admin.ModelAdmin):
     list_display = ('food', 'amount', 'unit')
     search_fields = ('food__name', 'unit__name')
+    actions = [delete_unattached_ingredients]
 
 
 admin.site.register(Ingredient, IngredientAdmin)
@@ -319,6 +342,20 @@ class ShareLinkAdmin(admin.ModelAdmin):
 admin.site.register(ShareLink, ShareLinkAdmin)
 
 
+class PropertyTypeAdmin(admin.ModelAdmin):
+    list_display = ('id', 'name')
+
+
+admin.site.register(PropertyType, PropertyTypeAdmin)
+
+
+class PropertyAdmin(admin.ModelAdmin):
+    list_display = ('property_amount', 'property_type')
+
+
+admin.site.register(Property, PropertyAdmin)
+
+
 class NutritionInformationAdmin(admin.ModelAdmin):
     list_display = ('id',)
 

cookbook/helper/cache_helper.py

@@ -0,0 +1,11 @@
+class CacheHelper:
+    space = None
+
+    BASE_UNITS_CACHE_KEY = None
+    PROPERTY_TYPE_CACHE_KEY = None
+
+    def __init__(self, space):
+        self.space = space
+
+        self.BASE_UNITS_CACHE_KEY = f'SPACE_{space.id}_BASE_UNITS'
+        self.PROPERTY_TYPE_CACHE_KEY = f'SPACE_{space.id}_PROPERTY_TYPES'

cookbook/helper/image_processing.py

@@ -40,7 +40,12 @@ def get_filetype(name):
 # TODO also add env variable to define which images sizes should be compressed
 # filetype argument can not be optional, otherwise this function will treat all images as if they were a jpeg
 # Because it's no longer optional, no reason to return it
-def handle_image(request, image_object, filetype): 
+def handle_image(request, image_object, filetype):
+    try:
+        Image.open(image_object).verify()
+    except Exception:
+        return None
+
     if (image_object.size / 1000) > 500:  # if larger than 500 kb compress
         if filetype == '.jpeg' or filetype == '.jpg':
             return rescale_image_jpeg(image_object)

cookbook/helper/open_data_importer.py

@@ -0,0 +1,214 @@
+from django.db.models import Q
+
+from cookbook.models import Unit, SupermarketCategory, Property, PropertyType, Supermarket, SupermarketCategoryRelation, Food, Automation, UnitConversion, FoodProperty
+
+
+class OpenDataImporter:
+    request = None
+    data = {}
+    slug_id_cache = {}
+    update_existing = False
+    use_metric = True
+
+    def __init__(self, request, data, update_existing=False, use_metric=True):
+        self.request = request
+        self.data = data
+        self.update_existing = update_existing
+        self.use_metric = use_metric
+
+    def _update_slug_cache(self, object_class, datatype):
+        self.slug_id_cache[datatype] = dict(object_class.objects.filter(space=self.request.space, open_data_slug__isnull=False).values_list('open_data_slug', 'id', ))
+
+    def import_units(self):
+        datatype = 'unit'
+
+        insert_list = []
+        for u in list(self.data[datatype].keys()):
+            insert_list.append(Unit(
+                name=self.data[datatype][u]['name'],
+                plural_name=self.data[datatype][u]['plural_name'],
+                base_unit=self.data[datatype][u]['base_unit'] if self.data[datatype][u]['base_unit'] != '' else None,
+                open_data_slug=u,
+                space=self.request.space
+            ))
+
+        if self.update_existing:
+            return Unit.objects.bulk_create(insert_list, update_conflicts=True, update_fields=('name', 'plural_name', 'base_unit', 'open_data_slug'), unique_fields=('space', 'name',))
+        else:
+            return Unit.objects.bulk_create(insert_list, update_conflicts=True, update_fields=('open_data_slug',), unique_fields=('space', 'name',))
+
+    def import_category(self):
+        datatype = 'category'
+
+        insert_list = []
+        for k in list(self.data[datatype].keys()):
+            insert_list.append(SupermarketCategory(
+                name=self.data[datatype][k]['name'],
+                open_data_slug=k,
+                space=self.request.space
+            ))
+
+        return SupermarketCategory.objects.bulk_create(insert_list, update_conflicts=True, update_fields=('open_data_slug',), unique_fields=('space', 'name',))
+
+    def import_property(self):
+        datatype = 'property'
+
+        insert_list = []
+        for k in list(self.data[datatype].keys()):
+            insert_list.append(PropertyType(
+                name=self.data[datatype][k]['name'],
+                unit=self.data[datatype][k]['unit'],
+                open_data_slug=k,
+                space=self.request.space
+            ))
+
+        return PropertyType.objects.bulk_create(insert_list, update_conflicts=True, update_fields=('open_data_slug',), unique_fields=('space', 'name',))
+
+    def import_supermarket(self):
+        datatype = 'store'
+
+        self._update_slug_cache(SupermarketCategory, 'category')
+        insert_list = []
+        for k in list(self.data[datatype].keys()):
+            insert_list.append(Supermarket(
+                name=self.data[datatype][k]['name'],
+                open_data_slug=k,
+                space=self.request.space
+            ))
+
+        # always add open data slug if matching supermarket is found, otherwise relation might fail
+        supermarkets = Supermarket.objects.bulk_create(insert_list, unique_fields=('space', 'name',), update_conflicts=True, update_fields=('open_data_slug',))
+        self._update_slug_cache(Supermarket, 'store')
+
+        insert_list = []
+        for k in list(self.data[datatype].keys()):
+            relations = []
+            order = 0
+            for c in self.data[datatype][k]['categories']:
+                relations.append(
+                    SupermarketCategoryRelation(
+                        supermarket_id=self.slug_id_cache[datatype][k],
+                        category_id=self.slug_id_cache['category'][c],
+                        order=order,
+                    )
+                )
+                order += 1
+
+            SupermarketCategoryRelation.objects.bulk_create(relations, ignore_conflicts=True, unique_fields=('supermarket', 'category',))
+
+        return supermarkets
+
+    def import_food(self):
+        identifier_list = []
+        datatype = 'food'
+        for k in list(self.data[datatype].keys()):
+            identifier_list.append(self.data[datatype][k]['name'])
+            identifier_list.append(self.data[datatype][k]['plural_name'])
+
+        existing_objects_flat = []
+        existing_objects = {}
+        for f in Food.objects.filter(space=self.request.space).filter(name__in=identifier_list).values_list('id', 'name', 'plural_name'):
+            existing_objects_flat.append(f[1])
+            existing_objects_flat.append(f[2])
+            existing_objects[f[1]] = f
+            existing_objects[f[2]] = f
+
+        self._update_slug_cache(Unit, 'unit')
+        self._update_slug_cache(PropertyType, 'property')
+
+        # pref_unit_key = 'preferred_unit_metric'
+        # pref_shopping_unit_key = 'preferred_packaging_unit_metric'
+        # if not self.use_metric:
+        #     pref_unit_key = 'preferred_unit_imperial'
+        #     pref_shopping_unit_key = 'preferred_packaging_unit_imperial'
+
+        insert_list = []
+        update_list = []
+        update_field_list = []
+        for k in list(self.data[datatype].keys()):
+            if not (self.data[datatype][k]['name'] in existing_objects_flat or self.data[datatype][k]['plural_name'] in existing_objects_flat):
+                insert_list.append({'data': {
+                    'name': self.data[datatype][k]['name'],
+                    'plural_name': self.data[datatype][k]['plural_name'] if self.data[datatype][k]['plural_name'] != '' else None,
+                    # 'preferred_unit_id': self.slug_id_cache['unit'][self.data[datatype][k][pref_unit_key]],
+                    # 'preferred_shopping_unit_id': self.slug_id_cache['unit'][self.data[datatype][k][pref_shopping_unit_key]],
+                    'supermarket_category_id': self.slug_id_cache['category'][self.data[datatype][k]['store_category']],
+                    'fdc_id': self.data[datatype][k]['fdc_id'] if self.data[datatype][k]['fdc_id'] != '' else None,
+                    'open_data_slug': k,
+                    'space': self.request.space.id,
+                }})
+            else:
+                if self.data[datatype][k]['name'] in existing_objects:
+                    existing_food_id = existing_objects[self.data[datatype][k]['name']][0]
+                else:
+                    existing_food_id = existing_objects[self.data[datatype][k]['plural_name']][0]
+
+                if self.update_existing:
+                    update_field_list = ['name', 'plural_name', 'preferred_unit_id', 'preferred_shopping_unit_id', 'supermarket_category_id', 'fdc_id', 'open_data_slug', ]
+                    update_list.append(Food(
+                        id=existing_food_id,
+                        name=self.data[datatype][k]['name'],
+                        plural_name=self.data[datatype][k]['plural_name'] if self.data[datatype][k]['plural_name'] != '' else None,
+                        # preferred_unit_id=self.slug_id_cache['unit'][self.data[datatype][k][pref_unit_key]],
+                        # preferred_shopping_unit_id=self.slug_id_cache['unit'][self.data[datatype][k][pref_shopping_unit_key]],
+                        supermarket_category_id=self.slug_id_cache['category'][self.data[datatype][k]['store_category']],
+                        fdc_id=self.data[datatype][k]['fdc_id'] if self.data[datatype][k]['fdc_id'] != '' else None,
+                        open_data_slug=k,
+                    ))
+                else:
+                    update_field_list = ['open_data_slug', ]
+                    update_list.append(Food(id=existing_food_id, open_data_slug=k, ))
+
+        Food.load_bulk(insert_list, None)
+        if len(update_list) > 0:
+            Food.objects.bulk_update(update_list, update_field_list)
+
+        self._update_slug_cache(Food, 'food')
+
+        food_property_list = []
+        alias_list = []
+        for k in list(self.data[datatype].keys()):
+            for fp in self.data[datatype][k]['properties']['type_values']:
+                food_property_list.append(Property(
+                    property_type_id=self.slug_id_cache['property'][fp['property_type']],
+                    property_amount=fp['property_value'],
+                    import_food_id=self.slug_id_cache['food'][k],
+                    space=self.request.space,
+                ))
+
+            # for a in self.data[datatype][k]['alias']:
+            #     alias_list.append(Automation(
+            #         param_1=a,
+            #         param_2=self.data[datatype][k]['name'],
+            #         space=self.request.space,
+            #         created_by=self.request.user,
+            #     ))
+
+        Property.objects.bulk_create(food_property_list, ignore_conflicts=True, unique_fields=('space', 'import_food_id', 'property_type',))
+
+        property_food_relation_list = []
+        for p in Property.objects.filter(space=self.request.space, import_food_id__isnull=False).values_list('import_food_id', 'id', ):
+            property_food_relation_list.append(Food.properties.through(food_id=p[0], property_id=p[1]))
+
+        FoodProperty.objects.bulk_create(property_food_relation_list, ignore_conflicts=True, unique_fields=('food_id', 'property_id',))
+
+        # Automation.objects.bulk_create(alias_list, ignore_conflicts=True, unique_fields=('space', 'param_1', 'param_2',))
+        return insert_list + update_list
+
+    def import_conversion(self):
+        datatype = 'conversion'
+
+        insert_list = []
+        for k in list(self.data[datatype].keys()):
+            insert_list.append(UnitConversion(
+                base_amount=self.data[datatype][k]['base_amount'],
+                base_unit_id=self.slug_id_cache['unit'][self.data[datatype][k]['base_unit']],
+                converted_amount=self.data[datatype][k]['converted_amount'],
+                converted_unit_id=self.slug_id_cache['unit'][self.data[datatype][k]['converted_unit']],
+                food_id=self.slug_id_cache['food'][self.data[datatype][k]['food']],
+                open_data_slug=k,
+                space=self.request.space,
+                created_by=self.request.user,
+            ))
+
+        return UnitConversion.objects.bulk_create(insert_list, ignore_conflicts=True, unique_fields=('space', 'base_unit', 'converted_unit', 'food', 'open_data_slug'))

cookbook/helper/property_helper.py

@@ -0,0 +1,71 @@
+from django.core.cache import caches
+
+from cookbook.helper.cache_helper import CacheHelper
+from cookbook.helper.unit_conversion_helper import UnitConversionHelper
+from cookbook.models import PropertyType, Unit, Food, Property, Recipe, Step
+
+
+class FoodPropertyHelper:
+    space = None
+
+    def __init__(self, space):
+        """
+        Helper to perform food property calculations
+        :param space: space to limit scope to
+        """
+        self.space = space
+
+    def calculate_recipe_properties(self, recipe):
+        """
+        Calculate all food properties for a given recipe.
+        :param recipe: recipe to calculate properties for
+        :return: dict of with property keys and total/food values for each property available
+        """
+        ingredients = []
+        computed_properties = {}
+
+        for s in recipe.steps.all():
+            ingredients += s.ingredients.all()
+
+        property_types = caches['default'].get(CacheHelper(self.space).PROPERTY_TYPE_CACHE_KEY, None)
+
+        if not property_types:
+            property_types = PropertyType.objects.filter(space=self.space).all()
+            caches['default'].set(CacheHelper(self.space).PROPERTY_TYPE_CACHE_KEY, property_types, 60 * 60)  # cache is cleared on property type save signal so long duration is fine
+
+        for fpt in property_types:
+            computed_properties[fpt.id] = {'id': fpt.id, 'name': fpt.name, 'icon': fpt.icon, 'description': fpt.description, 'unit': fpt.unit, 'food_values': {}, 'total_value': 0, 'missing_value': False}
+
+        uch = UnitConversionHelper(self.space)
+
+        for i in ingredients:
+            if i.food is not None:
+                conversions = uch.get_conversions(i)
+                for pt in property_types:
+                    found_property = False
+                    if i.food.properties_food_amount == 0 or i.food.properties_food_unit is None:
+                        computed_properties[pt.id]['food_values'][i.food.id] = {'id': i.food.id, 'food': i.food.name, 'value': 0}
+                        computed_properties[pt.id]['missing_value'] = i.food.properties_food_unit is None
+                    else:
+                        for p in i.food.properties.all():
+                            if p.property_type == pt:
+                                for c in conversions:
+                                    if c.unit == i.food.properties_food_unit:
+                                        found_property = True
+                                        computed_properties[pt.id]['total_value'] += (c.amount / i.food.properties_food_amount) * p.property_amount
+                                        computed_properties[pt.id]['food_values'] = self.add_or_create(computed_properties[p.property_type.id]['food_values'], c.food.id, (c.amount / i.food.properties_food_amount) * p.property_amount, c.food)
+                    if not found_property:
+                        computed_properties[pt.id]['missing_value'] = True
+                        computed_properties[pt.id]['food_values'][i.food.id] = {'id': i.food.id, 'food': i.food.name, 'value': 0}
+
+        return computed_properties
+
+    # small dict helper to add to existing key or create new, probably a better way of doing this
+    # TODO move to central helper ?
+    @staticmethod
+    def add_or_create(d, key, value, food):
+        if key in d:
+            d[key]['value'] += value
+        else:
+            d[key] = {'id': food.id, 'food': food.name, 'value': value}
+        return d