diff --git a/cookbook/helper/open_data_importer.py b/cookbook/helper/open_data_importer.py index 13a2091a..5574b83b 100644 --- a/cookbook/helper/open_data_importer.py +++ b/cookbook/helper/open_data_importer.py @@ -1,105 +1,174 @@ from django.db.models import Q -from cookbook.models import Unit, SupermarketCategory, FoodProperty, FoodPropertyType, Supermarket, SupermarketCategoryRelation +from cookbook.models import Unit, SupermarketCategory, FoodProperty, FoodPropertyType, Supermarket, SupermarketCategoryRelation, Food, Automation -def import_units(data, request): - unit_name_list = [] - for u in list(data['unit'].keys()): - unit_name_list.append(data['unit'][u]['name']) - unit_name_list.append(data['unit'][u]['plural_name']) +class OpenDataImporter: + request = None + data = {} + slug_id_cache = {} - existing_units = Unit.objects.filter(space=request.space).filter(Q(name__in=unit_name_list) | Q(plural_name__in=unit_name_list)).values_list('name', 'plural_name') - existing_units = [item for sublist in existing_units for item in sublist] + def __init__(self, request, data): + self.request = request + self.data = data - insert_list = [] - for u in list(data['unit'].keys()): - if not (data['unit'][u]['name'] in existing_units or data['unit'][u]['plural_name'] in existing_units): - insert_list.append(Unit( - name=data['unit'][u]['name'], - plural_name=data['unit'][u]['plural_name'], - base_unit=data['unit'][u]['base_unit'] if data['unit'][u]['base_unit'] != '' else None, - open_data_slug=u, - space=request.space - )) + def _update_slug_cache(self, object_class, datatype): + self.slug_id_cache[datatype] = dict(object_class.objects.filter(space=self.request.space, open_data_slug__isnull=False).values_list('open_data_slug', 'id', )) - Unit.objects.bulk_create(insert_list) - return len(insert_list) + def import_units(self): + unit_name_list = [] + for u in list(self.data['unit'].keys()): + unit_name_list.append(self.data['unit'][u]['name']) + unit_name_list.append(self.data['unit'][u]['plural_name']) + existing_units = Unit.objects.filter(space=self.request.space).filter(Q(name__in=unit_name_list) | Q(plural_name__in=unit_name_list)).values_list('name', 'plural_name') + existing_units = [item for sublist in existing_units for item in sublist] -def import_category(data, request): - identifier_list = [] - datatype = 'category' - for k in list(data[datatype].keys()): - identifier_list.append(data[datatype][k]['name']) + insert_list = [] + for u in list(self.data['unit'].keys()): + if not (self.data['unit'][u]['name'] in existing_units or self.data['unit'][u]['plural_name'] in existing_units): + insert_list.append(Unit( + name=self.data['unit'][u]['name'], + plural_name=self.data['unit'][u]['plural_name'], + base_unit=self.data['unit'][u]['base_unit'] if self.data['unit'][u]['base_unit'] != '' else None, + open_data_slug=u, + space=self.request.space + )) - existing_objects = SupermarketCategory.objects.filter(space=request.space).filter(name__in=identifier_list).values_list('name', flat=True) + Unit.objects.bulk_create(insert_list) + return len(insert_list) - insert_list = [] - for k in list(data[datatype].keys()): - if not (data[datatype][k]['name'] in existing_objects): - insert_list.append(SupermarketCategory( - name=data[datatype][k]['name'], - open_data_slug=k, - space=request.space - )) + def import_category(self): + identifier_list = [] + datatype = 'category' + for k in list(self.data[datatype].keys()): + identifier_list.append(self.data[datatype][k]['name']) - SupermarketCategory.objects.bulk_create(insert_list) - return len(insert_list) + existing_objects = SupermarketCategory.objects.filter(space=self.request.space).filter(name__in=identifier_list).values_list('name', flat=True) + insert_list = [] + for k in list(self.data[datatype].keys()): + if not (self.data[datatype][k]['name'] in existing_objects): + insert_list.append(SupermarketCategory( + name=self.data[datatype][k]['name'], + open_data_slug=k, + space=self.request.space + )) -def import_property(data, request): - identifier_list = [] - datatype = 'property' - for k in list(data[datatype].keys()): - identifier_list.append(data[datatype][k]['name']) + SupermarketCategory.objects.bulk_create(insert_list) + return len(insert_list) - existing_objects = FoodPropertyType.objects.filter(space=request.space).filter(name__in=identifier_list).values_list('name', flat=True) + def import_property(self): + identifier_list = [] + datatype = 'property' + for k in list(self.data[datatype].keys()): + identifier_list.append(self.data[datatype][k]['name']) - insert_list = [] - for k in list(data[datatype].keys()): - if not (data[datatype][k]['name'] in existing_objects): - insert_list.append(FoodPropertyType( - name=data[datatype][k]['name'], - unit=data[datatype][k]['unit'], - open_data_slug=k, - space=request.space - )) + existing_objects = FoodPropertyType.objects.filter(space=self.request.space).filter(name__in=identifier_list).values_list('name', flat=True) - FoodPropertyType.objects.bulk_create(insert_list) - return len(insert_list) + insert_list = [] + for k in list(self.data[datatype].keys()): + if not (self.data[datatype][k]['name'] in existing_objects): + insert_list.append(FoodPropertyType( + name=self.data[datatype][k]['name'], + unit=self.data[datatype][k]['unit'], + open_data_slug=k, + space=self.request.space + )) + FoodPropertyType.objects.bulk_create(insert_list) + return len(insert_list) -def import_supermarket(data, request): - identifier_list = [] - datatype = 'supermarket' - for k in list(data[datatype].keys()): - identifier_list.append(data[datatype][k]['name']) + def import_supermarket(self): + identifier_list = [] + datatype = 'supermarket' + for k in list(self.data[datatype].keys()): + identifier_list.append(self.data[datatype][k]['name']) - existing_objects = Supermarket.objects.filter(space=request.space).filter(name__in=identifier_list).values_list('name', flat=True) + existing_objects = Supermarket.objects.filter(space=self.request.space).filter(name__in=identifier_list).values_list('name', flat=True) - supermarket_categories = SupermarketCategory.objects.filter(space=request.space, open_data_slug__isnull=False).values_list('id', 'open_data_slug') - insert_list = [] - for k in list(data[datatype].keys()): - if not (data[datatype][k]['name'] in existing_objects): # TODO on large datasets see if bulk creating supermarkets and then relations as well is better - supermarket = Supermarket.objects.create( - name=data[datatype][k]['name'], - open_data_slug=k, - space=request.space - ) + self._update_slug_cache(SupermarketCategory, 'category') - relations = [] - order = 0 - for c in data[datatype][k]['categories']: - relations.append( - SupermarketCategoryRelation( - supermarket=supermarket, - category_id=[x[0] for x in supermarket_categories if x[1] == c][0], - order=order, - ) + insert_list = [] + for k in list(self.data[datatype].keys()): + if not (self.data[datatype][k]['name'] in existing_objects): # TODO on large datasets see if bulk creating supermarkets and then relations as well is better + supermarket = Supermarket.objects.create( + name=self.data[datatype][k]['name'], + open_data_slug=k, + space=self.request.space ) - order += 1 - SupermarketCategoryRelation.objects.bulk_create(relations) + relations = [] + order = 0 + for c in self.data[datatype][k]['categories']: + relations.append( + SupermarketCategoryRelation( + supermarket=supermarket, + category_id=self.slug_id_cache['category'][c], + order=order, + ) + ) + order += 1 - return len(insert_list) + SupermarketCategoryRelation.objects.bulk_create(relations) + + return len(insert_list) + + def import_food(self, metric=True): + identifier_list = [] + datatype = 'food' + for k in list(self.data[datatype].keys()): + identifier_list.append(self.data[datatype][k]['name']) + identifier_list.append(self.data[datatype][k]['plural_name']) + + existing_objects = Food.objects.filter(space=self.request.space).filter(name__in=identifier_list).values_list('name', flat=True) + + self._update_slug_cache(Unit, 'unit') + self._update_slug_cache(FoodPropertyType, 'property') + + pref_unit_key = 'preferred_unit_metric' + pref_shopping_unit_key = 'preferred_packaging_unit_metric' + if not metric: + pref_unit_key = 'preferred_unit_imperial' + pref_shopping_unit_key = 'preferred_packaging_unit_imperial' + + insert_list = [] + + for k in list(self.data[datatype].keys()): + if not (self.data[datatype][k]['name'] in existing_objects): + insert_list.append(Food( + name=self.data[datatype][k]['name'], + plural_name=self.data[datatype][k]['plural_name'] if self.data[datatype][k]['plural_name'] != '' else None, + preferred_unit_id=self.slug_id_cache['unit'][self.data[datatype][k][pref_unit_key]], + preferred_shopping_unit_id=self.slug_id_cache['unit'][self.data[datatype][k][pref_shopping_unit_key]], + supermarket_category_id=self.slug_id_cache['category'][self.data[datatype][k]['supermarket_category']], + fdc_id=self.data[datatype][k]['fdc_id'] if self.data[datatype][k]['fdc_id'] != '' else None, + open_data_slug=k, + space=self.request.space, + )) + + Food.objects.bulk_create(insert_list) + + self._update_slug_cache(Food, 'food') + + food_property_list = [] + alias_list = [] + for k in list(self.data[datatype].keys()): + for fp in self.data[datatype][k]['properties']['type_values']: + food_property_list.append(FoodProperty( + food_id=self.slug_id_cache['food'][k], + food_amount=self.data[datatype][k]['properties']['food_amount'], + food_unit_id=self.slug_id_cache['unit'][self.data[datatype][k]['properties']['food_unit']], + property_type_id=self.slug_id_cache['property'][fp['property_type']], + property_amount=fp['property_value'], + )) + + for a in self.data[datatype][k]['alias']: + alias_list.append(Automation( + param_1=a, + param_2=self.data[datatype][k]['name'] + )) + + FoodProperty.objects.bulk_create(food_property_list) + Automation.objects.bulk_create(alias_list) + return len(insert_list) diff --git a/cookbook/migrations/0192_remove_foodpropertytype_fdc_id_and_more.py b/cookbook/migrations/0192_remove_foodpropertytype_fdc_id_and_more.py new file mode 100644 index 00000000..a64f1f6c --- /dev/null +++ b/cookbook/migrations/0192_remove_foodpropertytype_fdc_id_and_more.py @@ -0,0 +1,41 @@ +# Generated by Django 4.1.7 on 2023-05-01 10:41 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('cookbook', '0191_food_open_data_slug_foodpropertytype_fdc_id_and_more'), + ] + + operations = [ + migrations.RemoveField( + model_name='foodpropertytype', + name='fdc_id', + ), + migrations.RemoveField( + model_name='foodpropertytype', + name='preferred_shopping_unit', + ), + migrations.RemoveField( + model_name='foodpropertytype', + name='preferred_unit', + ), + migrations.AddField( + model_name='food', + name='fdc_id', + field=models.CharField(blank=True, default=None, max_length=128, null=True), + ), + migrations.AddField( + model_name='food', + name='preferred_shopping_unit', + field=models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='preferred_shopping_unit', to='cookbook.unit'), + ), + migrations.AddField( + model_name='food', + name='preferred_unit', + field=models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='preferred_unit', to='cookbook.unit'), + ), + ] diff --git a/cookbook/models.py b/cookbook/models.py index 001ea5f3..a3088afc 100644 --- a/cookbook/models.py +++ b/cookbook/models.py @@ -573,6 +573,10 @@ class Food(ExportModelOperationsMixin('food'), TreeModel, PermissionModelMixin): substitute_children = models.BooleanField(default=False) child_inherit_fields = models.ManyToManyField(FoodInheritField, blank=True, related_name='child_inherit') + preferred_unit = models.ForeignKey(Unit, on_delete=models.SET_NULL, null=True, blank=True, default=None, related_name='preferred_unit') + preferred_shopping_unit = models.ForeignKey(Unit, on_delete=models.SET_NULL, null=True, blank=True, default=None, related_name='preferred_shopping_unit') + fdc_id = models.CharField(max_length=128, null=True, blank=True, default=None) + open_data_slug = models.CharField(max_length=128, null=True, blank=True, default=None) space = models.ForeignKey(Space, on_delete=models.CASCADE) objects = ScopedManager(space='space', _manager_class=TreeManager) @@ -755,9 +759,6 @@ class FoodPropertyType(models.Model, PermissionModelMixin): icon = models.CharField(max_length=16, blank=True, null=True) description = models.CharField(max_length=512, blank=True, null=True) category = models.CharField(max_length=64, choices=((NUTRITION, _('Nutrition')), (ALLERGEN, _('Allergen')), (PRICE, _('Price')), (GOAL, _('Goal')), (OTHER, _('Other'))), null=True, blank=True) - preferred_unit = models.ForeignKey(Unit, on_delete=models.SET_NULL, null=True, blank=True, default=None, related_name='preferred_unit') - preferred_shopping_unit = models.ForeignKey(Unit, on_delete=models.SET_NULL, null=True, blank=True, default=None, related_name='preferred_shopping_unit') - fdc_id = models.CharField(max_length=128, null=True, blank=True, default=None) open_data_slug = models.CharField(max_length=128, null=True, blank=True, default=None) # TODO show if empty property? diff --git a/cookbook/views/api.py b/cookbook/views/api.py index 8d95cd28..294be97a 100644 --- a/cookbook/views/api.py +++ b/cookbook/views/api.py @@ -54,7 +54,7 @@ from cookbook.helper import recipe_url_import as helper from cookbook.helper.HelperFunctions import str2bool from cookbook.helper.image_processing import handle_image from cookbook.helper.ingredient_parser import IngredientParser -from cookbook.helper.open_data_importer import import_units, import_category, import_property, import_supermarket +from cookbook.helper.open_data_importer import OpenDataImporter from cookbook.helper.permission_helper import (CustomIsAdmin, CustomIsOwner, CustomIsOwnerReadOnly, CustomIsShared, CustomIsSpaceOwner, CustomIsUser, group_required, @@ -1438,13 +1438,13 @@ class ImportOpenData(APIView): response = requests.get(f'https://raw.githubusercontent.com/TandoorRecipes/open-tandoor-data/main/build/{selected_version}.json') # TODO catch 404, timeout, ... data = json.loads(response.content) - import_units(data, request) - import_category(data, request) - import_property(data, request) - import_supermarket(data, request) + data_importer = OpenDataImporter(request, data) + data_importer.import_units() + data_importer.import_category() + data_importer.import_property() + data_importer.import_supermarket() + data_importer.import_food() # TODO pass metric parameter - # TODO hardcode insert order? - # TODO split into update/create lists with multiple parameters per datatype return Response({ 'test': '' })