diff --git a/cookbook/forms.py b/cookbook/forms.py index 18a67d6d..872bfa01 100644 --- a/cookbook/forms.py +++ b/cookbook/forms.py @@ -10,7 +10,7 @@ from django_scopes import scopes_disabled from django_scopes.forms import SafeModelChoiceField, SafeModelMultipleChoiceField from hcaptcha.fields import hCaptchaField -from .models import Comment, Food, InviteLink, Keyword, Recipe, RecipeBook, RecipeBookEntry, SearchPreference, Space, Storage, Sync, User, UserPreference, ConnectorConfig +from .models import Comment, InviteLink, Keyword, Recipe, SearchPreference, Space, Storage, Sync, User, UserPreference, ConnectorConfig class SelectWidget(widgets.Select): diff --git a/cookbook/helper/open_data_importer.py b/cookbook/helper/open_data_importer.py index e709f2f5..a6fad976 100644 --- a/cookbook/helper/open_data_importer.py +++ b/cookbook/helper/open_data_importer.py @@ -1,5 +1,20 @@ +import traceback +from collections import defaultdict +from decimal import Decimal + from cookbook.models import (Food, FoodProperty, Property, PropertyType, Supermarket, SupermarketCategory, SupermarketCategoryRelation, Unit, UnitConversion) +import re + + +class OpenDataImportResponse: + total_created = 0 + total_updated = 0 + total_untouched = 0 + total_errored = 0 + + def to_dict(self): + return {'total_created': self.total_created, 'total_updated': self.total_updated, 'total_untouched': self.total_untouched, 'total_errored': self.total_errored} class OpenDataImporter: @@ -18,69 +33,269 @@ class OpenDataImporter: def _update_slug_cache(self, object_class, datatype): self.slug_id_cache[datatype] = dict(object_class.objects.filter(space=self.request.space, open_data_slug__isnull=False).values_list('open_data_slug', 'id', )) - def import_units(self): - datatype = 'unit' + @staticmethod + def _is_obj_identical(field_list, obj, existing_obj): + """ + checks if the obj meant for import is identical to an already existing one + :param field_list: list of field names to check + :type field_list: list[str] + :param obj: object meant for import + :type obj: Object + :param existing_obj: object already in DB + :type existing_obj: Object + :return: if objects are identical + :rtype: bool + """ + for field in field_list: + if isinstance(getattr(obj, field), float) or isinstance(getattr(obj, field), Decimal): + if abs(float(getattr(obj, field)) - float(existing_obj[field])) > 0.001: # convert both to float and check if basically equal + print(f'comparing FLOAT {obj} failed because field {field} is not equal ({getattr(obj, field)} != {existing_obj[field]})') + return False + elif getattr(obj, field) != existing_obj[field]: + print(f'comparing {obj} failed because field {field} is not equal ({getattr(obj, field)} != {existing_obj[field]})') + return False + return True + + @staticmethod + def _merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names): + """ + sometimes there might be two objects conflicting for open data import (one has the slug, the other the name) + this function checks if that is the case and merges the two objects if possible + :param model_type: type of model to check/merge + :type model_type: Model + :param obj: object that should be created/updated + :type obj: Model + :param existing_data_slugs: dict of open data slugs mapped to objects + :type existing_data_slugs: dict + :param existing_data_names: dict of names mapped to objects + :type existing_data_names: dict + :return: true if merge was successful or not necessary else false + :rtype: bool + """ + if obj.open_data_slug in existing_data_slugs and obj.name in existing_data_names and existing_data_slugs[obj.open_data_slug]['pk'] != existing_data_names[obj.name]['pk']: + try: + source_obj = model_type.objects.get(pk=existing_data_slugs[obj.open_data_slug]['pk']) + del existing_data_slugs[obj.open_data_slug] + source_obj.merge_into(model_type.objects.get(pk=existing_data_names[obj.name]['pk'])) + return True + except RuntimeError: + return False # in the edge case (e.g. parent/child) that an object cannot be merged don't update it for now + else: + return True + + @staticmethod + def _get_existing_obj(obj, existing_data_slugs, existing_data_names): + """ + gets the existing object from slug or name cache + :param obj: object that should be found + :type obj: Model + :param existing_data_slugs: dict of open data slugs mapped to objects + :type existing_data_slugs: dict + :param existing_data_names: dict of names mapped to objects + :type existing_data_names: dict + :return: existing object + :rtype: dict + """ + existing_obj = None + if obj.open_data_slug in existing_data_slugs: + existing_obj = existing_data_slugs[obj.open_data_slug] + elif obj.name in existing_data_names: + existing_obj = existing_data_names[obj.name] + + return existing_obj + + def import_units(self): + od_response = OpenDataImportResponse() + datatype = 'unit' + model_type = Unit + field_list = ['name', 'plural_name', 'base_unit', 'open_data_slug'] + + existing_data_slugs = {} + existing_data_names = {} + for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list): + existing_data_slugs[obj['open_data_slug']] = obj + existing_data_names[obj['name']] = obj + + update_list = [] + create_list = [] - insert_list = [] for u in list(self.data[datatype].keys()): - insert_list.append(Unit( + obj = model_type( name=self.data[datatype][u]['name'], plural_name=self.data[datatype][u]['plural_name'], - base_unit=self.data[datatype][u]['base_unit'] if self.data[datatype][u]['base_unit'] != '' else None, + base_unit=self.data[datatype][u]['base_unit'].lower() if self.data[datatype][u]['base_unit'] != '' else None, open_data_slug=u, space=self.request.space - )) + ) + if obj.open_data_slug in existing_data_slugs or obj.name in existing_data_names: + if not self._merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names): + od_response.total_errored += 1 + continue # if conflicting objects exist and cannot be merged skip object - if self.update_existing: - return Unit.objects.bulk_create(insert_list, update_conflicts=True, update_fields=( - 'name', 'plural_name', 'base_unit', 'open_data_slug'), unique_fields=('space', 'name',)) - else: - return Unit.objects.bulk_create(insert_list, update_conflicts=True, update_fields=('open_data_slug',), unique_fields=('space', 'name',)) + existing_obj = self._get_existing_obj(obj, existing_data_slugs, existing_data_names) + + if not self._is_obj_identical(field_list, obj, existing_obj): + obj.pk = existing_obj['pk'] + update_list.append(obj) + else: + od_response.total_untouched += 1 + else: + create_list.append(obj) + + if self.update_existing and len(update_list) > 0: + model_type.objects.bulk_update(update_list, field_list) + od_response.total_updated += len(update_list) + + if len(create_list) > 0: + model_type.objects.bulk_create(create_list, update_conflicts=True, update_fields=field_list, unique_fields=('space', 'name',)) + od_response.total_created += len(create_list) + + return od_response def import_category(self): + od_response = OpenDataImportResponse() datatype = 'category' + model_type = SupermarketCategory + field_list = ['name', 'open_data_slug'] + + existing_data_slugs = {} + existing_data_names = {} + for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list): + existing_data_slugs[obj['open_data_slug']] = obj + existing_data_names[obj['name']] = obj + + update_list = [] + create_list = [] - insert_list = [] for k in list(self.data[datatype].keys()): - insert_list.append(SupermarketCategory( + obj = model_type( name=self.data[datatype][k]['name'], open_data_slug=k, space=self.request.space - )) + ) - return SupermarketCategory.objects.bulk_create(insert_list, update_conflicts=True, update_fields=('open_data_slug',), unique_fields=('space', 'name',)) + if obj.open_data_slug in existing_data_slugs or obj.name in existing_data_names: + if not self._merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names): + od_response.total_errored += 1 + continue # if conflicting objects exist and cannot be merged skip object + + existing_obj = self._get_existing_obj(obj, existing_data_slugs, existing_data_names) + + if not self._is_obj_identical(field_list, obj, existing_obj): + obj.pk = existing_obj['pk'] + update_list.append(obj) + else: + od_response.total_untouched += 1 + else: + create_list.append(obj) + + if self.update_existing and len(update_list) > 0: + model_type.objects.bulk_update(update_list, field_list) + od_response.total_updated += len(update_list) + + if len(create_list) > 0: + model_type.objects.bulk_create(create_list, update_conflicts=True, update_fields=field_list, unique_fields=('space', 'name',)) + od_response.total_created += len(create_list) + + return od_response def import_property(self): + od_response = OpenDataImportResponse() datatype = 'property' + model_type = PropertyType + field_list = ['name', 'unit', 'fdc_id', 'open_data_slug'] + + existing_data_slugs = {} + existing_data_names = {} + for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list): + existing_data_slugs[obj['open_data_slug']] = obj + existing_data_names[obj['name']] = obj + + update_list = [] + create_list = [] - insert_list = [] for k in list(self.data[datatype].keys()): - insert_list.append(PropertyType( + obj = model_type( name=self.data[datatype][k]['name'], unit=self.data[datatype][k]['unit'], + fdc_id=self.data[datatype][k]['fdc_id'], open_data_slug=k, space=self.request.space - )) + ) - return PropertyType.objects.bulk_create(insert_list, update_conflicts=True, update_fields=('open_data_slug',), unique_fields=('space', 'name',)) + if obj.open_data_slug in existing_data_slugs or obj.name in existing_data_names: + if not self._merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names): + od_response.total_errored += 1 + continue # if conflicting objects exist and cannot be merged skip object + + existing_obj = self._get_existing_obj(obj, existing_data_slugs, existing_data_names) + + if not self._is_obj_identical(field_list, obj, existing_obj): + obj.pk = existing_obj['pk'] + update_list.append(obj) + else: + od_response.total_untouched += 1 + else: + create_list.append(obj) + + if self.update_existing and len(update_list) > 0: + model_type.objects.bulk_update(update_list, field_list) + od_response.total_updated += len(update_list) + + if len(create_list) > 0: + model_type.objects.bulk_create(create_list, update_conflicts=True, update_fields=field_list, unique_fields=('space', 'name',)) + od_response.total_created += len(create_list) + + return od_response def import_supermarket(self): + od_response = OpenDataImportResponse() datatype = 'store' + model_type = Supermarket + field_list = ['name', 'open_data_slug'] + + existing_data_slugs = {} + existing_data_names = {} + for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list): + existing_data_slugs[obj['open_data_slug']] = obj + existing_data_names[obj['name']] = obj + + update_list = [] + create_list = [] self._update_slug_cache(SupermarketCategory, 'category') - insert_list = [] for k in list(self.data[datatype].keys()): - insert_list.append(Supermarket( + obj = model_type( name=self.data[datatype][k]['name'], open_data_slug=k, space=self.request.space - )) + ) + if obj.open_data_slug in existing_data_slugs or obj.name in existing_data_names: + if not self._merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names): + od_response.total_errored += 1 + continue # if conflicting objects exist and cannot be merged skip object + + existing_obj = self._get_existing_obj(obj, existing_data_slugs, existing_data_names) + + if not self._is_obj_identical(field_list, obj, existing_obj): + obj.pk = existing_obj['pk'] + update_list.append(obj) + else: + od_response.total_untouched += 1 + else: + create_list.append(obj) + + if self.update_existing and len(update_list) > 0: + model_type.objects.bulk_update(update_list, field_list) + od_response.total_updated += len(update_list) + + if len(create_list) > 0: + model_type.objects.bulk_create(create_list, update_conflicts=True, update_fields=field_list, unique_fields=('space', 'name',)) + od_response.total_created += len(create_list) # always add open data slug if matching supermarket is found, otherwise relation might fail - supermarkets = Supermarket.objects.bulk_create(insert_list, unique_fields=('space', 'name',), update_conflicts=True, update_fields=('open_data_slug',)) self._update_slug_cache(Supermarket, 'store') - insert_list = [] for k in list(self.data[datatype].keys()): relations = [] order = 0 @@ -96,115 +311,186 @@ class OpenDataImporter: SupermarketCategoryRelation.objects.bulk_create(relations, ignore_conflicts=True, unique_fields=('supermarket', 'category',)) - return supermarkets + return od_response def import_food(self): - identifier_list = [] + od_response = OpenDataImportResponse() datatype = 'food' - for k in list(self.data[datatype].keys()): - identifier_list.append(self.data[datatype][k]['name']) - identifier_list.append(self.data[datatype][k]['plural_name']) + model_type = Food + field_list = ['name', 'open_data_slug'] - existing_objects_flat = [] - existing_objects = {} - for f in Food.objects.filter(space=self.request.space).filter(name__in=identifier_list).values_list('id', 'name', 'plural_name'): - existing_objects_flat.append(f[1]) - existing_objects_flat.append(f[2]) - existing_objects[f[1]] = f - existing_objects[f[2]] = f + existing_data_slugs = {} + existing_data_names = {} + for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list): + existing_data_slugs[obj['open_data_slug']] = obj + existing_data_names[obj['name']] = obj + + update_list = [] + create_list = [] self._update_slug_cache(Unit, 'unit') self._update_slug_cache(PropertyType, 'property') + self._update_slug_cache(SupermarketCategory, 'category') + + unit_g = Unit.objects.filter(space=self.request.space, base_unit__iexact='g').first() - insert_list = [] - insert_list_flat = [] - update_list = [] - update_field_list = [] for k in list(self.data[datatype].keys()): - if not (self.data[datatype][k]['name'] in existing_objects_flat or self.data[datatype][k]['plural_name'] in existing_objects_flat): - if not (self.data[datatype][k]['name'] in insert_list_flat or self.data[datatype][k]['plural_name'] in insert_list_flat): - insert_list.append({'data': { - 'name': self.data[datatype][k]['name'], - 'plural_name': self.data[datatype][k]['plural_name'] if self.data[datatype][k]['plural_name'] != '' else None, - 'supermarket_category_id': self.slug_id_cache['category'][self.data[datatype][k]['store_category']], - 'fdc_id': self.data[datatype][k]['fdc_id'] if self.data[datatype][k]['fdc_id'] != '' else None, - 'open_data_slug': k, - 'space': self.request.space.id, - }}) - # build a fake second flat array to prevent duplicate foods from being inserted. - # trying to insert a duplicate would throw a db error :( - insert_list_flat.append(self.data[datatype][k]['name']) - insert_list_flat.append(self.data[datatype][k]['plural_name']) + + obj_dict = { + 'name': self.data[datatype][k]['name'], + 'plural_name': self.data[datatype][k]['plural_name'] if self.data[datatype][k]['plural_name'] != '' else None, + 'supermarket_category_id': self.slug_id_cache['category'][self.data[datatype][k]['store_category']], + 'fdc_id': re.sub(r'\D', '', self.data[datatype][k]['fdc_id']) if self.data[datatype][k]['fdc_id'] != '' else None, + 'open_data_slug': k, + 'properties_food_unit_id': None, + 'space_id': self.request.space.id, + } + + if unit_g: + obj_dict['properties_food_unit_id'] = unit_g.id + + obj = model_type(**obj_dict) + + if obj.open_data_slug in existing_data_slugs or obj.name in existing_data_names: + if not self._merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names): + od_response.total_errored += 1 + continue # if conflicting objects exist and cannot be merged skip object + + existing_obj = self._get_existing_obj(obj, existing_data_slugs, existing_data_names) + + if not self._is_obj_identical(field_list, obj, existing_obj): + obj.pk = existing_obj['pk'] + update_list.append(obj) + else: + od_response.total_untouched += 1 else: - if self.data[datatype][k]['name'] in existing_objects: - existing_food_id = existing_objects[self.data[datatype][k]['name']][0] - else: - existing_food_id = existing_objects[self.data[datatype][k]['plural_name']][0] + create_list.append({'data': obj_dict}) - if self.update_existing: - update_field_list = ['name', 'plural_name', 'preferred_unit_id', 'preferred_shopping_unit_id', 'supermarket_category_id', 'fdc_id', 'open_data_slug', ] - update_list.append(Food( - id=existing_food_id, - name=self.data[datatype][k]['name'], - plural_name=self.data[datatype][k]['plural_name'] if self.data[datatype][k]['plural_name'] != '' else None, - supermarket_category_id=self.slug_id_cache['category'][self.data[datatype][k]['store_category']], - fdc_id=self.data[datatype][k]['fdc_id'] if self.data[datatype][k]['fdc_id'] != '' else None, - open_data_slug=k, - )) - else: - update_field_list = ['open_data_slug', ] - update_list.append(Food(id=existing_food_id, open_data_slug=k, )) + if self.update_existing and len(update_list) > 0: + model_type.objects.bulk_update(update_list, field_list) + od_response.total_updated += len(update_list) - Food.load_bulk(insert_list, None) - if len(update_list) > 0: - Food.objects.bulk_update(update_list, update_field_list) + if len(create_list) > 0: + Food.load_bulk(create_list, None) + od_response.total_created += len(create_list) + + # --------------- PROPERTY STUFF ----------------------- + model_type = Property + field_list = ['property_type_id', 'property_amount', 'open_data_food_slug'] + + existing_data_slugs = {} + existing_data_property_types = {} + for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list): + existing_data_slugs[obj['open_data_food_slug']] = obj + existing_data_property_types[obj['property_type_id']] = obj + + update_list = [] + create_list = [] self._update_slug_cache(Food, 'food') - food_property_list = [] - # alias_list = [] + for k in list(self.data['food'].keys()): + for fp in self.data['food'][k]['properties']['type_values']: + obj = model_type( + property_type_id=self.slug_id_cache['property'][fp['property_type']], + property_amount=fp['property_value'], + open_data_food_slug=k, + space=self.request.space, + ) - for k in list(self.data[datatype].keys()): - for fp in self.data[datatype][k]['properties']['type_values']: - # try catch here because somettimes key "k" is not set for he food cache - try: - food_property_list.append(Property( - property_type_id=self.slug_id_cache['property'][fp['property_type']], - property_amount=fp['property_value'], - import_food_id=self.slug_id_cache['food'][k], - space=self.request.space, - )) - except KeyError: - print(str(k) + ' is not in self.slug_id_cache["food"]') + if obj.open_data_food_slug in existing_data_slugs and obj.property_type_id in existing_data_property_types and existing_data_slugs[obj.open_data_food_slug] == existing_data_property_types[obj.property_type_id]: + existing_obj = existing_data_slugs[obj.open_data_food_slug] - Property.objects.bulk_create(food_property_list, ignore_conflicts=True, unique_fields=('space', 'import_food_id', 'property_type',)) + if not self._is_obj_identical(field_list, obj, existing_obj): + obj.pk = existing_obj['pk'] + update_list.append(obj) + else: + create_list.append(obj) + + if self.update_existing and len(update_list) > 0: + model_type.objects.bulk_update(update_list, field_list) + + if len(create_list) > 0: + model_type.objects.bulk_create(create_list, ignore_conflicts=True, unique_fields=('space', 'open_data_food_slug', 'property_type',)) + + linked_properties = list(FoodProperty.objects.filter(food__space=self.request.space).values_list('property_id', flat=True).all()) property_food_relation_list = [] - for p in Property.objects.filter(space=self.request.space, import_food_id__isnull=False).values_list('import_food_id', 'id', ): - property_food_relation_list.append(Food.properties.through(food_id=p[0], property_id=p[1])) + for p in model_type.objects.filter(space=self.request.space, open_data_food_slug__isnull=False).values_list('open_data_food_slug', 'id', ): + if p[1] == 147: + pass + # slug_id_cache should always exist, don't create relations for already linked properties (ignore_conflicts would do that as well but this is more performant) + if p[0] in self.slug_id_cache['food'] and p[1] not in linked_properties: + property_food_relation_list.append(Food.properties.through(food_id=self.slug_id_cache['food'][p[0]], property_id=p[1])) - FoodProperty.objects.bulk_create(property_food_relation_list, ignore_conflicts=True, unique_fields=('food_id', 'property_id',)) + FoodProperty.objects.bulk_create(property_food_relation_list, unique_fields=('food_id', 'property_id',)) - return insert_list + update_list + return od_response def import_conversion(self): + od_response = OpenDataImportResponse() datatype = 'conversion' + model_type = UnitConversion + field_list = ['base_amount', 'base_unit_id', 'converted_amount', 'converted_unit_id', 'food_id', 'open_data_slug'] + + self._update_slug_cache(Food, 'food') + self._update_slug_cache(Unit, 'unit') + + existing_data_slugs = {} + existing_data_foods = defaultdict(list) + for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list): + existing_data_slugs[obj['open_data_slug']] = obj + existing_data_foods[obj['food_id']].append(obj) + + update_list = [] + create_list = [] - insert_list = [] for k in list(self.data[datatype].keys()): - # try catch here because sometimes key "k" is not set for he food cache + # try catch here because sometimes key "k" is not set for the food cache try: - insert_list.append(UnitConversion( - base_amount=self.data[datatype][k]['base_amount'], + obj = model_type( + base_amount=Decimal(self.data[datatype][k]['base_amount']), base_unit_id=self.slug_id_cache['unit'][self.data[datatype][k]['base_unit']], - converted_amount=self.data[datatype][k]['converted_amount'], + converted_amount=Decimal(self.data[datatype][k]['converted_amount']), converted_unit_id=self.slug_id_cache['unit'][self.data[datatype][k]['converted_unit']], food_id=self.slug_id_cache['food'][self.data[datatype][k]['food']], open_data_slug=k, space=self.request.space, - created_by=self.request.user, - )) - except KeyError: - print(str(k) + ' is not in self.slug_id_cache["food"]') + created_by_id=self.request.user.id, + ) - return UnitConversion.objects.bulk_create(insert_list, ignore_conflicts=True, unique_fields=('space', 'base_unit', 'converted_unit', 'food', 'open_data_slug')) + if obj.open_data_slug in existing_data_slugs: + existing_obj = existing_data_slugs[obj.open_data_slug] + if not self._is_obj_identical(field_list, obj, existing_obj): + obj.pk = existing_obj['pk'] + update_list.append(obj) + else: + od_response.total_untouched += 1 + else: + matching_existing_found = False + if obj.food_id in existing_data_foods: + for edf in existing_data_foods[obj.food_id]: + if obj.base_unit_id == edf['base_unit_id'] and obj.converted_unit_id == edf['converted_unit_id']: + matching_existing_found = True + if not self._is_obj_identical(field_list, obj, edf): + obj.pk = edf['pk'] + update_list.append(obj) + else: + od_response.total_untouched += 1 + if not matching_existing_found: + create_list.append(obj) + except KeyError as e: + traceback.print_exc() + od_response.total_errored += 1 + print(self.data[datatype][k]['food'] + ' is not in self.slug_id_cache["food"]') + + if self.update_existing and len(update_list) > 0: + od_response.total_updated = model_type.objects.bulk_update(update_list, field_list) + od_response.total_errored += len(update_list) - od_response.total_updated + + if len(create_list) > 0: + objs_created = model_type.objects.bulk_create(create_list, ignore_conflicts=True, unique_fields=('space', 'base_unit', 'converted_unit', 'food', 'open_data_slug')) + od_response.total_created = len(objs_created) + od_response.total_errored += len(create_list) - od_response.total_created + + return od_response diff --git a/cookbook/helper/property_helper.py b/cookbook/helper/property_helper.py index 04521c65..3cc2f2ec 100644 --- a/cookbook/helper/property_helper.py +++ b/cookbook/helper/property_helper.py @@ -45,12 +45,12 @@ class FoodPropertyHelper: conversions = uch.get_conversions(i) for pt in property_types: found_property = False - if i.food.properties_food_amount == 0 or i.food.properties_food_unit is None: - computed_properties[pt.id]['food_values'][i.food.id] = {'id': i.food.id, 'food': i.food.name, 'value': 0} - computed_properties[pt.id]['missing_value'] = i.food.properties_food_unit is None + if i.food.properties_food_amount == 0 or i.food.properties_food_unit is None: # if food is configured incorrectly + computed_properties[pt.id]['food_values'][i.food.id] = {'id': i.food.id, 'food': i.food.name, 'value': None} + computed_properties[pt.id]['missing_value'] = True else: for p in i.food.properties.all(): - if p.property_type == pt: + if p.property_type == pt and p.property_amount is not None: for c in conversions: if c.unit == i.food.properties_food_unit: found_property = True @@ -58,13 +58,17 @@ class FoodPropertyHelper: computed_properties[pt.id]['food_values'] = self.add_or_create( computed_properties[p.property_type.id]['food_values'], c.food.id, (c.amount / i.food.properties_food_amount) * p.property_amount, c.food) if not found_property: - computed_properties[pt.id]['missing_value'] = True - computed_properties[pt.id]['food_values'][i.food.id] = {'id': i.food.id, 'food': i.food.name, 'value': 0} + if i.amount == 0: # don't count ingredients without an amount as missing + computed_properties[pt.id]['missing_value'] = computed_properties[pt.id]['missing_value'] or False # don't override if another food was already missing + computed_properties[pt.id]['food_values'][i.food.id] = {'id': i.food.id, 'food': i.food.name, 'value': 0} + else: + computed_properties[pt.id]['missing_value'] = True + computed_properties[pt.id]['food_values'][i.food.id] = {'id': i.food.id, 'food': i.food.name, 'value': None} return computed_properties # small dict helper to add to existing key or create new, probably a better way of doing this - # TODO move to central helper ? + # TODO move to central helper ? --> use defaultdict @staticmethod def add_or_create(d, key, value, food): if key in d: diff --git a/cookbook/helper/recipe_url_import.py b/cookbook/helper/recipe_url_import.py index 416bb54a..aa0bb2d8 100644 --- a/cookbook/helper/recipe_url_import.py +++ b/cookbook/helper/recipe_url_import.py @@ -231,7 +231,7 @@ def get_recipe_properties(space, property_data): 'id': pt.id, 'name': pt.name, }, - 'property_amount': parse_servings(property_data[properties[p]]) / float(property_data['servingSize']), + 'property_amount': parse_servings(property_data[properties[p]]) / parse_servings(property_data['servingSize']), }) return recipe_properties diff --git a/cookbook/integration/rezeptsuitede.py b/cookbook/integration/rezeptsuitede.py index afe3e543..3df81424 100644 --- a/cookbook/integration/rezeptsuitede.py +++ b/cookbook/integration/rezeptsuitede.py @@ -1,6 +1,6 @@ import base64 from io import BytesIO -from xml import etree +from lxml import etree from cookbook.helper.ingredient_parser import IngredientParser from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text @@ -53,7 +53,10 @@ class Rezeptsuitede(Integration): u = ingredient_parser.get_unit(ingredient.attrib['unit']) amount = 0 if ingredient.attrib['qty'].strip() != '': - amount, unit, note = ingredient_parser.parse_amount(ingredient.attrib['qty']) + try: + amount, unit, note = ingredient_parser.parse_amount(ingredient.attrib['qty']) + except ValueError: # sometimes quantities contain words which cant be parsed + pass ingredient_step.ingredients.add(Ingredient.objects.create(food=f, unit=u, amount=amount, space=self.request.space, )) try: diff --git a/cookbook/locale/it/LC_MESSAGES/django.po b/cookbook/locale/it/LC_MESSAGES/django.po index 6750519c..3dca22c1 100644 --- a/cookbook/locale/it/LC_MESSAGES/django.po +++ b/cookbook/locale/it/LC_MESSAGES/django.po @@ -12,8 +12,8 @@ msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2023-05-18 14:28+0200\n" -"PO-Revision-Date: 2024-02-01 17:22+0000\n" -"Last-Translator: Lorenzo \n" +"PO-Revision-Date: 2024-02-17 19:16+0000\n" +"Last-Translator: Andrea \n" "Language-Team: Italian \n" "Language: it\n" @@ -2093,7 +2093,7 @@ msgstr "Proprietario" #, fuzzy #| msgid "Create Space" msgid "Leave Space" -msgstr "Crea Istanza" +msgstr "Lascia Istanza" #: .\cookbook\templates\space_overview.html:78 #: .\cookbook\templates\space_overview.html:88 diff --git a/cookbook/management/commands/fix_duplicate_properties.py b/cookbook/management/commands/fix_duplicate_properties.py new file mode 100644 index 00000000..6f38ffc7 --- /dev/null +++ b/cookbook/management/commands/fix_duplicate_properties.py @@ -0,0 +1,34 @@ +from django.conf import settings +from django.contrib.postgres.search import SearchVector +from django.core.management.base import BaseCommand +from django.db.models import Count +from django.utils import translation +from django.utils.translation import gettext_lazy as _ +from django_scopes import scopes_disabled + +from cookbook.managers import DICTIONARY +from cookbook.models import Recipe, Step, FoodProperty, Food + + +# can be executed at the command line with 'python manage.py rebuildindex' +class Command(BaseCommand): + help = _('Fixes foods with ') + + def add_arguments(self, parser): + parser.add_argument('-d', '--dry-run', help='does not delete properties but instead prints them', action='store_true') + + def handle(self, *args, **options): + with scopes_disabled(): + foods_with_duplicate_properties = Food.objects.annotate(property_type_count=Count('foodproperty__property__property_type') - Count('foodproperty__property__property_type', distinct=True)).filter(property_type_count__gt=0).all() + for f in foods_with_duplicate_properties: + found_property_types = [] + for fp in f.properties.all(): + if fp.property_type.id in found_property_types: + if options['dry_run']: + print(f'Property id {fp.id} duplicate type {fp.property_type}({fp.property_type.id}) for food {f}({f.id})') + else: + print(f'DELETING property id {fp.id} duplicate type {fp.property_type}({fp.property_type.id}) for food {f}({f.id})') + fp.delete() + + else: + found_property_types.append(fp.property_type.id) diff --git a/cookbook/migrations/0211_recipebook_order.py b/cookbook/migrations/0211_recipebook_order.py new file mode 100644 index 00000000..c6044a0b --- /dev/null +++ b/cookbook/migrations/0211_recipebook_order.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.7 on 2024-02-16 19:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('cookbook', '0210_shoppinglistentry_updated_at'), + ] + + operations = [ + migrations.AddField( + model_name='recipebook', + name='order', + field=models.IntegerField(default=0), + ), + ] diff --git a/cookbook/migrations/0212_alter_property_property_amount.py b/cookbook/migrations/0212_alter_property_property_amount.py new file mode 100644 index 00000000..2ec04c47 --- /dev/null +++ b/cookbook/migrations/0212_alter_property_property_amount.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.7 on 2024-02-18 07:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('cookbook', '0211_recipebook_order'), + ] + + operations = [ + migrations.AlterField( + model_name='property', + name='property_amount', + field=models.DecimalField(decimal_places=4, default=None, max_digits=32, null=True), + ), + ] diff --git a/cookbook/migrations/0213_remove_property_property_unique_import_food_per_space_and_more.py b/cookbook/migrations/0213_remove_property_property_unique_import_food_per_space_and_more.py new file mode 100644 index 00000000..bc22b22e --- /dev/null +++ b/cookbook/migrations/0213_remove_property_property_unique_import_food_per_space_and_more.py @@ -0,0 +1,54 @@ +# Generated by Django 4.2.10 on 2024-02-19 13:48 + +from django.db import migrations, models +from django_scopes import scopes_disabled + + +def migrate_property_import_slug(apps, schema_editor): + with scopes_disabled(): + Property = apps.get_model('cookbook', 'Property') + Food = apps.get_model('cookbook', 'Food') + + id_slug_mapping = {} + with scopes_disabled(): + for f in Food.objects.filter(open_data_slug__isnull=False).values('id', 'open_data_slug').all(): + id_slug_mapping[f['id']] = f['open_data_slug'] + + property_update_list = [] + + for p in Property.objects.filter().values('id', 'import_food_id').all(): + if p['import_food_id'] in id_slug_mapping: + property_update_list.append(Property( + id=p['id'], + open_data_food_slug=id_slug_mapping[p['import_food_id']] + )) + + Property.objects.bulk_update(property_update_list, ('open_data_food_slug',)) + + +class Migration(migrations.Migration): + dependencies = [ + ('cookbook', '0212_alter_property_property_amount'), + ] + + operations = [ + migrations.AddField( + model_name='property', + name='open_data_food_slug', + field=models.CharField(blank=True, default=None, max_length=128, null=True), + ), + migrations.RunPython(migrate_property_import_slug), + migrations.RemoveConstraint( + model_name='property', + name='property_unique_import_food_per_space', + ), + migrations.RemoveField( + model_name='property', + name='import_food_id', + ), + + migrations.AddConstraint( + model_name='property', + constraint=models.UniqueConstraint(fields=('space', 'property_type', 'open_data_food_slug'), name='property_unique_import_food_per_space'), + ), + ] diff --git a/cookbook/models.py b/cookbook/models.py index 0c332a19..8493b231 100644 --- a/cookbook/models.py +++ b/cookbook/models.py @@ -209,6 +209,27 @@ class TreeModel(MP_Node): abstract = True +class MergeModelMixin: + + def merge_into(self, target): + """ + very simple merge function that replaces the current instance with the target instance + :param target: target object + :return: target with data merged + """ + + if self == target: + raise ValueError('Cannot merge an object with itself') + + if getattr(self, 'space', 0) != getattr(target, 'space', 0): + raise RuntimeError('Cannot merge objects from different spaces') + + if hasattr(self, 'get_descendants_and_self') and target in callable(getattr(self, 'get_descendants_and_self')): + raise RuntimeError('Cannot merge parent (source) with child (target) object') + + # TODO copy field values + + class PermissionModelMixin: @staticmethod def get_space_key(): @@ -320,10 +341,18 @@ class Space(ExportModelOperationsMixin('space'), models.Model): BookmarkletImport.objects.filter(space=self).delete() CustomFilter.objects.filter(space=self).delete() + Property.objects.filter(space=self).delete() + PropertyType.objects.filter(space=self).delete() + Comment.objects.filter(recipe__space=self).delete() - Keyword.objects.filter(space=self).delete() Ingredient.objects.filter(space=self).delete() - Food.objects.filter(space=self).delete() + Keyword.objects.filter(space=self).delete() + + # delete food in batches because treabeard might fail to delete otherwise + while Food.objects.filter(space=self).count() > 0: + pks = Food.objects.filter(space=self).values_list('pk')[:200] + Food.objects.filter(pk__in=pks).delete() + Unit.objects.filter(space=self).delete() Step.objects.filter(space=self).delete() NutritionInformation.objects.filter(space=self).delete() @@ -348,9 +377,11 @@ class Space(ExportModelOperationsMixin('space'), models.Model): SupermarketCategory.objects.filter(space=self).delete() Supermarket.objects.filter(space=self).delete() - InviteLink.objects.filter(space=self).delete() UserFile.objects.filter(space=self).delete() + UserSpace.objects.filter(space=self).delete() Automation.objects.filter(space=self).delete() + InviteLink.objects.filter(space=self).delete() + TelegramBot.objects.filter(space=self).delete() self.delete() def get_owner(self): @@ -468,6 +499,7 @@ class UserPreference(models.Model, PermissionModelMixin): self.use_fractions = FRACTION_PREF_DEFAULT return super().save(*args, **kwargs) + def __str__(self): return str(self.user) @@ -527,7 +559,7 @@ class Sync(models.Model, PermissionModelMixin): return self.path -class SupermarketCategory(models.Model, PermissionModelMixin): +class SupermarketCategory(models.Model, PermissionModelMixin, MergeModelMixin): name = models.CharField(max_length=128, validators=[MinLengthValidator(1)]) description = models.TextField(blank=True, null=True) open_data_slug = models.CharField(max_length=128, null=True, blank=True, default=None) @@ -538,6 +570,14 @@ class SupermarketCategory(models.Model, PermissionModelMixin): def __str__(self): return self.name + def merge_into(self, target): + super().merge_into(target) + + Food.objects.filter(supermarket_category=self).update(supermarket_category=target) + SupermarketCategoryRelation.objects.filter(category=self).update(category=target) + self.delete() + return target + class Meta: constraints = [ models.UniqueConstraint(fields=['space', 'name'], name='smc_unique_name_per_space'), @@ -612,7 +652,7 @@ class Keyword(ExportModelOperationsMixin('keyword'), TreeModel, PermissionModelM indexes = (Index(fields=['id', 'name']),) -class Unit(ExportModelOperationsMixin('unit'), models.Model, PermissionModelMixin): +class Unit(ExportModelOperationsMixin('unit'), models.Model, PermissionModelMixin, MergeModelMixin): name = models.CharField(max_length=128, validators=[MinLengthValidator(1)]) plural_name = models.CharField(max_length=128, null=True, blank=True, default=None) description = models.TextField(blank=True, null=True) @@ -622,6 +662,17 @@ class Unit(ExportModelOperationsMixin('unit'), models.Model, PermissionModelMixi space = models.ForeignKey(Space, on_delete=models.CASCADE) objects = ScopedManager(space='space') + def merge_into(self, target): + super().merge_into(target) + + Ingredient.objects.filter(unit=self).update(unit=target) + ShoppingListEntry.objects.filter(unit=self).update(unit=target) + Food.objects.filter(properties_food_unit=self).update(properties_food_unit=target) + Food.objects.filter(preferred_unit=self).update(preferred_unit=target) + Food.objects.filter(preferred_shopping_unit=self).update(preferred_shopping_unit=target) + self.delete() + return target + def __str__(self): return self.name @@ -670,6 +721,32 @@ class Food(ExportModelOperationsMixin('food'), TreeModel, PermissionModelMixin): def __str__(self): return self.name + def merge_into(self, target): + """ + very simple merge function that replaces the current food with the target food + also replaces a few attributes on the target field if they were empty before + :param target: target food object + :return: target with data merged + """ + if self == target: + raise ValueError('Cannot merge an object with itself') + + if self.space != target.space: + raise RuntimeError('Cannot merge objects from different spaces') + + try: + if target in self.get_descendants_and_self(): + raise RuntimeError('Cannot merge parent (source) with child (target) object') + except AttributeError: + pass # AttributeError is raised when the object is not a tree and thus does not have the get_descendants_and_self() function + + self.properties.all().delete() + self.properties.clear() + Ingredient.objects.filter(food=self).update(food=target) + ShoppingListEntry.objects.filter(food=self).update(food=target) + self.delete() + return target + def delete(self): if self.ingredient_set.all().exclude(step=None).count() > 0: raise ProtectedError(self.name + _(" is part of a recipe step and cannot be deleted"), self.ingredient_set.all().exclude(step=None)) @@ -827,7 +904,7 @@ class Step(ExportModelOperationsMixin('step'), models.Model, PermissionModelMixi indexes = (GinIndex(fields=["search_vector"]),) -class PropertyType(models.Model, PermissionModelMixin): +class PropertyType(models.Model, PermissionModelMixin, MergeModelMixin): NUTRITION = 'NUTRITION' ALLERGEN = 'ALLERGEN' PRICE = 'PRICE' @@ -852,6 +929,13 @@ class PropertyType(models.Model, PermissionModelMixin): def __str__(self): return f'{self.name}' + def merge_into(self, target): + super().merge_into(target) + + Property.objects.filter(property_type=self).update(property_type=target) + self.delete() + return target + class Meta: constraints = [ models.UniqueConstraint(fields=['space', 'name'], name='property_type_unique_name_per_space'), @@ -861,10 +945,10 @@ class PropertyType(models.Model, PermissionModelMixin): class Property(models.Model, PermissionModelMixin): - property_amount = models.DecimalField(default=0, decimal_places=4, max_digits=32) + property_amount = models.DecimalField(default=None, null=True, decimal_places=4, max_digits=32) property_type = models.ForeignKey(PropertyType, on_delete=models.PROTECT) - import_food_id = models.IntegerField(null=True, blank=True) # field to hold food id when importing properties from the open data project + open_data_food_slug = models.CharField(max_length=128, null=True, blank=True, default=None) # field to hold food id when importing properties from the open data project space = models.ForeignKey(Space, on_delete=models.CASCADE) objects = ScopedManager(space='space') @@ -874,7 +958,7 @@ class Property(models.Model, PermissionModelMixin): class Meta: constraints = [ - models.UniqueConstraint(fields=['space', 'property_type', 'import_food_id'], name='property_unique_import_food_per_space') + models.UniqueConstraint(fields=['space', 'property_type', 'open_data_food_slug'], name='property_unique_import_food_per_space') ] @@ -1011,7 +1095,6 @@ class RecipeBook(ExportModelOperationsMixin('book'), models.Model, PermissionMod filter = models.ForeignKey('cookbook.CustomFilter', null=True, blank=True, on_delete=models.SET_NULL) order = models.IntegerField(default=0) - space = models.ForeignKey(Space, on_delete=models.CASCADE) objects = ScopedManager(space='space') diff --git a/cookbook/schemas.py b/cookbook/schemas.py index e465de45..504ea971 100644 --- a/cookbook/schemas.py +++ b/cookbook/schemas.py @@ -44,12 +44,12 @@ class TreeSchema(AutoSchema): "name": 'root', "in": "query", "required": False, "description": 'Return first level children of {obj} with ID [int]. Integer 0 will return root {obj}s.'.format( obj=api_name), - 'schema': {'type': 'int', }, + 'schema': {'type': 'integer', }, }) parameters.append({ "name": 'tree', "in": "query", "required": False, "description": 'Return all self and children of {} with ID [int].'.format(api_name), - 'schema': {'type': 'int', }, + 'schema': {'type': 'integer', }, }) return parameters diff --git a/cookbook/serializer.py b/cookbook/serializer.py index f3a6532b..7eb6b800 100644 --- a/cookbook/serializer.py +++ b/cookbook/serializer.py @@ -584,7 +584,7 @@ class PropertyTypeSerializer(OpenDataModelMixin, WritableNestedModelSerializer, class PropertySerializer(UniqueFieldsMixin, WritableNestedModelSerializer): property_type = PropertyTypeSerializer() - property_amount = CustomDecimalField() + property_amount = CustomDecimalField(allow_null=True) def create(self, validated_data): validated_data['space'] = self.context['request'].space diff --git a/cookbook/templates/space_manage.html b/cookbook/templates/space_manage.html index 52db3f4e..daa82aa4 100644 --- a/cookbook/templates/space_manage.html +++ b/cookbook/templates/space_manage.html @@ -4,7 +4,7 @@ {% load i18n %} {% load l10n %} -{% block title %}{% trans 'Search' %}{% endblock %} +{% block title %}{% trans 'Space Management' %}{% endblock %} {% block content %} diff --git a/cookbook/templatetags/custom_tags.py b/cookbook/templatetags/custom_tags.py index 45985510..bacfbc7e 100644 --- a/cookbook/templatetags/custom_tags.py +++ b/cookbook/templatetags/custom_tags.py @@ -117,6 +117,7 @@ def page_help(page_name): 'edit_connector_config': 'https://docs.tandoor.dev/features/connectors/', 'view_shopping': 'https://docs.tandoor.dev/features/shopping/', 'view_import': 'https://docs.tandoor.dev/features/import_export/', + 'data_import_url': 'https://docs.tandoor.dev/features/import_export/', 'view_export': 'https://docs.tandoor.dev/features/import_export/', 'list_automation': 'https://docs.tandoor.dev/features/automation/', } diff --git a/cookbook/tests/other/test_food_property.py b/cookbook/tests/other/test_food_property.py index 534dcaf3..caeceae3 100644 --- a/cookbook/tests/other/test_food_property.py +++ b/cookbook/tests/other/test_food_property.py @@ -67,7 +67,7 @@ def test_food_property(space_1, space_2, u1_s1): assert property_values[property_fat.id]['name'] == property_fat.name assert abs(property_values[property_fat.id]['total_value']) < 0.0001 - assert abs(property_values[property_fat.id]['food_values'][food_1.id]['value']) < 0.0001 + assert property_values[property_fat.id]['food_values'][food_1.id]['value'] is None print('\n----------- TEST PROPERTY - PROPERTY CALCULATION UNIT CONVERSION ---------------') uc1 = UnitConversion.objects.create( diff --git a/cookbook/views/api.py b/cookbook/views/api.py index 2720df72..ea2a3da5 100644 --- a/cookbook/views/api.py +++ b/cookbook/views/api.py @@ -262,7 +262,9 @@ class MergeMixin(ViewSetMixin): try: if isinstance(source, Food): - source.properties.remove() + source.properties.all().delete() + source.properties.clear() + UnitConversion.objects.filter(food=source).delete() for link in [field for field in source._meta.get_fields() if issubclass(type(field), ForeignObjectRel)]: linkManager = getattr(source, link.get_accessor_name()) @@ -564,11 +566,8 @@ class FoodViewSet(viewsets.ModelViewSet, TreeMixin): shared_users = c else: try: - shared_users = [x.id for x in list(self.request.user.get_shopping_share())] + [ - self.request.user.id] - caches['default'].set( - f'shopping_shared_users_{self.request.space.id}_{self.request.user.id}', - shared_users, timeout=5 * 60) + shared_users = [x.id for x in list(self.request.user.get_shopping_share())] + [self.request.user.id] + caches['default'].set(f'shopping_shared_users_{self.request.space.id}_{self.request.user.id}', shared_users, timeout=5 * 60) # TODO ugly hack that improves API performance significantly, should be done properly except AttributeError: # Anonymous users (using share links) don't have shared users pass @@ -616,11 +615,21 @@ class FoodViewSet(viewsets.ModelViewSet, TreeMixin): if properties with a fdc_id already exist they will be overridden, if existing properties don't have a fdc_id they won't be changed """ food = self.get_object() + if not food.fdc_id: + return JsonResponse({'msg': 'Food has no FDC ID associated.'}, status=400, + json_dumps_params={'indent': 4}) response = requests.get(f'https://api.nal.usda.gov/fdc/v1/food/{food.fdc_id}?api_key={FDC_API_KEY}') if response.status_code == 429: - return JsonResponse({'msg', 'API Key Rate Limit reached/exceeded, see https://api.data.gov/docs/rate-limits/ for more information. Configure your key in Tandoor using environment FDC_API_KEY variable.'}, status=429, + return JsonResponse({'msg': 'API Key Rate Limit reached/exceeded, see https://api.data.gov/docs/rate-limits/ for more information. Configure your key in Tandoor using environment FDC_API_KEY variable.'}, status=429, json_dumps_params={'indent': 4}) + if response.status_code != 200: + return JsonResponse({'msg': f'Error while requesting FDC data using url https://api.nal.usda.gov/fdc/v1/food/{food.fdc_id}?api_key=****'}, status=response.status_code, + json_dumps_params={'indent': 4}) + + food.properties_food_amount = 100 + food.properties_food_unit = Unit.objects.get_or_create(base_unit__iexact='g', space=self.request.space, defaults={'name': 'g', 'base_unit': 'g', 'space': self.request.space})[0] + food.save() try: data = json.loads(response.content) @@ -634,23 +643,29 @@ class FoodViewSet(viewsets.ModelViewSet, TreeMixin): for pt in PropertyType.objects.filter(space=request.space, fdc_id__gte=0).all(): if pt.fdc_id: + property_found = False for fn in data['foodNutrients']: if fn['nutrient']['id'] == pt.fdc_id: + property_found = True food_property_list.append(Property( property_type_id=pt.id, - property_amount=round(fn['amount'], 2), - import_food_id=food.id, + property_amount=max(0, round(fn['amount'], 2)), # sometimes FDC might return negative values which make no sense, set to 0 space=self.request.space, )) + if not property_found: + food_property_list.append(Property( + property_type_id=pt.id, + property_amount=0, # if field not in FDC data the food does not have that property + space=self.request.space, + )) - Property.objects.bulk_create(food_property_list, ignore_conflicts=True, unique_fields=('space', 'import_food_id', 'property_type',)) + properties = Property.objects.bulk_create(food_property_list, unique_fields=('space', 'property_type',)) property_food_relation_list = [] - for p in Property.objects.filter(space=self.request.space, import_food_id=food.id).values_list('import_food_id', 'id', ): - property_food_relation_list.append(Food.properties.through(food_id=p[0], property_id=p[1])) + for p in properties: + property_food_relation_list.append(Food.properties.through(food_id=food.id, property_id=p.pk)) FoodProperty.objects.bulk_create(property_food_relation_list, ignore_conflicts=True, unique_fields=('food_id', 'property_id',)) - Property.objects.filter(space=self.request.space, import_food_id=food.id).update(import_food_id=None) return self.retrieve(request, pk) except Exception: @@ -680,7 +695,7 @@ class RecipeBookViewSet(viewsets.ModelViewSet, StandardFilterMixin): ordering = f"{'' if order_direction == 'asc' else '-'}{order_field}" self.queryset = self.queryset.filter(Q(created_by=self.request.user) | Q(shared=self.request.user)).filter( - space=self.request.space).distinct().order_by(ordering) + space=self.request.space).distinct().order_by(ordering) return super().get_queryset() @@ -728,7 +743,7 @@ class MealPlanViewSet(viewsets.ModelViewSet): query_params = [ QueryParam(name='from_date', description=_('Filter meal plans from date (inclusive) in the format of YYYY-MM-DD.'), qtype='string'), QueryParam(name='to_date', description=_('Filter meal plans to date (inclusive) in the format of YYYY-MM-DD.'), qtype='string'), - QueryParam(name='meal_type', description=_('Filter meal plans with MealType ID. For multiple repeat parameter.'), qtype='int'), + QueryParam(name='meal_type', description=_('Filter meal plans with MealType ID. For multiple repeat parameter.'), qtype='integer'), ] schema = QueryParamAutoSchema() @@ -858,7 +873,7 @@ class StepViewSet(viewsets.ModelViewSet): permission_classes = [CustomIsUser & CustomTokenHasReadWriteScope] pagination_class = DefaultPagination query_params = [ - QueryParam(name='recipe', description=_('ID of recipe a step is part of. For multiple repeat parameter.'), qtype='int'), + QueryParam(name='recipe', description=_('ID of recipe a step is part of. For multiple repeat parameter.'), qtype='integer'), QueryParam(name='query', description=_('Query string matched (fuzzy) against object name.'), qtype='string'), ] schema = QueryParamAutoSchema() @@ -901,27 +916,27 @@ class RecipeViewSet(viewsets.ModelViewSet): query_params = [ QueryParam(name='query', description=_('Query string matched (fuzzy) against recipe name. In the future also fulltext search.')), - QueryParam(name='keywords', description=_('ID of keyword a recipe should have. For multiple repeat parameter. Equivalent to keywords_or'), qtype='int'), - QueryParam(name='keywords_or', description=_('Keyword IDs, repeat for multiple. Return recipes with any of the keywords'), qtype='int'), - QueryParam(name='keywords_and', description=_('Keyword IDs, repeat for multiple. Return recipes with all of the keywords.'), qtype='int'), - QueryParam(name='keywords_or_not', description=_('Keyword IDs, repeat for multiple. Exclude recipes with any of the keywords.'), qtype='int'), - QueryParam(name='keywords_and_not', description=_('Keyword IDs, repeat for multiple. Exclude recipes with all of the keywords.'), qtype='int'), - QueryParam(name='foods', description=_('ID of food a recipe should have. For multiple repeat parameter.'), qtype='int'), - QueryParam(name='foods_or', description=_('Food IDs, repeat for multiple. Return recipes with any of the foods'), qtype='int'), - QueryParam(name='foods_and', description=_('Food IDs, repeat for multiple. Return recipes with all of the foods.'), qtype='int'), - QueryParam(name='foods_or_not', description=_('Food IDs, repeat for multiple. Exclude recipes with any of the foods.'), qtype='int'), - QueryParam(name='foods_and_not', description=_('Food IDs, repeat for multiple. Exclude recipes with all of the foods.'), qtype='int'), - QueryParam(name='units', description=_('ID of unit a recipe should have.'), qtype='int'), - QueryParam(name='rating', description=_('Rating a recipe should have or greater. [0 - 5] Negative value filters rating less than.'), qtype='int'), + QueryParam(name='keywords', description=_('ID of keyword a recipe should have. For multiple repeat parameter. Equivalent to keywords_or'), qtype='integer'), + QueryParam(name='keywords_or', description=_('Keyword IDs, repeat for multiple. Return recipes with any of the keywords'), qtype='integer'), + QueryParam(name='keywords_and', description=_('Keyword IDs, repeat for multiple. Return recipes with all of the keywords.'), qtype='integer'), + QueryParam(name='keywords_or_not', description=_('Keyword IDs, repeat for multiple. Exclude recipes with any of the keywords.'), qtype='integer'), + QueryParam(name='keywords_and_not', description=_('Keyword IDs, repeat for multiple. Exclude recipes with all of the keywords.'), qtype='integer'), + QueryParam(name='foods', description=_('ID of food a recipe should have. For multiple repeat parameter.'), qtype='integer'), + QueryParam(name='foods_or', description=_('Food IDs, repeat for multiple. Return recipes with any of the foods'), qtype='integer'), + QueryParam(name='foods_and', description=_('Food IDs, repeat for multiple. Return recipes with all of the foods.'), qtype='integer'), + QueryParam(name='foods_or_not', description=_('Food IDs, repeat for multiple. Exclude recipes with any of the foods.'), qtype='integer'), + QueryParam(name='foods_and_not', description=_('Food IDs, repeat for multiple. Exclude recipes with all of the foods.'), qtype='integer'), + QueryParam(name='units', description=_('ID of unit a recipe should have.'), qtype='integer'), + QueryParam(name='rating', description=_('Rating a recipe should have or greater. [0 - 5] Negative value filters rating less than.'), qtype='integer'), QueryParam(name='books', description=_('ID of book a recipe should be in. For multiple repeat parameter.')), - QueryParam(name='books_or', description=_('Book IDs, repeat for multiple. Return recipes with any of the books'), qtype='int'), - QueryParam(name='books_and', description=_('Book IDs, repeat for multiple. Return recipes with all of the books.'), qtype='int'), - QueryParam(name='books_or_not', description=_('Book IDs, repeat for multiple. Exclude recipes with any of the books.'), qtype='int'), - QueryParam(name='books_and_not', description=_('Book IDs, repeat for multiple. Exclude recipes with all of the books.'), qtype='int'), + QueryParam(name='books_or', description=_('Book IDs, repeat for multiple. Return recipes with any of the books'), qtype='integer'), + QueryParam(name='books_and', description=_('Book IDs, repeat for multiple. Return recipes with all of the books.'), qtype='integer'), + QueryParam(name='books_or_not', description=_('Book IDs, repeat for multiple. Exclude recipes with any of the books.'), qtype='integer'), + QueryParam(name='books_and_not', description=_('Book IDs, repeat for multiple. Exclude recipes with all of the books.'), qtype='integer'), QueryParam(name='internal', description=_('If only internal recipes should be returned. [''true''/''false'']')), QueryParam(name='random', description=_('Returns the results in randomized order. [''true''/''false'']')), QueryParam(name='new', description=_('Returns new results first in search results. [''true''/''false'']')), - QueryParam(name='timescooked', description=_('Filter recipes cooked X times or more. Negative values returns cooked less than X times'), qtype='int'), + QueryParam(name='timescooked', description=_('Filter recipes cooked X times or more. Negative values returns cooked less than X times'), qtype='integer'), QueryParam(name='cookedon', description=_('Filter recipes last cooked on or after YYYY-MM-DD. Prepending ''-'' filters on or before date.')), QueryParam(name='createdon', description=_('Filter recipes created on or after YYYY-MM-DD. Prepending ''-'' filters on or before date.')), QueryParam(name='updatedon', description=_('Filter recipes updated on or after YYYY-MM-DD. Prepending ''-'' filters on or before date.')), @@ -1095,7 +1110,7 @@ class UnitConversionViewSet(viewsets.ModelViewSet): serializer_class = UnitConversionSerializer permission_classes = [CustomIsUser & CustomTokenHasReadWriteScope] query_params = [ - QueryParam(name='food_id', description='ID of food to filter for', qtype='int'), + QueryParam(name='food_id', description='ID of food to filter for', qtype='integer'), ] schema = QueryParamAutoSchema() @@ -1146,10 +1161,10 @@ class ShoppingListEntryViewSet(viewsets.ModelViewSet): serializer_class = ShoppingListEntrySerializer permission_classes = [(CustomIsOwner | CustomIsShared) & CustomTokenHasReadWriteScope] query_params = [ - QueryParam(name='id', description=_('Returns the shopping list entry with a primary key of id. Multiple values allowed.'), qtype='int'), + QueryParam(name='id', description=_('Returns the shopping list entry with a primary key of id. Multiple values allowed.'), qtype='integer'), QueryParam(name='checked', description=_('Filter shopping list entries on checked. [''true'', ''false'', ''both'', ''recent'']
- ''recent'' includes unchecked items and recently completed items.') ), - QueryParam(name='supermarket', description=_('Returns the shopping list entries sorted by supermarket category order.'), qtype='int'), + QueryParam(name='supermarket', description=_('Returns the shopping list entries sorted by supermarket category order.'), qtype='integer'), ] schema = QueryParamAutoSchema() @@ -1614,6 +1629,7 @@ class ImportOpenData(APIView): # TODO validate data print(request.data) selected_version = request.data['selected_version'] + selected_datatypes = request.data['selected_datatypes'] update_existing = str2bool(request.data['update_existing']) use_metric = str2bool(request.data['use_metric']) @@ -1623,12 +1639,19 @@ class ImportOpenData(APIView): response_obj = {} data_importer = OpenDataImporter(request, data, update_existing=update_existing, use_metric=use_metric) - response_obj['unit'] = len(data_importer.import_units()) - response_obj['category'] = len(data_importer.import_category()) - response_obj['property'] = len(data_importer.import_property()) - response_obj['store'] = len(data_importer.import_supermarket()) - response_obj['food'] = len(data_importer.import_food()) - response_obj['conversion'] = len(data_importer.import_conversion()) + + if selected_datatypes['unit']['selected']: + response_obj['unit'] = data_importer.import_units().to_dict() + if selected_datatypes['category']['selected']: + response_obj['category'] = data_importer.import_category().to_dict() + if selected_datatypes['property']['selected']: + response_obj['property'] = data_importer.import_property().to_dict() + if selected_datatypes['store']['selected']: + response_obj['store'] = data_importer.import_supermarket().to_dict() + if selected_datatypes['food']['selected']: + response_obj['food'] = data_importer.import_food().to_dict() + if selected_datatypes['conversion']['selected']: + response_obj['conversion'] = data_importer.import_conversion().to_dict() return Response(response_obj) diff --git a/requirements.txt b/requirements.txt index 8fc695c7..ae4ded83 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ Django==4.2.10 -cryptography===42.0.0 +cryptography===42.0.2 django-annoying==0.10.6 django-autocomplete-light==3.9.7 django-cleanup==8.0.0 @@ -33,9 +33,9 @@ git+https://github.com/BITSOLVER/django-js-reverse@071e304fd600107bc64bbde6f2491 django-allauth==0.61.1 recipe-scrapers==14.52.0 django-scopes==2.0.0 -pytest==7.4.3 +pytest==8.0.0 pytest-asyncio==0.23.3 -pytest-django==4.6.0 +pytest-django==4.8.0 django-treebeard==4.7 django-cors-headers==4.2.0 django-storages==1.14.2 diff --git a/vue/src/apps/CookbookView/CookbookView.vue b/vue/src/apps/CookbookView/CookbookView.vue index 9ad465d5..3695eaa5 100644 --- a/vue/src/apps/CookbookView/CookbookView.vue +++ b/vue/src/apps/CookbookView/CookbookView.vue @@ -1,5 +1,5 @@