not yet fully working food import
This commit is contained in:
vabene1111
parent
7e3313f48c
commit
d62c49eb2f
@ -1,105 +1,174 @@
|
||||
from django.db.models import Q
|
||||
|
||||
from cookbook.models import Unit, SupermarketCategory, FoodProperty, FoodPropertyType, Supermarket, SupermarketCategoryRelation
|
||||
from cookbook.models import Unit, SupermarketCategory, FoodProperty, FoodPropertyType, Supermarket, SupermarketCategoryRelation, Food, Automation
|
||||
|
||||
|
||||
def import_units(data, request):
|
||||
unit_name_list = []
|
||||
for u in list(data['unit'].keys()):
|
||||
unit_name_list.append(data['unit'][u]['name'])
|
||||
unit_name_list.append(data['unit'][u]['plural_name'])
|
||||
class OpenDataImporter:
|
||||
request = None
|
||||
data = {}
|
||||
slug_id_cache = {}
|
||||
|
||||
existing_units = Unit.objects.filter(space=request.space).filter(Q(name__in=unit_name_list) | Q(plural_name__in=unit_name_list)).values_list('name', 'plural_name')
|
||||
existing_units = [item for sublist in existing_units for item in sublist]
|
||||
def __init__(self, request, data):
|
||||
self.request = request
|
||||
self.data = data
|
||||
|
||||
insert_list = []
|
||||
for u in list(data['unit'].keys()):
|
||||
if not (data['unit'][u]['name'] in existing_units or data['unit'][u]['plural_name'] in existing_units):
|
||||
insert_list.append(Unit(
|
||||
name=data['unit'][u]['name'],
|
||||
plural_name=data['unit'][u]['plural_name'],
|
||||
base_unit=data['unit'][u]['base_unit'] if data['unit'][u]['base_unit'] != '' else None,
|
||||
open_data_slug=u,
|
||||
space=request.space
|
||||
))
|
||||
def _update_slug_cache(self, object_class, datatype):
|
||||
self.slug_id_cache[datatype] = dict(object_class.objects.filter(space=self.request.space, open_data_slug__isnull=False).values_list('open_data_slug', 'id', ))
|
||||
|
||||
Unit.objects.bulk_create(insert_list)
|
||||
return len(insert_list)
|
||||
def import_units(self):
|
||||
unit_name_list = []
|
||||
for u in list(self.data['unit'].keys()):
|
||||
unit_name_list.append(self.data['unit'][u]['name'])
|
||||
unit_name_list.append(self.data['unit'][u]['plural_name'])
|
||||
|
||||
existing_units = Unit.objects.filter(space=self.request.space).filter(Q(name__in=unit_name_list) | Q(plural_name__in=unit_name_list)).values_list('name', 'plural_name')
|
||||
existing_units = [item for sublist in existing_units for item in sublist]
|
||||
|
||||
def import_category(data, request):
|
||||
identifier_list = []
|
||||
datatype = 'category'
|
||||
for k in list(data[datatype].keys()):
|
||||
identifier_list.append(data[datatype][k]['name'])
|
||||
insert_list = []
|
||||
for u in list(self.data['unit'].keys()):
|
||||
if not (self.data['unit'][u]['name'] in existing_units or self.data['unit'][u]['plural_name'] in existing_units):
|
||||
insert_list.append(Unit(
|
||||
name=self.data['unit'][u]['name'],
|
||||
plural_name=self.data['unit'][u]['plural_name'],
|
||||
base_unit=self.data['unit'][u]['base_unit'] if self.data['unit'][u]['base_unit'] != '' else None,
|
||||
open_data_slug=u,
|
||||
space=self.request.space
|
||||
))
|
||||
|
||||
existing_objects = SupermarketCategory.objects.filter(space=request.space).filter(name__in=identifier_list).values_list('name', flat=True)
|
||||
Unit.objects.bulk_create(insert_list)
|
||||
return len(insert_list)
|
||||
|
||||
insert_list = []
|
||||
for k in list(data[datatype].keys()):
|
||||
if not (data[datatype][k]['name'] in existing_objects):
|
||||
insert_list.append(SupermarketCategory(
|
||||
name=data[datatype][k]['name'],
|
||||
open_data_slug=k,
|
||||
space=request.space
|
||||
))
|
||||
def import_category(self):
|
||||
identifier_list = []
|
||||
datatype = 'category'
|
||||
for k in list(self.data[datatype].keys()):
|
||||
identifier_list.append(self.data[datatype][k]['name'])
|
||||
|
||||
SupermarketCategory.objects.bulk_create(insert_list)
|
||||
return len(insert_list)
|
||||
existing_objects = SupermarketCategory.objects.filter(space=self.request.space).filter(name__in=identifier_list).values_list('name', flat=True)
|
||||
|
||||
insert_list = []
|
||||
for k in list(self.data[datatype].keys()):
|
||||
if not (self.data[datatype][k]['name'] in existing_objects):
|
||||
insert_list.append(SupermarketCategory(
|
||||
name=self.data[datatype][k]['name'],
|
||||
open_data_slug=k,
|
||||
space=self.request.space
|
||||
))
|
||||
|
||||
def import_property(data, request):
|
||||
identifier_list = []
|
||||
datatype = 'property'
|
||||
for k in list(data[datatype].keys()):
|
||||
identifier_list.append(data[datatype][k]['name'])
|
||||
SupermarketCategory.objects.bulk_create(insert_list)
|
||||
return len(insert_list)
|
||||
|
||||
existing_objects = FoodPropertyType.objects.filter(space=request.space).filter(name__in=identifier_list).values_list('name', flat=True)
|
||||
def import_property(self):
|
||||
identifier_list = []
|
||||
datatype = 'property'
|
||||
for k in list(self.data[datatype].keys()):
|
||||
identifier_list.append(self.data[datatype][k]['name'])
|
||||
|
||||
insert_list = []
|
||||
for k in list(data[datatype].keys()):
|
||||
if not (data[datatype][k]['name'] in existing_objects):
|
||||
insert_list.append(FoodPropertyType(
|
||||
name=data[datatype][k]['name'],
|
||||
unit=data[datatype][k]['unit'],
|
||||
open_data_slug=k,
|
||||
space=request.space
|
||||
))
|
||||
existing_objects = FoodPropertyType.objects.filter(space=self.request.space).filter(name__in=identifier_list).values_list('name', flat=True)
|
||||
|
||||
FoodPropertyType.objects.bulk_create(insert_list)
|
||||
return len(insert_list)
|
||||
insert_list = []
|
||||
for k in list(self.data[datatype].keys()):
|
||||
if not (self.data[datatype][k]['name'] in existing_objects):
|
||||
insert_list.append(FoodPropertyType(
|
||||
name=self.data[datatype][k]['name'],
|
||||
unit=self.data[datatype][k]['unit'],
|
||||
open_data_slug=k,
|
||||
space=self.request.space
|
||||
))
|
||||
|
||||
FoodPropertyType.objects.bulk_create(insert_list)
|
||||
return len(insert_list)
|
||||
|
||||
def import_supermarket(data, request):
|
||||
identifier_list = []
|
||||
datatype = 'supermarket'
|
||||
for k in list(data[datatype].keys()):
|
||||
identifier_list.append(data[datatype][k]['name'])
|
||||
def import_supermarket(self):
|
||||
identifier_list = []
|
||||
datatype = 'supermarket'
|
||||
for k in list(self.data[datatype].keys()):
|
||||
identifier_list.append(self.data[datatype][k]['name'])
|
||||
|
||||
existing_objects = Supermarket.objects.filter(space=request.space).filter(name__in=identifier_list).values_list('name', flat=True)
|
||||
existing_objects = Supermarket.objects.filter(space=self.request.space).filter(name__in=identifier_list).values_list('name', flat=True)
|
||||
|
||||
supermarket_categories = SupermarketCategory.objects.filter(space=request.space, open_data_slug__isnull=False).values_list('id', 'open_data_slug')
|
||||
insert_list = []
|
||||
for k in list(data[datatype].keys()):
|
||||
if not (data[datatype][k]['name'] in existing_objects): # TODO on large datasets see if bulk creating supermarkets and then relations as well is better
|
||||
supermarket = Supermarket.objects.create(
|
||||
name=data[datatype][k]['name'],
|
||||
open_data_slug=k,
|
||||
space=request.space
|
||||
)
|
||||
self._update_slug_cache(SupermarketCategory, 'category')
|
||||
|
||||
relations = []
|
||||
order = 0
|
||||
for c in data[datatype][k]['categories']:
|
||||
relations.append(
|
||||
SupermarketCategoryRelation(
|
||||
supermarket=supermarket,
|
||||
category_id=[x[0] for x in supermarket_categories if x[1] == c][0],
|
||||
order=order,
|
||||
)
|
||||
insert_list = []
|
||||
for k in list(self.data[datatype].keys()):
|
||||
if not (self.data[datatype][k]['name'] in existing_objects): # TODO on large datasets see if bulk creating supermarkets and then relations as well is better
|
||||
supermarket = Supermarket.objects.create(
|
||||
name=self.data[datatype][k]['name'],
|
||||
open_data_slug=k,
|
||||
space=self.request.space
|
||||
)
|
||||
order += 1
|
||||
|
||||
SupermarketCategoryRelation.objects.bulk_create(relations)
|
||||
relations = []
|
||||
order = 0
|
||||
for c in self.data[datatype][k]['categories']:
|
||||
relations.append(
|
||||
SupermarketCategoryRelation(
|
||||
supermarket=supermarket,
|
||||
category_id=self.slug_id_cache['category'][c],
|
||||
order=order,
|
||||
)
|
||||
)
|
||||
order += 1
|
||||
|
||||
return len(insert_list)
|
||||
SupermarketCategoryRelation.objects.bulk_create(relations)
|
||||
|
||||
return len(insert_list)
|
||||
|
||||
def import_food(self, metric=True):
|
||||
identifier_list = []
|
||||
datatype = 'food'
|
||||
for k in list(self.data[datatype].keys()):
|
||||
identifier_list.append(self.data[datatype][k]['name'])
|
||||
identifier_list.append(self.data[datatype][k]['plural_name'])
|
||||
|
||||
existing_objects = Food.objects.filter(space=self.request.space).filter(name__in=identifier_list).values_list('name', flat=True)
|
||||
|
||||
self._update_slug_cache(Unit, 'unit')
|
||||
self._update_slug_cache(FoodPropertyType, 'property')
|
||||
|
||||
pref_unit_key = 'preferred_unit_metric'
|
||||
pref_shopping_unit_key = 'preferred_packaging_unit_metric'
|
||||
if not metric:
|
||||
pref_unit_key = 'preferred_unit_imperial'
|
||||
pref_shopping_unit_key = 'preferred_packaging_unit_imperial'
|
||||
|
||||
insert_list = []
|
||||
|
||||
for k in list(self.data[datatype].keys()):
|
||||
if not (self.data[datatype][k]['name'] in existing_objects):
|
||||
insert_list.append(Food(
|
||||
name=self.data[datatype][k]['name'],
|
||||
plural_name=self.data[datatype][k]['plural_name'] if self.data[datatype][k]['plural_name'] != '' else None,
|
||||
preferred_unit_id=self.slug_id_cache['unit'][self.data[datatype][k][pref_unit_key]],
|
||||
preferred_shopping_unit_id=self.slug_id_cache['unit'][self.data[datatype][k][pref_shopping_unit_key]],
|
||||
supermarket_category_id=self.slug_id_cache['category'][self.data[datatype][k]['supermarket_category']],
|
||||
fdc_id=self.data[datatype][k]['fdc_id'] if self.data[datatype][k]['fdc_id'] != '' else None,
|
||||
open_data_slug=k,
|
||||
space=self.request.space,
|
||||
))
|
||||
|
||||
Food.objects.bulk_create(insert_list)
|
||||
|
||||
self._update_slug_cache(Food, 'food')
|
||||
|
||||
food_property_list = []
|
||||
alias_list = []
|
||||
for k in list(self.data[datatype].keys()):
|
||||
for fp in self.data[datatype][k]['properties']['type_values']:
|
||||
food_property_list.append(FoodProperty(
|
||||
food_id=self.slug_id_cache['food'][k],
|
||||
food_amount=self.data[datatype][k]['properties']['food_amount'],
|
||||
food_unit_id=self.slug_id_cache['unit'][self.data[datatype][k]['properties']['food_unit']],
|
||||
property_type_id=self.slug_id_cache['property'][fp['property_type']],
|
||||
property_amount=fp['property_value'],
|
||||
))
|
||||
|
||||
for a in self.data[datatype][k]['alias']:
|
||||
alias_list.append(Automation(
|
||||
param_1=a,
|
||||
param_2=self.data[datatype][k]['name']
|
||||
))
|
||||
|
||||
FoodProperty.objects.bulk_create(food_property_list)
|
||||
Automation.objects.bulk_create(alias_list)
|
||||
return len(insert_list)
|
||||
|
||||
@ -0,0 +1,41 @@
|
||||
# Generated by Django 4.1.7 on 2023-05-01 10:41
|
||||
|
||||
from django.db import migrations, models
|
||||
import django.db.models.deletion
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('cookbook', '0191_food_open_data_slug_foodpropertytype_fdc_id_and_more'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RemoveField(
|
||||
model_name='foodpropertytype',
|
||||
name='fdc_id',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='foodpropertytype',
|
||||
name='preferred_shopping_unit',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='foodpropertytype',
|
||||
name='preferred_unit',
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='food',
|
||||
name='fdc_id',
|
||||
field=models.CharField(blank=True, default=None, max_length=128, null=True),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='food',
|
||||
name='preferred_shopping_unit',
|
||||
field=models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='preferred_shopping_unit', to='cookbook.unit'),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='food',
|
||||
name='preferred_unit',
|
||||
field=models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='preferred_unit', to='cookbook.unit'),
|
||||
),
|
||||
]
|
||||
@ -573,6 +573,10 @@ class Food(ExportModelOperationsMixin('food'), TreeModel, PermissionModelMixin):
|
||||
substitute_children = models.BooleanField(default=False)
|
||||
child_inherit_fields = models.ManyToManyField(FoodInheritField, blank=True, related_name='child_inherit')
|
||||
|
||||
preferred_unit = models.ForeignKey(Unit, on_delete=models.SET_NULL, null=True, blank=True, default=None, related_name='preferred_unit')
|
||||
preferred_shopping_unit = models.ForeignKey(Unit, on_delete=models.SET_NULL, null=True, blank=True, default=None, related_name='preferred_shopping_unit')
|
||||
fdc_id = models.CharField(max_length=128, null=True, blank=True, default=None)
|
||||
|
||||
open_data_slug = models.CharField(max_length=128, null=True, blank=True, default=None)
|
||||
space = models.ForeignKey(Space, on_delete=models.CASCADE)
|
||||
objects = ScopedManager(space='space', _manager_class=TreeManager)
|
||||
@ -755,9 +759,6 @@ class FoodPropertyType(models.Model, PermissionModelMixin):
|
||||
icon = models.CharField(max_length=16, blank=True, null=True)
|
||||
description = models.CharField(max_length=512, blank=True, null=True)
|
||||
category = models.CharField(max_length=64, choices=((NUTRITION, _('Nutrition')), (ALLERGEN, _('Allergen')), (PRICE, _('Price')), (GOAL, _('Goal')), (OTHER, _('Other'))), null=True, blank=True)
|
||||
preferred_unit = models.ForeignKey(Unit, on_delete=models.SET_NULL, null=True, blank=True, default=None, related_name='preferred_unit')
|
||||
preferred_shopping_unit = models.ForeignKey(Unit, on_delete=models.SET_NULL, null=True, blank=True, default=None, related_name='preferred_shopping_unit')
|
||||
fdc_id = models.CharField(max_length=128, null=True, blank=True, default=None)
|
||||
open_data_slug = models.CharField(max_length=128, null=True, blank=True, default=None)
|
||||
|
||||
# TODO show if empty property?
|
||||
|
||||
@ -54,7 +54,7 @@ from cookbook.helper import recipe_url_import as helper
|
||||
from cookbook.helper.HelperFunctions import str2bool
|
||||
from cookbook.helper.image_processing import handle_image
|
||||
from cookbook.helper.ingredient_parser import IngredientParser
|
||||
from cookbook.helper.open_data_importer import import_units, import_category, import_property, import_supermarket
|
||||
from cookbook.helper.open_data_importer import OpenDataImporter
|
||||
from cookbook.helper.permission_helper import (CustomIsAdmin, CustomIsOwner,
|
||||
CustomIsOwnerReadOnly, CustomIsShared,
|
||||
CustomIsSpaceOwner, CustomIsUser, group_required,
|
||||
@ -1438,13 +1438,13 @@ class ImportOpenData(APIView):
|
||||
response = requests.get(f'https://raw.githubusercontent.com/TandoorRecipes/open-tandoor-data/main/build/{selected_version}.json') # TODO catch 404, timeout, ...
|
||||
data = json.loads(response.content)
|
||||
|
||||
import_units(data, request)
|
||||
import_category(data, request)
|
||||
import_property(data, request)
|
||||
import_supermarket(data, request)
|
||||
data_importer = OpenDataImporter(request, data)
|
||||
data_importer.import_units()
|
||||
data_importer.import_category()
|
||||
data_importer.import_property()
|
||||
data_importer.import_supermarket()
|
||||
data_importer.import_food() # TODO pass metric parameter
|
||||
|
||||
# TODO hardcode insert order?
|
||||
# TODO split into update/create lists with multiple parameters per datatype
|
||||
return Response({
|
||||
'test': ''
|
||||
})
|
||||
|
||||
Loading…
Reference in New Issue
Block a user