import open data content
This commit is contained in:
parent
89e3e85d1e
commit
1bb6eb7141
105
cookbook/helper/open_data_importer.py
Normal file
105
cookbook/helper/open_data_importer.py
Normal file
@ -0,0 +1,105 @@
|
|||||||
|
from django.db.models import Q
|
||||||
|
|
||||||
|
from cookbook.models import Unit, SupermarketCategory, FoodProperty, FoodPropertyType, Supermarket, SupermarketCategoryRelation
|
||||||
|
|
||||||
|
|
||||||
|
def import_units(data, request):
|
||||||
|
unit_name_list = []
|
||||||
|
for u in list(data['unit'].keys()):
|
||||||
|
unit_name_list.append(data['unit'][u]['name'])
|
||||||
|
unit_name_list.append(data['unit'][u]['plural_name'])
|
||||||
|
|
||||||
|
existing_units = Unit.objects.filter(space=request.space).filter(Q(name__in=unit_name_list) | Q(plural_name__in=unit_name_list)).values_list('name', 'plural_name')
|
||||||
|
existing_units = [item for sublist in existing_units for item in sublist]
|
||||||
|
|
||||||
|
insert_list = []
|
||||||
|
for u in list(data['unit'].keys()):
|
||||||
|
if not (data['unit'][u]['name'] in existing_units or data['unit'][u]['plural_name'] in existing_units):
|
||||||
|
insert_list.append(Unit(
|
||||||
|
name=data['unit'][u]['name'],
|
||||||
|
plural_name=data['unit'][u]['plural_name'],
|
||||||
|
base_unit=data['unit'][u]['base_unit'] if data['unit'][u]['base_unit'] != '' else None,
|
||||||
|
open_data_slug=u,
|
||||||
|
space=request.space
|
||||||
|
))
|
||||||
|
|
||||||
|
Unit.objects.bulk_create(insert_list)
|
||||||
|
return len(insert_list)
|
||||||
|
|
||||||
|
|
||||||
|
def import_category(data, request):
|
||||||
|
identifier_list = []
|
||||||
|
datatype = 'category'
|
||||||
|
for k in list(data[datatype].keys()):
|
||||||
|
identifier_list.append(data[datatype][k]['name'])
|
||||||
|
|
||||||
|
existing_objects = SupermarketCategory.objects.filter(space=request.space).filter(name__in=identifier_list).values_list('name', flat=True)
|
||||||
|
|
||||||
|
insert_list = []
|
||||||
|
for k in list(data[datatype].keys()):
|
||||||
|
if not (data[datatype][k]['name'] in existing_objects):
|
||||||
|
insert_list.append(SupermarketCategory(
|
||||||
|
name=data[datatype][k]['name'],
|
||||||
|
open_data_slug=k,
|
||||||
|
space=request.space
|
||||||
|
))
|
||||||
|
|
||||||
|
SupermarketCategory.objects.bulk_create(insert_list)
|
||||||
|
return len(insert_list)
|
||||||
|
|
||||||
|
|
||||||
|
def import_property(data, request):
|
||||||
|
identifier_list = []
|
||||||
|
datatype = 'property'
|
||||||
|
for k in list(data[datatype].keys()):
|
||||||
|
identifier_list.append(data[datatype][k]['name'])
|
||||||
|
|
||||||
|
existing_objects = FoodPropertyType.objects.filter(space=request.space).filter(name__in=identifier_list).values_list('name', flat=True)
|
||||||
|
|
||||||
|
insert_list = []
|
||||||
|
for k in list(data[datatype].keys()):
|
||||||
|
if not (data[datatype][k]['name'] in existing_objects):
|
||||||
|
insert_list.append(FoodPropertyType(
|
||||||
|
name=data[datatype][k]['name'],
|
||||||
|
unit=data[datatype][k]['unit'],
|
||||||
|
open_data_slug=k,
|
||||||
|
space=request.space
|
||||||
|
))
|
||||||
|
|
||||||
|
FoodPropertyType.objects.bulk_create(insert_list)
|
||||||
|
return len(insert_list)
|
||||||
|
|
||||||
|
|
||||||
|
def import_supermarket(data, request):
|
||||||
|
identifier_list = []
|
||||||
|
datatype = 'supermarket'
|
||||||
|
for k in list(data[datatype].keys()):
|
||||||
|
identifier_list.append(data[datatype][k]['name'])
|
||||||
|
|
||||||
|
existing_objects = Supermarket.objects.filter(space=request.space).filter(name__in=identifier_list).values_list('name', flat=True)
|
||||||
|
|
||||||
|
supermarket_categories = SupermarketCategory.objects.filter(space=request.space, open_data_slug__isnull=False).values_list('id', 'open_data_slug')
|
||||||
|
insert_list = []
|
||||||
|
for k in list(data[datatype].keys()):
|
||||||
|
if not (data[datatype][k]['name'] in existing_objects): # TODO on large datasets see if bulk creating supermarkets and then relations as well is better
|
||||||
|
supermarket = Supermarket.objects.create(
|
||||||
|
name=data[datatype][k]['name'],
|
||||||
|
open_data_slug=k,
|
||||||
|
space=request.space
|
||||||
|
)
|
||||||
|
|
||||||
|
relations = []
|
||||||
|
order = 0
|
||||||
|
for c in data[datatype][k]['categories']:
|
||||||
|
relations.append(
|
||||||
|
SupermarketCategoryRelation(
|
||||||
|
supermarket=supermarket,
|
||||||
|
category_id=[x[0] for x in supermarket_categories if x[1] == c][0],
|
||||||
|
order=order,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
order += 1
|
||||||
|
|
||||||
|
SupermarketCategoryRelation.objects.bulk_create(relations)
|
||||||
|
|
||||||
|
return len(insert_list)
|
@ -0,0 +1,64 @@
|
|||||||
|
# Generated by Django 4.1.7 on 2023-04-30 20:11
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
import django.db.models.deletion
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('cookbook', '0190_remove_foodproperty_food_nutrition_unique_per_space_and_more'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AddField(
|
||||||
|
model_name='food',
|
||||||
|
name='open_data_slug',
|
||||||
|
field=models.CharField(blank=True, default=None, max_length=128, null=True),
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name='foodpropertytype',
|
||||||
|
name='fdc_id',
|
||||||
|
field=models.CharField(blank=True, default=None, max_length=128, null=True),
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name='foodpropertytype',
|
||||||
|
name='open_data_slug',
|
||||||
|
field=models.CharField(blank=True, default=None, max_length=128, null=True),
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name='foodpropertytype',
|
||||||
|
name='preferred_shopping_unit',
|
||||||
|
field=models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='preferred_shopping_unit', to='cookbook.unit'),
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name='foodpropertytype',
|
||||||
|
name='preferred_unit',
|
||||||
|
field=models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='preferred_unit', to='cookbook.unit'),
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name='supermarket',
|
||||||
|
name='open_data_slug',
|
||||||
|
field=models.CharField(blank=True, default=None, max_length=128, null=True),
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name='supermarketcategory',
|
||||||
|
name='open_data_slug',
|
||||||
|
field=models.CharField(blank=True, default=None, max_length=128, null=True),
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name='unit',
|
||||||
|
name='open_data_slug',
|
||||||
|
field=models.CharField(blank=True, default=None, max_length=128, null=True),
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name='unitconversion',
|
||||||
|
name='open_data_slug',
|
||||||
|
field=models.CharField(blank=True, default=None, max_length=128, null=True),
|
||||||
|
),
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='foodpropertytype',
|
||||||
|
name='category',
|
||||||
|
field=models.CharField(blank=True, choices=[('NUTRITION', 'Nutrition'), ('ALLERGEN', 'Allergen'), ('PRICE', 'Price'), ('GOAL', 'Goal'), ('OTHER', 'Other')], max_length=64, null=True),
|
||||||
|
),
|
||||||
|
]
|
@ -454,6 +454,7 @@ class Sync(models.Model, PermissionModelMixin):
|
|||||||
class SupermarketCategory(models.Model, PermissionModelMixin):
|
class SupermarketCategory(models.Model, PermissionModelMixin):
|
||||||
name = models.CharField(max_length=128, validators=[MinLengthValidator(1)])
|
name = models.CharField(max_length=128, validators=[MinLengthValidator(1)])
|
||||||
description = models.TextField(blank=True, null=True)
|
description = models.TextField(blank=True, null=True)
|
||||||
|
open_data_slug = models.CharField(max_length=128, null=True, blank=True, default=None)
|
||||||
|
|
||||||
space = models.ForeignKey(Space, on_delete=models.CASCADE)
|
space = models.ForeignKey(Space, on_delete=models.CASCADE)
|
||||||
objects = ScopedManager(space='space')
|
objects = ScopedManager(space='space')
|
||||||
@ -471,6 +472,7 @@ class Supermarket(models.Model, PermissionModelMixin):
|
|||||||
name = models.CharField(max_length=128, validators=[MinLengthValidator(1)])
|
name = models.CharField(max_length=128, validators=[MinLengthValidator(1)])
|
||||||
description = models.TextField(blank=True, null=True)
|
description = models.TextField(blank=True, null=True)
|
||||||
categories = models.ManyToManyField(SupermarketCategory, through='SupermarketCategoryRelation')
|
categories = models.ManyToManyField(SupermarketCategory, through='SupermarketCategoryRelation')
|
||||||
|
open_data_slug = models.CharField(max_length=128, null=True, blank=True, default=None)
|
||||||
|
|
||||||
space = models.ForeignKey(Space, on_delete=models.CASCADE)
|
space = models.ForeignKey(Space, on_delete=models.CASCADE)
|
||||||
objects = ScopedManager(space='space')
|
objects = ScopedManager(space='space')
|
||||||
@ -535,6 +537,7 @@ class Unit(ExportModelOperationsMixin('unit'), models.Model, PermissionModelMixi
|
|||||||
plural_name = models.CharField(max_length=128, null=True, blank=True, default=None)
|
plural_name = models.CharField(max_length=128, null=True, blank=True, default=None)
|
||||||
description = models.TextField(blank=True, null=True)
|
description = models.TextField(blank=True, null=True)
|
||||||
base_unit = models.TextField(max_length=256, null=True, blank=True, default=None)
|
base_unit = models.TextField(max_length=256, null=True, blank=True, default=None)
|
||||||
|
open_data_slug = models.CharField(max_length=128, null=True, blank=True, default=None)
|
||||||
|
|
||||||
space = models.ForeignKey(Space, on_delete=models.CASCADE)
|
space = models.ForeignKey(Space, on_delete=models.CASCADE)
|
||||||
objects = ScopedManager(space='space')
|
objects = ScopedManager(space='space')
|
||||||
@ -570,6 +573,7 @@ class Food(ExportModelOperationsMixin('food'), TreeModel, PermissionModelMixin):
|
|||||||
substitute_children = models.BooleanField(default=False)
|
substitute_children = models.BooleanField(default=False)
|
||||||
child_inherit_fields = models.ManyToManyField(FoodInheritField, blank=True, related_name='child_inherit')
|
child_inherit_fields = models.ManyToManyField(FoodInheritField, blank=True, related_name='child_inherit')
|
||||||
|
|
||||||
|
open_data_slug = models.CharField(max_length=128, null=True, blank=True, default=None)
|
||||||
space = models.ForeignKey(Space, on_delete=models.CASCADE)
|
space = models.ForeignKey(Space, on_delete=models.CASCADE)
|
||||||
objects = ScopedManager(space='space', _manager_class=TreeManager)
|
objects = ScopedManager(space='space', _manager_class=TreeManager)
|
||||||
|
|
||||||
@ -663,6 +667,7 @@ class UnitConversion(ExportModelOperationsMixin('unit_conversion'), models.Model
|
|||||||
created_at = models.DateTimeField(auto_now_add=True)
|
created_at = models.DateTimeField(auto_now_add=True)
|
||||||
updated_at = models.DateTimeField(auto_now=True)
|
updated_at = models.DateTimeField(auto_now=True)
|
||||||
|
|
||||||
|
open_data_slug = models.CharField(max_length=128, null=True, blank=True, default=None)
|
||||||
space = models.ForeignKey(Space, on_delete=models.CASCADE)
|
space = models.ForeignKey(Space, on_delete=models.CASCADE)
|
||||||
objects = ScopedManager(space='space')
|
objects = ScopedManager(space='space')
|
||||||
|
|
||||||
@ -750,6 +755,10 @@ class FoodPropertyType(models.Model, PermissionModelMixin):
|
|||||||
icon = models.CharField(max_length=16, blank=True, null=True)
|
icon = models.CharField(max_length=16, blank=True, null=True)
|
||||||
description = models.CharField(max_length=512, blank=True, null=True)
|
description = models.CharField(max_length=512, blank=True, null=True)
|
||||||
category = models.CharField(max_length=64, choices=((NUTRITION, _('Nutrition')), (ALLERGEN, _('Allergen')), (PRICE, _('Price')), (GOAL, _('Goal')), (OTHER, _('Other'))), null=True, blank=True)
|
category = models.CharField(max_length=64, choices=((NUTRITION, _('Nutrition')), (ALLERGEN, _('Allergen')), (PRICE, _('Price')), (GOAL, _('Goal')), (OTHER, _('Other'))), null=True, blank=True)
|
||||||
|
preferred_unit = models.ForeignKey(Unit, on_delete=models.SET_NULL, null=True, blank=True, default=None, related_name='preferred_unit')
|
||||||
|
preferred_shopping_unit = models.ForeignKey(Unit, on_delete=models.SET_NULL, null=True, blank=True, default=None, related_name='preferred_shopping_unit')
|
||||||
|
fdc_id = models.CharField(max_length=128, null=True, blank=True, default=None)
|
||||||
|
open_data_slug = models.CharField(max_length=128, null=True, blank=True, default=None)
|
||||||
|
|
||||||
# TODO show if empty property?
|
# TODO show if empty property?
|
||||||
# TODO formatting property?
|
# TODO formatting property?
|
||||||
|
@ -54,6 +54,7 @@ from cookbook.helper import recipe_url_import as helper
|
|||||||
from cookbook.helper.HelperFunctions import str2bool
|
from cookbook.helper.HelperFunctions import str2bool
|
||||||
from cookbook.helper.image_processing import handle_image
|
from cookbook.helper.image_processing import handle_image
|
||||||
from cookbook.helper.ingredient_parser import IngredientParser
|
from cookbook.helper.ingredient_parser import IngredientParser
|
||||||
|
from cookbook.helper.open_data_importer import import_units, import_category, import_property, import_supermarket
|
||||||
from cookbook.helper.permission_helper import (CustomIsAdmin, CustomIsOwner,
|
from cookbook.helper.permission_helper import (CustomIsAdmin, CustomIsOwner,
|
||||||
CustomIsOwnerReadOnly, CustomIsShared,
|
CustomIsOwnerReadOnly, CustomIsShared,
|
||||||
CustomIsSpaceOwner, CustomIsUser, group_required,
|
CustomIsSpaceOwner, CustomIsUser, group_required,
|
||||||
@ -1437,31 +1438,15 @@ class ImportOpenData(APIView):
|
|||||||
response = requests.get(f'https://raw.githubusercontent.com/TandoorRecipes/open-tandoor-data/main/build/{selected_version}.json') # TODO catch 404, timeout, ...
|
response = requests.get(f'https://raw.githubusercontent.com/TandoorRecipes/open-tandoor-data/main/build/{selected_version}.json') # TODO catch 404, timeout, ...
|
||||||
data = json.loads(response.content)
|
data = json.loads(response.content)
|
||||||
|
|
||||||
unit_name_list = []
|
import_units(data, request)
|
||||||
for u in list(data['unit'].keys()):
|
import_category(data, request)
|
||||||
unit_name_list.append(data['unit'][u]['name'])
|
import_property(data, request)
|
||||||
unit_name_list.append(data['unit'][u]['plural_name'])
|
import_supermarket(data, request)
|
||||||
|
|
||||||
existing_units = Unit.objects.filter(space=request.space).filter(Q(name__in=unit_name_list) | Q(plural_name__in=unit_name_list)).values_list('name', 'plural_name')
|
|
||||||
existing_units = [item for sublist in existing_units for item in sublist]
|
|
||||||
|
|
||||||
insert_list = []
|
|
||||||
for u in list(data['unit'].keys()):
|
|
||||||
if not (data['unit'][u]['name'] in existing_units or data['unit'][u]['plural_name'] in existing_units):
|
|
||||||
insert_list.append(Unit(
|
|
||||||
name=data['unit'][u]['name'],
|
|
||||||
plural_name=data['unit'][u]['plural_name'],
|
|
||||||
base_unit=data['unit'][u]['base_unit'] if data['unit'][u]['base_unit'] != '' else None,
|
|
||||||
space=request.space
|
|
||||||
))
|
|
||||||
|
|
||||||
Unit.objects.bulk_create(insert_list)
|
|
||||||
|
|
||||||
# TODO hardcode insert order?
|
# TODO hardcode insert order?
|
||||||
# TODO split into update/create lists with multiple parameters per datatype
|
# TODO split into update/create lists with multiple parameters per datatype
|
||||||
print(existing_units)
|
|
||||||
return Response({
|
return Response({
|
||||||
'test': existing_units
|
'test': ''
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
||||||
|
@ -9,6 +9,17 @@
|
|||||||
<option v-for="v in metadata.versions" v-bind:key="v">{{ v }}</option>
|
<option v-for="v in metadata.versions" v-bind:key="v">{{ v }}</option>
|
||||||
</select>
|
</select>
|
||||||
|
|
||||||
|
<select class="form-control">
|
||||||
|
<option>{{$t('Metric')}}</option>
|
||||||
|
<option>{{$t('Imperial')}}</option>
|
||||||
|
</select>
|
||||||
|
|
||||||
|
<select class="form-control">
|
||||||
|
<option selected>{{$t('Ignore')}}</option>
|
||||||
|
<option>{{$t('Merge')}}</option>
|
||||||
|
<option>{{$t('Replace')}}</option>
|
||||||
|
</select>
|
||||||
|
|
||||||
<div v-if="selected_version !== undefined">
|
<div v-if="selected_version !== undefined">
|
||||||
<table class="table">
|
<table class="table">
|
||||||
<tr>
|
<tr>
|
||||||
|
Loading…
Reference in New Issue
Block a user