cgiacofei/TandoorRecipes
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

improved importer merging behavior

Browse Source
This commit is contained in:
vabene1111
2023-05-04 15:29:06 +02:00
parent 33d048e623
commit 071926aada
9 changed files with 74 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
cookbook/helper/open_data_importer.py
View File

@ -7,10 +7,12 @@ class OpenDataImporter:
request = None
data = {}
slug_id_cache = {}
update_existing = False
def __init__(self, request, data):
def __init__(self, request, data, update_existing=False):
self.request = request
self.data = data
self.update_existing = update_existing
def _update_slug_cache(self, object_class, datatype):
self.slug_id_cache[datatype] = dict(object_class.objects.filter(space=self.request.space, open_data_slug__isnull=False).values_list('open_data_slug', 'id', ))
@ -80,37 +82,36 @@ class OpenDataImporter:
return len(insert_list)
def import_supermarket(self):
identifier_list = []
datatype = 'supermarket'
for k in list(self.data[datatype].keys()):
identifier_list.append(self.data[datatype][k]['name'])
existing_objects = Supermarket.objects.filter(space=self.request.space).filter(name__in=identifier_list).values_list('name', flat=True)
self._update_slug_cache(SupermarketCategory, 'category')
insert_list = []
for k in list(self.data[datatype].keys()):
if not (self.data[datatype][k]['name'] in existing_objects): # TODO on large datasets see if bulk creating supermarkets and then relations as well is better
supermarket = Supermarket.objects.create(
insert_list.append(Supermarket(
name=self.data[datatype][k]['name'],
open_data_slug=k,
space=self.request.space
)
))
# always add open data slug if matching supermarket is found, otherwise relation might fail
Supermarket.objects.bulk_create(insert_list, unique_fields=('space', 'name',), update_conflicts=True, update_fields=('open_data_slug',))
self._update_slug_cache(Supermarket, 'supermarket')
insert_list = []
for k in list(self.data[datatype].keys()):
relations = []
order = 0
for c in self.data[datatype][k]['categories']:
relations.append(
SupermarketCategoryRelation(
supermarket=supermarket,
supermarket_id=self.slug_id_cache[datatype][k],
category_id=self.slug_id_cache['category'][c],
order=order,
)
)
order += 1
SupermarketCategoryRelation.objects.bulk_create(relations)
SupermarketCategoryRelation.objects.bulk_create(relations, ignore_conflicts=True, unique_fields=('supermarket', 'category',))
return len(insert_list)
@ -172,8 +173,8 @@ class OpenDataImporter:
created_by=self.request.user,
))
FoodProperty.objects.bulk_create(food_property_list, ignore_conflicts=True, unique_fields=('space', 'food', 'property_type'))
Automation.objects.bulk_create(alias_list)
FoodProperty.objects.bulk_create(food_property_list, ignore_conflicts=True, unique_fields=('space', 'food', 'property_type',))
Automation.objects.bulk_create(alias_list, ignore_conflicts=True, unique_fields=('space', 'param_1', 'param_2',))
return len(insert_list)
def import_conversion(self):

17
cookbook/migrations/0194_supermarketcategoryrelation_unique_sm_category_relation.py Normal file
View File

@ -0,0 +1,17 @@
# Generated by Django 4.1.7 on 2023-05-04 13:24
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('cookbook', '0193_unitconversion_f_unique_conversion_per_space'),
]
operations = [
migrations.AddConstraint(
model_name='supermarketcategoryrelation',
constraint=models.UniqueConstraint(fields=('supermarket', 'category'), name='unique_sm_category_relation'),
),
]

3
cookbook/models.py
View File

@ -498,6 +498,9 @@ class SupermarketCategoryRelation(models.Model, PermissionModelMixin):
return 'supermarket', 'space'
class Meta:
constraints = [
models.UniqueConstraint(fields=['supermarket', 'category'], name='unique_sm_category_relation')
]
ordering = ('order',)

4
cookbook/serializer.py
View File

@ -497,7 +497,7 @@ class SupermarketSerializer(UniqueFieldsMixin, SpacedModelSerializer):
class Meta:
model = Supermarket
fields = ('id', 'name', 'description', 'category_to_supermarket')
fields = ('id', 'name', 'description', 'category_to_supermarket', 'open_data_slug')
class RecipeSimpleSerializer(WritableNestedModelSerializer):
@ -737,7 +737,7 @@ class UnitConversionSerializer(WritableNestedModelSerializer):
class Meta:
model = UnitConversion
fields = ('id', 'base_amount', 'base_unit', 'converted_amount', 'converted_unit', 'food')
fields = ('id', 'base_amount', 'base_unit', 'converted_amount', 'converted_unit', 'food', 'open_data_slug')
class FoodPropertyTypeSerializer(serializers.ModelSerializer):

2
cookbook/views/api.py
View File

@ -1438,7 +1438,7 @@ class ImportOpenData(APIView):
response = requests.get(f'https://raw.githubusercontent.com/TandoorRecipes/open-tandoor-data/main/build/{selected_version}.json') # TODO catch 404, timeout, ...
data = json.loads(response.content)
data_importer = OpenDataImporter(request, data)
data_importer = OpenDataImporter(request, data, update_existing=True)
data_importer.import_units()
data_importer.import_category()
data_importer.import_property()

2
vue/src/components/Modals/GenericModalForm.vue
View File

@ -9,7 +9,7 @@
<p v-if="visibleCondition(f, 'instruction')">{{ f.label }}</p>
<lookup-input v-if="visibleCondition(f, 'lookup')" :form="f" :model="listModel(f.list)" @change="storeValue" :help="showHelp && f.help" />
<checkbox-input class="mb-3" v-if="visibleCondition(f, 'checkbox')" :label="f.label" :value="f.value" :field="f.field" :help="showHelp && f.help" />
<text-input v-if="visibleCondition(f, 'text')" :label="f.label" :value="f.value" :field="f.field" :placeholder="f.placeholder" :help="showHelp && f.help" :subtitle="f.subtitle" />
<text-input v-if="visibleCondition(f, 'text')" :label="f.label" :value="f.value" :field="f.field" :placeholder="f.placeholder" :help="showHelp && f.help" :subtitle="f.subtitle" :disabled="f.disabled"/>
<choice-input v-if="visibleCondition(f, 'choice')" :label="f.label" :value="f.value" :field="f.field" :options="f.options" :placeholder="f.placeholder" />
<emoji-input v-if="visibleCondition(f, 'emoji')" :label="f.label" :value="f.value" :field="f.field" @change="storeValue" />
<file-input v-if="visibleCondition(f, 'file')" :label="f.label" :value="f.value" :field="f.field" @change="storeValue" />

3
vue/src/components/Modals/TextInput.vue
View File

@ -1,7 +1,7 @@
<template>
<div>
<b-form-group v-bind:label="label" class="mb-3">
<b-form-input v-model="new_value" type="text" :placeholder="placeholder"></b-form-input>
<b-form-input v-model="new_value" type="text" :placeholder="placeholder" :disabled="disabled"></b-form-input>
<em v-if="help" class="small text-muted">{{ help }}</em>
<small v-if="subtitle" class="text-muted">{{ subtitle }}</small>
</b-form-group>
@ -18,6 +18,7 @@ export default {
placeholder: { type: String, default: "You Should Add Placeholder Text" },
help: { type: String, default: undefined },
subtitle: { type: String, default: undefined },
disabled: { type: Boolean, default: false }
},
data() {
return {

2
vue/src/locales/en.json
View File

@ -76,6 +76,8 @@
"Private_Recipe": "Private Recipe",
"Private_Recipe_Help": "Recipe is only shown to you and people its shared with.",
"reusable_help_text": "Should the invite link be usable for more than one user.",
"open_data_help_text": "The Tandoor Open Data project provides community contributed data for Tandoor. This field is filled automatically when importing it and allows updates in the future.",
"Open_Data_Slug": "Open Data Slug",
"Add_Step": "Add Step",
"Keywords": "Keywords",
"Books": "Books",

20
vue/src/utils/models.js
View File

@ -418,6 +418,7 @@ export class Models {
create: {
params: [["name", "description", "category_to_supermarket"]],
form: {
show_help: true,
name: {
form_field: true,
type: "text",
@ -442,6 +443,14 @@ export class Models {
label: "Categories", // form.label always translated in utils.getForm()
placeholder: "",
},
open_data_slug: {
form_field: true,
type: "text",
field: "open_data_slug",
disabled: true,
label: "Open_Data_Slug",
help_text: "open_data_help_text",
},
},
config: {
function: "SupermarketWithCategories",
@ -572,8 +581,9 @@ export class Models {
},
},
create: {
params: [['base_amount', 'base_unit', 'converted_amount', 'converted_unit', 'food']],
params: [['base_amount', 'base_unit', 'converted_amount', 'converted_unit', 'food', 'open_data_slug']],
form: {
show_help: true,
// TODO add proper help texts for everything
base_amount: {
form_field: true,
@ -616,6 +626,14 @@ export class Models {
label: "Food",
multiple: false,
},
open_data_slug: {
form_field: true,
type: "text",
field: "open_data_slug",
disabled: true,
label: "Open_Data_Slug",
help_text: "open_data_help_text",
},
},