improved importer merging behavior

cookbook/helper/open_data_importer.py

@@ -7,10 +7,12 @@ class OpenDataImporter:
     request = None
     data = {}
     slug_id_cache = {}
+    update_existing = False
 
-    def __init__(self, request, data):
+    def __init__(self, request, data, update_existing=False):
         self.request = request
         self.data = data
+        self.update_existing = update_existing
 
     def _update_slug_cache(self, object_class, datatype):
         self.slug_id_cache[datatype] = dict(object_class.objects.filter(space=self.request.space, open_data_slug__isnull=False).values_list('open_data_slug', 'id', ))
@@ -80,37 +82,36 @@ class OpenDataImporter:
         return len(insert_list)
 
     def import_supermarket(self):
-        identifier_list = []
         datatype = 'supermarket'
-        for k in list(self.data[datatype].keys()):
-            identifier_list.append(self.data[datatype][k]['name'])
-
-        existing_objects = Supermarket.objects.filter(space=self.request.space).filter(name__in=identifier_list).values_list('name', flat=True)
 
         self._update_slug_cache(SupermarketCategory, 'category')
+        insert_list = []
+        for k in list(self.data[datatype].keys()):
+            insert_list.append(Supermarket(
+                name=self.data[datatype][k]['name'],
+                open_data_slug=k,
+                space=self.request.space
+            ))
+
+        # always add open data slug if matching supermarket is found, otherwise relation might fail
+        Supermarket.objects.bulk_create(insert_list, unique_fields=('space', 'name',), update_conflicts=True, update_fields=('open_data_slug',))
+        self._update_slug_cache(Supermarket, 'supermarket')
 
         insert_list = []
         for k in list(self.data[datatype].keys()):
-            if not (self.data[datatype][k]['name'] in existing_objects):  # TODO on large datasets see if bulk creating supermarkets and then relations as well is better
+            relations = []
-                supermarket = Supermarket.objects.create(
+            order = 0
-                    name=self.data[datatype][k]['name'],
+            for c in self.data[datatype][k]['categories']:
-                    open_data_slug=k,
+                relations.append(
-                    space=self.request.space
+                    SupermarketCategoryRelation(
-                )
+                        supermarket_id=self.slug_id_cache[datatype][k],
-
+                        category_id=self.slug_id_cache['category'][c],
-                relations = []
+                        order=order,
-                order = 0
-                for c in self.data[datatype][k]['categories']:
-                    relations.append(
-                        SupermarketCategoryRelation(
-                            supermarket=supermarket,
-                            category_id=self.slug_id_cache['category'][c],
-                            order=order,
-                        )
                     )
-                    order += 1
+                )
+                order += 1
 
-                SupermarketCategoryRelation.objects.bulk_create(relations)
+            SupermarketCategoryRelation.objects.bulk_create(relations, ignore_conflicts=True, unique_fields=('supermarket', 'category',))
 
         return len(insert_list)
 
@@ -172,8 +173,8 @@ class OpenDataImporter:
                     created_by=self.request.user,
                 ))
 
-        FoodProperty.objects.bulk_create(food_property_list, ignore_conflicts=True, unique_fields=('space', 'food', 'property_type'))
+        FoodProperty.objects.bulk_create(food_property_list, ignore_conflicts=True, unique_fields=('space', 'food', 'property_type',))
-        Automation.objects.bulk_create(alias_list)
+        Automation.objects.bulk_create(alias_list, ignore_conflicts=True, unique_fields=('space', 'param_1', 'param_2',))
         return len(insert_list)
 
     def import_conversion(self):

cookbook/migrations/0194_supermarketcategoryrelation_unique_sm_category_relation.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.1.7 on 2023-05-04 13:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('cookbook', '0193_unitconversion_f_unique_conversion_per_space'),
+    ]
+
+    operations = [
+        migrations.AddConstraint(
+            model_name='supermarketcategoryrelation',
+            constraint=models.UniqueConstraint(fields=('supermarket', 'category'), name='unique_sm_category_relation'),
+        ),
+    ]

cookbook/models.py

@@ -498,6 +498,9 @@ class SupermarketCategoryRelation(models.Model, PermissionModelMixin):
         return 'supermarket', 'space'
 
     class Meta:
+        constraints = [
+            models.UniqueConstraint(fields=['supermarket', 'category'], name='unique_sm_category_relation')
+        ]
         ordering = ('order',)
 
 

cookbook/serializer.py

@@ -497,7 +497,7 @@ class SupermarketSerializer(UniqueFieldsMixin, SpacedModelSerializer):
 
     class Meta:
         model = Supermarket
-        fields = ('id', 'name', 'description', 'category_to_supermarket')
+        fields = ('id', 'name', 'description', 'category_to_supermarket', 'open_data_slug')
 
 
 class RecipeSimpleSerializer(WritableNestedModelSerializer):
@@ -737,7 +737,7 @@ class UnitConversionSerializer(WritableNestedModelSerializer):
 
     class Meta:
         model = UnitConversion
-        fields = ('id', 'base_amount', 'base_unit', 'converted_amount', 'converted_unit', 'food')
+        fields = ('id', 'base_amount', 'base_unit', 'converted_amount', 'converted_unit', 'food', 'open_data_slug')
 
 
 class FoodPropertyTypeSerializer(serializers.ModelSerializer):

cookbook/views/api.py

@@ -1438,7 +1438,7 @@ class ImportOpenData(APIView):
         response = requests.get(f'https://raw.githubusercontent.com/TandoorRecipes/open-tandoor-data/main/build/{selected_version}.json')  # TODO catch 404, timeout, ...
         data = json.loads(response.content)
 
-        data_importer = OpenDataImporter(request, data)
+        data_importer = OpenDataImporter(request, data, update_existing=True)
         data_importer.import_units()
         data_importer.import_category()
         data_importer.import_property()

vue/src/components/Modals/GenericModalForm.vue

@@ -9,7 +9,7 @@
                 <p v-if="visibleCondition(f, 'instruction')">{{ f.label }}</p>
                 <lookup-input v-if="visibleCondition(f, 'lookup')" :form="f" :model="listModel(f.list)" @change="storeValue" :help="showHelp && f.help" />
                 <checkbox-input class="mb-3" v-if="visibleCondition(f, 'checkbox')" :label="f.label" :value="f.value" :field="f.field" :help="showHelp && f.help" />
-                <text-input v-if="visibleCondition(f, 'text')" :label="f.label" :value="f.value" :field="f.field" :placeholder="f.placeholder" :help="showHelp && f.help" :subtitle="f.subtitle" />
+                <text-input v-if="visibleCondition(f, 'text')" :label="f.label" :value="f.value" :field="f.field" :placeholder="f.placeholder" :help="showHelp && f.help" :subtitle="f.subtitle" :disabled="f.disabled"/>
                 <choice-input v-if="visibleCondition(f, 'choice')" :label="f.label" :value="f.value" :field="f.field" :options="f.options" :placeholder="f.placeholder" />
                 <emoji-input v-if="visibleCondition(f, 'emoji')" :label="f.label" :value="f.value" :field="f.field" @change="storeValue" />
                 <file-input v-if="visibleCondition(f, 'file')" :label="f.label" :value="f.value" :field="f.field" @change="storeValue" />

vue/src/components/Modals/TextInput.vue

@@ -1,7 +1,7 @@
 <template>
     <div>
         <b-form-group v-bind:label="label" class="mb-3">
-            <b-form-input v-model="new_value" type="text" :placeholder="placeholder"></b-form-input>
+            <b-form-input v-model="new_value" type="text" :placeholder="placeholder" :disabled="disabled"></b-form-input>
             <em v-if="help" class="small text-muted">{{ help }}</em>
             <small v-if="subtitle" class="text-muted">{{ subtitle }}</small>
         </b-form-group>
@@ -18,6 +18,7 @@ export default {
         placeholder: { type: String, default: "You Should Add Placeholder Text" },
         help: { type: String, default: undefined },
         subtitle: { type: String, default: undefined },
+        disabled: { type: Boolean, default: false }
     },
     data() {
         return {

vue/src/locales/en.json

@@ -76,6 +76,8 @@
     "Private_Recipe": "Private Recipe",
     "Private_Recipe_Help": "Recipe is only shown to you and people its shared with.",
     "reusable_help_text": "Should the invite link be usable for more than one user.",
+    "open_data_help_text": "The Tandoor Open Data project provides community contributed data for Tandoor. This field is filled automatically when importing it and allows updates in the future.",
+    "Open_Data_Slug": "Open Data Slug",
     "Add_Step": "Add Step",
     "Keywords": "Keywords",
     "Books": "Books",

vue/src/utils/models.js

@@ -418,6 +418,7 @@ export class Models {
         create: {
             params: [["name", "description", "category_to_supermarket"]],
             form: {
+                show_help: true,
                 name: {
                     form_field: true,
                     type: "text",
@@ -442,6 +443,14 @@ export class Models {
                     label: "Categories", // form.label always translated in utils.getForm()
                     placeholder: "",
                 },
+                open_data_slug: {
+                    form_field: true,
+                    type: "text",
+                    field: "open_data_slug",
+                    disabled: true,
+                    label: "Open_Data_Slug",
+                    help_text: "open_data_help_text",
+                },
             },
             config: {
                 function: "SupermarketWithCategories",
@@ -572,8 +581,9 @@ export class Models {
             },
         },
         create: {
-            params: [['base_amount', 'base_unit', 'converted_amount', 'converted_unit', 'food']],
+            params: [['base_amount', 'base_unit', 'converted_amount', 'converted_unit', 'food', 'open_data_slug']],
             form: {
+                show_help: true,
                 // TODO add proper help texts for everything
                 base_amount: {
                     form_field: true,
@@ -616,6 +626,14 @@ export class Models {
                     label: "Food",
                     multiple: false,
                 },
+                open_data_slug: {
+                    form_field: true,
+                    type: "text",
+                    field: "open_data_slug",
+                    disabled: true,
+                    label: "Open_Data_Slug",
+                    help_text: "open_data_help_text",
+                },
 
             },