fixed open data property type and category

cookbook/helper/open_data_importer.py

@@ -23,26 +23,25 @@ class OpenDataImporter:
         datatype = 'unit'
 
         existing_data = {}
-        for u in Unit.objects.filter(space=self.request.space, open_data_slug__isnull=False).values('pk', 'name', 'open_data_slug'):
+        for obj in Unit.objects.filter(space=self.request.space, open_data_slug__isnull=False).values('pk', 'name', 'open_data_slug'):
-            existing_data[u['open_data_slug']] = u
+            existing_data[obj['open_data_slug']] = obj
 
         update_list = []
         create_list = []
 
-        insert_list = []
         for u in list(self.data[datatype].keys()):
-            unit = Unit(
+            obj = Unit(
                 name=self.data[datatype][u]['name'],
                 plural_name=self.data[datatype][u]['plural_name'],
                 base_unit=self.data[datatype][u]['base_unit'] if self.data[datatype][u]['base_unit'] != '' else None,
                 open_data_slug=u,
                 space=self.request.space
             )
-            if unit.open_data_slug in existing_data:
+            if obj.open_data_slug in existing_data:
-                unit.pk = existing_data[unit.open_data_slug]['pk']
+                obj.pk = existing_data[obj.open_data_slug]['pk']
-                update_list.append(unit)
+                update_list.append(obj)
             else:
-                create_list.append(unit)
+                create_list.append(obj)
 
         total_count = 0
         if self.update_existing and len(update_list) > 0:
@@ -58,29 +57,70 @@ class OpenDataImporter:
     def import_category(self):
         datatype = 'category'
 
-        insert_list = []
+        existing_data = {}
+        for obj in SupermarketCategory.objects.filter(space=self.request.space, open_data_slug__isnull=False).values('pk', 'name', 'open_data_slug'):
+            existing_data[obj['open_data_slug']] = obj
+
+        update_list = []
+        create_list = []
+
         for k in list(self.data[datatype].keys()):
-            insert_list.append(SupermarketCategory(
+            obj = SupermarketCategory(
                 name=self.data[datatype][k]['name'],
                 open_data_slug=k,
                 space=self.request.space
-            ))
+            )
 
-        return SupermarketCategory.objects.bulk_create(insert_list, update_conflicts=True, update_fields=('open_data_slug',), unique_fields=('space', 'name',))
+            if obj.open_data_slug in existing_data:
+                obj.pk = existing_data[obj.open_data_slug]['pk']
+                update_list.append(obj)
+            else:
+                create_list.append(obj)
+
+        total_count = 0
+        if self.update_existing and len(update_list) > 0:
+            SupermarketCategory.objects.bulk_update(update_list, ('name', 'open_data_slug'))
+            total_count += len(update_list)
+
+        if len(create_list) > 0:
+            SupermarketCategory.objects.bulk_create(create_list, update_conflicts=True, update_fields=('open_data_slug',), unique_fields=('space', 'name',))
+            total_count += len(create_list)
+
+        return total_count
 
     def import_property(self):
         datatype = 'property'
 
-        insert_list = []
+        existing_data = {}
+        for obj in PropertyType.objects.filter(space=self.request.space, open_data_slug__isnull=False).values('pk', 'name', 'open_data_slug'):
+            existing_data[obj['open_data_slug']] = obj
+
+        update_list = []
+        create_list = []
+
         for k in list(self.data[datatype].keys()):
-            insert_list.append(PropertyType(
+            obj = PropertyType(
                 name=self.data[datatype][k]['name'],
                 unit=self.data[datatype][k]['unit'],
                 open_data_slug=k,
                 space=self.request.space
-            ))
+            )
+            if obj.open_data_slug in existing_data:
+                obj.pk = existing_data[obj.open_data_slug]['pk']
+                update_list.append(obj)
+            else:
+                create_list.append(obj)
 
-        return PropertyType.objects.bulk_create(insert_list, update_conflicts=True, update_fields=('open_data_slug',), unique_fields=('space', 'name',))
+        total_count = 0
+        if self.update_existing and len(update_list) > 0:
+            PropertyType.objects.bulk_update(update_list, ('name', 'open_data_slug'))
+            total_count += len(update_list)
+
+        if len(create_list) > 0:
+            PropertyType.objects.bulk_create(create_list, update_conflicts=True, update_fields=('open_data_slug',), unique_fields=('space', 'name',))
+            total_count += len(create_list)
+
+        return total_count
 
     def import_supermarket(self):
         datatype = 'store'

cookbook/views/api.py

@@ -1605,6 +1605,7 @@ class ImportOpenData(APIView):
         # TODO validate data
         print(request.data)
         selected_version = request.data['selected_version']
+        selected_datatypes = request.data['selected_datatypes']
         update_existing = str2bool(request.data['update_existing'])
         use_metric = str2bool(request.data['use_metric'])
 
@@ -1614,12 +1615,20 @@ class ImportOpenData(APIView):
         response_obj = {}
 
         data_importer = OpenDataImporter(request, data, update_existing=update_existing, use_metric=use_metric)
-        response_obj['unit'] = len(data_importer.import_units())
+
-        response_obj['category'] = len(data_importer.import_category())
+        if selected_datatypes['unit']['selected']:
-        response_obj['property'] = len(data_importer.import_property())
+            response_obj['unit'] = data_importer.import_units()
-        response_obj['store'] = len(data_importer.import_supermarket())
+        if selected_datatypes['category']['selected']:
-        response_obj['food'] = len(data_importer.import_food())
+            response_obj['category'] = data_importer.import_category()
-        response_obj['conversion'] = len(data_importer.import_conversion())
+        if selected_datatypes['property']['selected']:
+            print('importin properties')
+            response_obj['property'] = data_importer.import_property()
+        if selected_datatypes['store']['selected']:
+            response_obj['store'] = len(data_importer.import_supermarket())
+        if selected_datatypes['food']['selected']:
+            response_obj['food'] = len(data_importer.import_food())
+        if selected_datatypes['conversion']['selected']:
+            response_obj['conversion'] = len(data_importer.import_conversion())
 
         return Response(response_obj)
 

vue/src/components/OpenDataImportComponent.vue

@@ -4,7 +4,7 @@
 
         <div v-if="metadata !== undefined">
             {{ $t('Data_Import_Info') }}
-            <a href="https://github.com/TandoorRecipes/open-tandoor-data" target="_blank" rel="noreferrer nofollow">{{$t('Learn_More')}}</a>
+            <a href="https://github.com/TandoorRecipes/open-tandoor-data" target="_blank" rel="noreferrer nofollow">{{ $t('Learn_More') }}</a>
 
 
             <select class="form-control" v-model="selected_version">
@@ -18,13 +18,17 @@
             <div v-if="selected_version !== undefined" class="mt-3">
                 <table class="table">
                     <tr>
+                        <th>{{ $t('Import') }}</th>
                         <th>{{ $t('Datatype') }}</th>
                         <th>{{ $t('Number of Objects') }}</th>
                         <th>{{ $t('Imported') }}</th>
                     </tr>
-                    <tr v-for="d in metadata.datatypes" v-bind:key="d">
+                    <tr v-for="d in datatypes" v-bind:key="d.name">
-                        <td>{{ $t(d.charAt(0).toUpperCase() + d.slice(1)) }}</td>
+                        <td>
-                        <td>{{ metadata[selected_version][d] }}</td>
+                            <b-checkbox v-model="d.selected"></b-checkbox>
+                        </td>
+                        <td>{{ $t(d.name.charAt(0).toUpperCase() + d.name.slice(1)) }}</td>
+                        <td>{{ metadata[selected_version][d.name] }}</td>
                         <td>
                             <template v-if="import_count !== undefined">{{ import_count[d] }}</template>
                         </td>
@@ -59,6 +63,7 @@ export default {
     data() {
         return {
             metadata: undefined,
+            datatypes: {},
             selected_version: undefined,
             update_existing: true,
             use_metric: true,
@@ -70,6 +75,12 @@ export default {
 
         axios.get(resolveDjangoUrl('api_import_open_data')).then(r => {
             this.metadata = r.data
+            for (let i in this.metadata.datatypes) {
+                this.datatypes[this.metadata.datatypes[i]] = {
+                    name: this.metadata.datatypes[i],
+                    selected: false,
+                }
+            }
         }).catch(err => {
             StandardToasts.makeStandardToast(this, StandardToasts.FAIL_FETCH, err)
         })
@@ -78,7 +89,7 @@ export default {
         doImport: function () {
             axios.post(resolveDjangoUrl('api_import_open_data'), {
                 'selected_version': this.selected_version,
-                'selected_datatypes': this.metadata.datatypes,
+                'selected_datatypes': this.datatypes,
                 'update_existing': this.update_existing,
                 'use_metric': this.use_metric,
             }).then(r => {