all types of unit conversion fixed
This commit is contained in:
parent
705fa18dd6
commit
9cf13e898c
@ -125,20 +125,38 @@ class OpenDataImporter:
|
|||||||
def import_supermarket(self):
|
def import_supermarket(self):
|
||||||
datatype = 'store'
|
datatype = 'store'
|
||||||
|
|
||||||
|
existing_data = {}
|
||||||
|
for obj in Supermarket.objects.filter(space=self.request.space, open_data_slug__isnull=False).values('pk', 'name', 'open_data_slug'):
|
||||||
|
existing_data[obj['open_data_slug']] = obj
|
||||||
|
|
||||||
|
update_list = []
|
||||||
|
create_list = []
|
||||||
|
|
||||||
self._update_slug_cache(SupermarketCategory, 'category')
|
self._update_slug_cache(SupermarketCategory, 'category')
|
||||||
insert_list = []
|
|
||||||
for k in list(self.data[datatype].keys()):
|
for k in list(self.data[datatype].keys()):
|
||||||
insert_list.append(Supermarket(
|
obj = Supermarket(
|
||||||
name=self.data[datatype][k]['name'],
|
name=self.data[datatype][k]['name'],
|
||||||
open_data_slug=k,
|
open_data_slug=k,
|
||||||
space=self.request.space
|
space=self.request.space
|
||||||
))
|
)
|
||||||
|
if obj.open_data_slug in existing_data:
|
||||||
|
obj.pk = existing_data[obj.open_data_slug]['pk']
|
||||||
|
update_list.append(obj)
|
||||||
|
else:
|
||||||
|
create_list.append(obj)
|
||||||
|
|
||||||
|
total_count = 0
|
||||||
|
if self.update_existing and len(update_list) > 0:
|
||||||
|
Supermarket.objects.bulk_update(update_list, ('name', 'open_data_slug'))
|
||||||
|
total_count += len(update_list)
|
||||||
|
|
||||||
|
if len(create_list) > 0:
|
||||||
|
Supermarket.objects.bulk_create(create_list, unique_fields=('space', 'name',), update_conflicts=True, update_fields=('open_data_slug',))
|
||||||
|
total_count += len(create_list)
|
||||||
|
|
||||||
# always add open data slug if matching supermarket is found, otherwise relation might fail
|
# always add open data slug if matching supermarket is found, otherwise relation might fail
|
||||||
supermarkets = Supermarket.objects.bulk_create(insert_list, unique_fields=('space', 'name',), update_conflicts=True, update_fields=('open_data_slug',))
|
|
||||||
self._update_slug_cache(Supermarket, 'store')
|
self._update_slug_cache(Supermarket, 'store')
|
||||||
|
|
||||||
insert_list = []
|
|
||||||
for k in list(self.data[datatype].keys()):
|
for k in list(self.data[datatype].keys()):
|
||||||
relations = []
|
relations = []
|
||||||
order = 0
|
order = 0
|
||||||
@ -154,68 +172,49 @@ class OpenDataImporter:
|
|||||||
|
|
||||||
SupermarketCategoryRelation.objects.bulk_create(relations, ignore_conflicts=True, unique_fields=('supermarket', 'category',))
|
SupermarketCategoryRelation.objects.bulk_create(relations, ignore_conflicts=True, unique_fields=('supermarket', 'category',))
|
||||||
|
|
||||||
return supermarkets
|
return total_count
|
||||||
|
|
||||||
def import_food(self):
|
def import_food(self):
|
||||||
identifier_list = []
|
|
||||||
datatype = 'food'
|
datatype = 'food'
|
||||||
for k in list(self.data[datatype].keys()):
|
|
||||||
identifier_list.append(self.data[datatype][k]['name'])
|
|
||||||
identifier_list.append(self.data[datatype][k]['plural_name'])
|
|
||||||
|
|
||||||
existing_objects_flat = []
|
|
||||||
existing_objects = {}
|
|
||||||
for f in Food.objects.filter(space=self.request.space).filter(name__in=identifier_list).values_list('id', 'name', 'plural_name'):
|
|
||||||
existing_objects_flat.append(f[1])
|
|
||||||
existing_objects_flat.append(f[2])
|
|
||||||
existing_objects[f[1]] = f
|
|
||||||
existing_objects[f[2]] = f
|
|
||||||
|
|
||||||
self._update_slug_cache(Unit, 'unit')
|
self._update_slug_cache(Unit, 'unit')
|
||||||
self._update_slug_cache(PropertyType, 'property')
|
self._update_slug_cache(PropertyType, 'property')
|
||||||
|
self._update_slug_cache(SupermarketCategory, 'category')
|
||||||
|
|
||||||
|
existing_data = {}
|
||||||
|
for obj in Food.objects.filter(space=self.request.space, open_data_slug__isnull=False).values('pk', 'name', 'open_data_slug'):
|
||||||
|
existing_data[obj['open_data_slug']] = obj
|
||||||
|
|
||||||
insert_list = []
|
|
||||||
insert_list_flat = []
|
|
||||||
update_list = []
|
update_list = []
|
||||||
update_field_list = []
|
create_list = []
|
||||||
|
|
||||||
for k in list(self.data[datatype].keys()):
|
for k in list(self.data[datatype].keys()):
|
||||||
if not (self.data[datatype][k]['name'] in existing_objects_flat or self.data[datatype][k]['plural_name'] in existing_objects_flat):
|
|
||||||
if not (self.data[datatype][k]['name'] in insert_list_flat or self.data[datatype][k]['plural_name'] in insert_list_flat):
|
obj = {
|
||||||
insert_list.append({'data': {
|
'name': self.data[datatype][k]['name'],
|
||||||
'name': self.data[datatype][k]['name'],
|
'plural_name': self.data[datatype][k]['plural_name'] if self.data[datatype][k]['plural_name'] != '' else None,
|
||||||
'plural_name': self.data[datatype][k]['plural_name'] if self.data[datatype][k]['plural_name'] != '' else None,
|
'supermarket_category_id': self.slug_id_cache['category'][self.data[datatype][k]['store_category']],
|
||||||
'supermarket_category_id': self.slug_id_cache['category'][self.data[datatype][k]['store_category']],
|
'fdc_id': re.sub(r'\D', '', self.data[datatype][k]['fdc_id']) if self.data[datatype][k]['fdc_id'] != '' else None,
|
||||||
'fdc_id': re.sub(r'\D', '', self.data[datatype][k]['fdc_id']) if self.data[datatype][k]['fdc_id'] != '' else None,
|
'open_data_slug': k,
|
||||||
'open_data_slug': k,
|
'space': self.request.space.id,
|
||||||
'space': self.request.space.id,
|
}
|
||||||
}})
|
|
||||||
# build a fake second flat array to prevent duplicate foods from being inserted.
|
if self.update_existing:
|
||||||
# trying to insert a duplicate would throw a db error :(
|
obj['space'] = self.request.space
|
||||||
insert_list_flat.append(self.data[datatype][k]['name'])
|
obj['pk'] = existing_data[obj['open_data_slug']]['pk']
|
||||||
insert_list_flat.append(self.data[datatype][k]['plural_name'])
|
obj = Food(**obj)
|
||||||
|
update_list.append(obj)
|
||||||
else:
|
else:
|
||||||
if self.data[datatype][k]['name'] in existing_objects:
|
create_list.append({'data': obj})
|
||||||
existing_food_id = existing_objects[self.data[datatype][k]['name']][0]
|
|
||||||
else:
|
|
||||||
existing_food_id = existing_objects[self.data[datatype][k]['plural_name']][0]
|
|
||||||
|
|
||||||
if self.update_existing:
|
total_count = 0
|
||||||
update_field_list = ['name', 'plural_name', 'preferred_unit_id', 'preferred_shopping_unit_id', 'supermarket_category_id', 'fdc_id', 'open_data_slug', ]
|
if self.update_existing and len(update_list) > 0:
|
||||||
update_list.append(Food(
|
Food.objects.bulk_update(update_list, ['name', 'plural_name', 'preferred_unit_id', 'preferred_shopping_unit_id', 'supermarket_category_id', 'fdc_id', 'open_data_slug', ])
|
||||||
id=existing_food_id,
|
total_count += len(update_list)
|
||||||
name=self.data[datatype][k]['name'],
|
|
||||||
plural_name=self.data[datatype][k]['plural_name'] if self.data[datatype][k]['plural_name'] != '' else None,
|
|
||||||
supermarket_category_id=self.slug_id_cache['category'][self.data[datatype][k]['store_category']],
|
|
||||||
fdc_id=re.sub(r'\D', '', self.data[datatype][k]['fdc_id']) if self.data[datatype][k]['fdc_id'] != '' else None,
|
|
||||||
open_data_slug=k,
|
|
||||||
))
|
|
||||||
else:
|
|
||||||
update_field_list = ['open_data_slug', ]
|
|
||||||
update_list.append(Food(id=existing_food_id, open_data_slug=k, ))
|
|
||||||
|
|
||||||
Food.load_bulk(insert_list, None)
|
if len(create_list) > 0:
|
||||||
if len(update_list) > 0:
|
Food.load_bulk(create_list, None)
|
||||||
Food.objects.bulk_update(update_list, update_field_list)
|
total_count += len(create_list)
|
||||||
|
|
||||||
self._update_slug_cache(Food, 'food')
|
self._update_slug_cache(Food, 'food')
|
||||||
|
|
||||||
@ -243,16 +242,25 @@ class OpenDataImporter:
|
|||||||
|
|
||||||
FoodProperty.objects.bulk_create(property_food_relation_list, ignore_conflicts=True, unique_fields=('food_id', 'property_id',))
|
FoodProperty.objects.bulk_create(property_food_relation_list, ignore_conflicts=True, unique_fields=('food_id', 'property_id',))
|
||||||
|
|
||||||
return insert_list + update_list
|
return total_count
|
||||||
|
|
||||||
def import_conversion(self):
|
def import_conversion(self):
|
||||||
datatype = 'conversion'
|
datatype = 'conversion'
|
||||||
|
|
||||||
insert_list = []
|
self._update_slug_cache(Food, 'food')
|
||||||
|
self._update_slug_cache(Unit, 'unit')
|
||||||
|
|
||||||
|
existing_data = {}
|
||||||
|
for obj in UnitConversion.objects.filter(space=self.request.space, open_data_slug__isnull=False).values('pk', 'open_data_slug'):
|
||||||
|
existing_data[obj['open_data_slug']] = obj
|
||||||
|
|
||||||
|
update_list = []
|
||||||
|
create_list = []
|
||||||
|
|
||||||
for k in list(self.data[datatype].keys()):
|
for k in list(self.data[datatype].keys()):
|
||||||
# try catch here because sometimes key "k" is not set for he food cache
|
# try catch here because sometimes key "k" is not set for he food cache
|
||||||
try:
|
try:
|
||||||
insert_list.append(UnitConversion(
|
obj = UnitConversion(
|
||||||
base_amount=self.data[datatype][k]['base_amount'],
|
base_amount=self.data[datatype][k]['base_amount'],
|
||||||
base_unit_id=self.slug_id_cache['unit'][self.data[datatype][k]['base_unit']],
|
base_unit_id=self.slug_id_cache['unit'][self.data[datatype][k]['base_unit']],
|
||||||
converted_amount=self.data[datatype][k]['converted_amount'],
|
converted_amount=self.data[datatype][k]['converted_amount'],
|
||||||
@ -261,8 +269,23 @@ class OpenDataImporter:
|
|||||||
open_data_slug=k,
|
open_data_slug=k,
|
||||||
space=self.request.space,
|
space=self.request.space,
|
||||||
created_by=self.request.user,
|
created_by=self.request.user,
|
||||||
))
|
)
|
||||||
|
|
||||||
|
if obj.open_data_slug in existing_data:
|
||||||
|
obj.pk = existing_data[obj.open_data_slug]['pk']
|
||||||
|
update_list.append(obj)
|
||||||
|
else:
|
||||||
|
create_list.append(obj)
|
||||||
except KeyError:
|
except KeyError:
|
||||||
print(str(k) + ' is not in self.slug_id_cache["food"]')
|
print(str(k) + ' is not in self.slug_id_cache["food"]')
|
||||||
|
|
||||||
return UnitConversion.objects.bulk_create(insert_list, ignore_conflicts=True, unique_fields=('space', 'base_unit', 'converted_unit', 'food', 'open_data_slug'))
|
total_count = 0
|
||||||
|
if self.update_existing and len(update_list) > 0:
|
||||||
|
UnitConversion.objects.bulk_update(update_list, ('base_unit', 'base_amount', 'converted_unit', 'converted_amount', 'food',))
|
||||||
|
total_count += len(update_list)
|
||||||
|
|
||||||
|
if len(create_list) > 0:
|
||||||
|
UnitConversion.objects.bulk_create(create_list, ignore_conflicts=True, unique_fields=('space', 'base_unit', 'converted_unit', 'food', 'open_data_slug'))
|
||||||
|
total_count += len(create_list)
|
||||||
|
|
||||||
|
return total_count
|
||||||
|
@ -1621,14 +1621,13 @@ class ImportOpenData(APIView):
|
|||||||
if selected_datatypes['category']['selected']:
|
if selected_datatypes['category']['selected']:
|
||||||
response_obj['category'] = data_importer.import_category()
|
response_obj['category'] = data_importer.import_category()
|
||||||
if selected_datatypes['property']['selected']:
|
if selected_datatypes['property']['selected']:
|
||||||
print('importin properties')
|
|
||||||
response_obj['property'] = data_importer.import_property()
|
response_obj['property'] = data_importer.import_property()
|
||||||
if selected_datatypes['store']['selected']:
|
if selected_datatypes['store']['selected']:
|
||||||
response_obj['store'] = len(data_importer.import_supermarket())
|
response_obj['store'] = data_importer.import_supermarket()
|
||||||
if selected_datatypes['food']['selected']:
|
if selected_datatypes['food']['selected']:
|
||||||
response_obj['food'] = len(data_importer.import_food())
|
response_obj['food'] = data_importer.import_food()
|
||||||
if selected_datatypes['conversion']['selected']:
|
if selected_datatypes['conversion']['selected']:
|
||||||
response_obj['conversion'] = len(data_importer.import_conversion())
|
response_obj['conversion'] = data_importer.import_conversion()
|
||||||
|
|
||||||
return Response(response_obj)
|
return Response(response_obj)
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user