From 151f43b0d501ccfbba36342af44764d8282b7beb Mon Sep 17 00:00:00 2001 From: vabene1111 Date: Thu, 17 Jun 2021 13:38:57 +0200 Subject: [PATCH] lots of importer adjustments --- cookbook/integration/Pepperplate.py | 4 +- cookbook/integration/cheftap.py | 4 +- cookbook/integration/chowdown.py | 4 +- cookbook/integration/domestica.py | 4 +- cookbook/integration/integration.py | 47 ++++++++++++++++------ cookbook/integration/mealie.py | 36 +++++++++++------ cookbook/integration/mealmaster.py | 4 +- cookbook/integration/nextcloud_cookbook.py | 4 +- cookbook/integration/openeats.py | 4 +- cookbook/integration/paprika.py | 4 +- cookbook/integration/recettetek.py | 4 +- cookbook/integration/recipekeeper.py | 4 +- cookbook/integration/recipesage.py | 4 +- cookbook/integration/rezkonv.py | 4 +- cookbook/integration/safron.py | 4 +- cookbook/models.py | 4 ++ cookbook/serializer.py | 2 +- cookbook/views/data.py | 4 +- 18 files changed, 90 insertions(+), 55 deletions(-) diff --git a/cookbook/integration/Pepperplate.py b/cookbook/integration/Pepperplate.py index f5d463dd..76615570 100644 --- a/cookbook/integration/Pepperplate.py +++ b/cookbook/integration/Pepperplate.py @@ -40,7 +40,7 @@ class Pepperplate(Integration): recipe = Recipe.objects.create(name=title, description=description, created_by=self.request.user, internal=True, space=self.request.space) step = Step.objects.create( - instruction='\n'.join(directions) + '\n\n' + instruction='\n'.join(directions) + '\n\n', space=self.request.space, ) for ingredient in ingredients: @@ -49,7 +49,7 @@ class Pepperplate(Integration): f = get_food(ingredient, self.request.space) u = get_unit(unit, self.request.space) step.ingredients.add(Ingredient.objects.create( - food=f, unit=u, amount=amount, note=note + food=f, unit=u, amount=amount, note=note, space=self.request.space, )) recipe.steps.add(step) diff --git a/cookbook/integration/cheftap.py b/cookbook/integration/cheftap.py index 095ba103..4dd67830 100644 --- a/cookbook/integration/cheftap.py +++ b/cookbook/integration/cheftap.py @@ -38,7 +38,7 @@ class ChefTap(Integration): recipe = Recipe.objects.create(name=title, created_by=self.request.user, internal=True, space=self.request.space, ) - step = Step.objects.create(instruction='\n'.join(directions)) + step = Step.objects.create(instruction='\n'.join(directions), space=self.request.space,) if source_url != '': step.instruction += '\n' + source_url @@ -50,7 +50,7 @@ class ChefTap(Integration): f = get_food(ingredient, self.request.space) u = get_unit(unit, self.request.space) step.ingredients.add(Ingredient.objects.create( - food=f, unit=u, amount=amount, note=note + food=f, unit=u, amount=amount, note=note, space=self.request.space, )) recipe.steps.add(step) diff --git a/cookbook/integration/chowdown.py b/cookbook/integration/chowdown.py index b23cafdb..4b36e3b2 100644 --- a/cookbook/integration/chowdown.py +++ b/cookbook/integration/chowdown.py @@ -55,7 +55,7 @@ class Chowdown(Integration): recipe.keywords.add(keyword) step = Step.objects.create( - instruction='\n'.join(directions) + '\n\n' + '\n'.join(descriptions) + instruction='\n'.join(directions) + '\n\n' + '\n'.join(descriptions), space=self.request.space, ) for ingredient in ingredients: @@ -63,7 +63,7 @@ class Chowdown(Integration): f = get_food(ingredient, self.request.space) u = get_unit(unit, self.request.space) step.ingredients.add(Ingredient.objects.create( - food=f, unit=u, amount=amount, note=note + food=f, unit=u, amount=amount, note=note, space=self.request.space, )) recipe.steps.add(step) diff --git a/cookbook/integration/domestica.py b/cookbook/integration/domestica.py index 50d92d59..da55e7c3 100644 --- a/cookbook/integration/domestica.py +++ b/cookbook/integration/domestica.py @@ -28,7 +28,7 @@ class Domestica(Integration): recipe.save() step = Step.objects.create( - instruction=file['directions'] + instruction=file['directions'], space=self.request.space, ) if file['source'] != '': @@ -40,7 +40,7 @@ class Domestica(Integration): f = get_food(ingredient, self.request.space) u = get_unit(unit, self.request.space) step.ingredients.add(Ingredient.objects.create( - food=f, unit=u, amount=amount, note=note + food=f, unit=u, amount=amount, note=note, space=self.request.space, )) recipe.steps.add(step) diff --git a/cookbook/integration/integration.py b/cookbook/integration/integration.py index b25a1e83..b7ee9f29 100644 --- a/cookbook/integration/integration.py +++ b/cookbook/integration/integration.py @@ -109,35 +109,52 @@ class Integration: for f in files: if 'RecipeKeeper' in f['name']: import_zip = ZipFile(f['file']) + file_list = [] for z in import_zip.filelist: if self.import_file_name_filter(z): - data_list = self.split_recipe_file(import_zip.read(z.filename).decode('utf-8')) - for d in data_list: - recipe = self.get_recipe_from_file(d) - recipe.keywords.add(self.keyword) - il.msg += f'{recipe.pk} - {recipe.name} \n' - self.handle_duplicates(recipe, import_duplicates) + file_list.append(z) + il.total_recipes += len(file_list) + + for z in file_list: + data_list = self.split_recipe_file(import_zip.read(z.filename).decode('utf-8')) + for d in data_list: + recipe = self.get_recipe_from_file(d) + recipe.keywords.add(self.keyword) + il.msg += f'{recipe.pk} - {recipe.name} \n' + self.handle_duplicates(recipe, import_duplicates) + il.imported_recipes += 1 + il.save() import_zip.close() elif '.zip' in f['name'] or '.paprikarecipes' in f['name']: import_zip = ZipFile(f['file']) + file_list = [] for z in import_zip.filelist: if self.import_file_name_filter(z): - try: - recipe = self.get_recipe_from_file(BytesIO(import_zip.read(z.filename))) - recipe.keywords.add(self.keyword) - il.msg += f'{recipe.pk} - {recipe.name} \n' - self.handle_duplicates(recipe, import_duplicates) - except Exception as e: - il.msg += f'-------------------- \n ERROR \n{e}\n--------------------\n' + file_list.append(z) + il.total_recipes += len(file_list) + + for z in file_list: + try: + recipe = self.get_recipe_from_file(BytesIO(import_zip.read(z.filename))) + recipe.keywords.add(self.keyword) + il.msg += f'{recipe.pk} - {recipe.name} \n' + self.handle_duplicates(recipe, import_duplicates) + il.imported_recipes += 1 + il.save() + except Exception as e: + il.msg += f'-------------------- \n ERROR \n{e}\n--------------------\n' import_zip.close() elif '.json' in f['name'] or '.txt' in f['name']: data_list = self.split_recipe_file(f['file']) + il.total_recipes += len(data_list) for d in data_list: try: recipe = self.get_recipe_from_file(d) recipe.keywords.add(self.keyword) il.msg += f'{recipe.pk} - {recipe.name} \n' self.handle_duplicates(recipe, import_duplicates) + il.imported_recipes += 1 + il.save() except Exception as e: il.msg += f'-------------------- \n ERROR \n{e}\n--------------------\n' elif '.rtk' in f['name']: @@ -145,12 +162,16 @@ class Integration: for z in import_zip.filelist: if self.import_file_name_filter(z): data_list = self.split_recipe_file(import_zip.read(z.filename).decode('utf-8')) + il.total_recipes += len(data_list) + for d in data_list: try: recipe = self.get_recipe_from_file(d) recipe.keywords.add(self.keyword) il.msg += f'{recipe.pk} - {recipe.name} \n' self.handle_duplicates(recipe, import_duplicates) + il.imported_recipes += 1 + il.save() except Exception as e: il.msg += f'-------------------- \n ERROR \n{e}\n--------------------\n' import_zip.close() diff --git a/cookbook/integration/mealie.py b/cookbook/integration/mealie.py index 722a26be..e1144472 100644 --- a/cookbook/integration/mealie.py +++ b/cookbook/integration/mealie.py @@ -12,7 +12,7 @@ from cookbook.models import Recipe, Step, Food, Unit, Ingredient class Mealie(Integration): def import_file_name_filter(self, zip_info_object): - return re.match(r'^recipes/([A-Za-z\d-])+.json$', zip_info_object.filename) + return re.match(r'^recipes/([A-Za-z\d-])+/([A-Za-z\d-])+.json$', zip_info_object.filename) def get_recipe_from_file(self, file): recipe_json = json.loads(file.getvalue().decode("utf-8")) @@ -26,9 +26,9 @@ class Mealie(Integration): # TODO parse times (given in PT2H3M ) ingredients_added = False - for s in recipe_json['recipeInstructions']: + for s in recipe_json['recipe_instructions']: step = Step.objects.create( - instruction=s['text'] + instruction=s['text'], space=self.request.space, ) if not ingredients_added: ingredients_added = True @@ -36,21 +36,31 @@ class Mealie(Integration): if len(recipe_json['description'].strip()) > 500: step.instruction = recipe_json['description'].strip() + '\n\n' + step.instruction - for ingredient in recipe_json['recipeIngredient']: - amount, unit, ingredient, note = parse(ingredient) - f = get_food(ingredient, self.request.space) - u = get_unit(unit, self.request.space) - step.ingredients.add(Ingredient.objects.create( - food=f, unit=u, amount=amount, note=note - )) + for ingredient in recipe_json['recipe_ingredient']: + try: + if ingredient['food']: + f = get_food(ingredient['food'], self.request.space) + u = get_unit(ingredient['unit'], self.request.space) + amount = ingredient['quantity'] + note = ingredient['note'] + else: + amount, unit, ingredient, note = parse(ingredient['note']) + f = get_food(ingredient, self.request.space) + u = get_unit(unit, self.request.space) + step.ingredients.add(Ingredient.objects.create( + food=f, unit=u, amount=amount, note=note, space=self.request.space, + )) + except: + pass recipe.steps.add(step) for f in self.files: if '.zip' in f['name']: import_zip = ZipFile(f['file']) - for z in import_zip.filelist: - if re.match(f'^images/{recipe_json["slug"]}.jpg$', z.filename): - self.import_recipe_image(recipe, BytesIO(import_zip.read(z.filename)), filetype=get_filetype(z.filename)) + try: + self.import_recipe_image(recipe, BytesIO(import_zip.read(f'recipes/{recipe_json["slug"]}/images/min-original.webp')), filetype=get_filetype(f'recipes/{recipe_json["slug"]}/images/original')) + except: + pass return recipe diff --git a/cookbook/integration/mealmaster.py b/cookbook/integration/mealmaster.py index 01f58331..0baf4157 100644 --- a/cookbook/integration/mealmaster.py +++ b/cookbook/integration/mealmaster.py @@ -44,7 +44,7 @@ class MealMaster(Integration): recipe.keywords.add(keyword) step = Step.objects.create( - instruction='\n'.join(directions) + '\n\n' + instruction='\n'.join(directions) + '\n\n', space=self.request.space, ) for ingredient in ingredients: @@ -53,7 +53,7 @@ class MealMaster(Integration): f = get_food(ingredient, self.request.space) u = get_unit(unit, self.request.space) step.ingredients.add(Ingredient.objects.create( - food=f, unit=u, amount=amount, note=note + food=f, unit=u, amount=amount, note=note, space=self.request.space, )) recipe.steps.add(step) diff --git a/cookbook/integration/nextcloud_cookbook.py b/cookbook/integration/nextcloud_cookbook.py index 538fe1a3..2e668f7e 100644 --- a/cookbook/integration/nextcloud_cookbook.py +++ b/cookbook/integration/nextcloud_cookbook.py @@ -30,7 +30,7 @@ class NextcloudCookbook(Integration): ingredients_added = False for s in recipe_json['recipeInstructions']: step = Step.objects.create( - instruction=s + instruction=s, space=self.request.space, ) if not ingredients_added: if len(recipe_json['description'].strip()) > 500: @@ -43,7 +43,7 @@ class NextcloudCookbook(Integration): f = get_food(ingredient, self.request.space) u = get_unit(unit, self.request.space) step.ingredients.add(Ingredient.objects.create( - food=f, unit=u, amount=amount, note=note + food=f, unit=u, amount=amount, note=note, space=self.request.space, )) recipe.steps.add(step) diff --git a/cookbook/integration/openeats.py b/cookbook/integration/openeats.py index 09edef68..e258becb 100644 --- a/cookbook/integration/openeats.py +++ b/cookbook/integration/openeats.py @@ -24,13 +24,13 @@ class OpenEats(Integration): if file["source"] != '': instructions += file["source"] - step = Step.objects.create(instruction=instructions) + step = Step.objects.create(instruction=instructions, space=self.request.space,) for ingredient in file['ingredients']: f = get_food(ingredient['food'], self.request.space) u = get_unit(ingredient['unit'], self.request.space) step.ingredients.add(Ingredient.objects.create( - food=f, unit=u, amount=ingredient['amount'] + food=f, unit=u, amount=ingredient['amount'], space=self.request.space, )) recipe.steps.add(step) diff --git a/cookbook/integration/paprika.py b/cookbook/integration/paprika.py index 6feca172..6a8c5076 100644 --- a/cookbook/integration/paprika.py +++ b/cookbook/integration/paprika.py @@ -55,7 +55,7 @@ class Paprika(Integration): pass step = Step.objects.create( - instruction=instructions + instruction=instructions, space=self.request.space, ) if len(recipe_json['description'].strip()) > 500: @@ -73,7 +73,7 @@ class Paprika(Integration): f = get_food(ingredient, self.request.space) u = get_unit(unit, self.request.space) step.ingredients.add(Ingredient.objects.create( - food=f, unit=u, amount=amount, note=note + food=f, unit=u, amount=amount, note=note, space=self.request.space, )) except AttributeError: pass diff --git a/cookbook/integration/recettetek.py b/cookbook/integration/recettetek.py index 04946d62..c6443e0a 100644 --- a/cookbook/integration/recettetek.py +++ b/cookbook/integration/recettetek.py @@ -45,7 +45,7 @@ class RecetteTek(Integration): if not instructions: instructions = '' - step = Step.objects.create(instruction=instructions) + step = Step.objects.create(instruction=instructions, space=self.request.space,) # Append the original import url to the step (if it exists) try: @@ -63,7 +63,7 @@ class RecetteTek(Integration): f = get_food(ingredient, self.request.space) u = get_unit(unit, self.request.space) step.ingredients.add(Ingredient.objects.create( - food=f, unit=u, amount=amount, note=note + food=f, unit=u, amount=amount, note=note, space=self.request.space, )) except Exception as e: print(recipe.name, ': failed to parse recipe ingredients ', str(e)) diff --git a/cookbook/integration/recipekeeper.py b/cookbook/integration/recipekeeper.py index ae198795..f819a772 100644 --- a/cookbook/integration/recipekeeper.py +++ b/cookbook/integration/recipekeeper.py @@ -41,7 +41,7 @@ class RecipeKeeper(Integration): except AttributeError: pass - step = Step.objects.create(instruction='') + step = Step.objects.create(instruction='', space=self.request.space,) for ingredient in file.find("div", {"itemprop": "recipeIngredients"}).findChildren("p"): if ingredient.text == "": @@ -50,7 +50,7 @@ class RecipeKeeper(Integration): f = get_food(ingredient, self.request.space) u = get_unit(unit, self.request.space) step.ingredients.add(Ingredient.objects.create( - food=f, unit=u, amount=amount, note=note + food=f, unit=u, amount=amount, note=note, space=self.request.space, )) for s in file.find("div", {"itemprop": "recipeDirections"}).find_all("p"): diff --git a/cookbook/integration/recipesage.py b/cookbook/integration/recipesage.py index bba76d08..a76a88fb 100644 --- a/cookbook/integration/recipesage.py +++ b/cookbook/integration/recipesage.py @@ -36,7 +36,7 @@ class RecipeSage(Integration): ingredients_added = False for s in file['recipeInstructions']: step = Step.objects.create( - instruction=s['text'] + instruction=s['text'], space=self.request.space, ) if not ingredients_added: ingredients_added = True @@ -46,7 +46,7 @@ class RecipeSage(Integration): f = get_food(ingredient, self.request.space) u = get_unit(unit, self.request.space) step.ingredients.add(Ingredient.objects.create( - food=f, unit=u, amount=amount, note=note + food=f, unit=u, amount=amount, note=note, space=self.request.space, )) recipe.steps.add(step) diff --git a/cookbook/integration/rezkonv.py b/cookbook/integration/rezkonv.py index 2a2bbe18..51145f05 100644 --- a/cookbook/integration/rezkonv.py +++ b/cookbook/integration/rezkonv.py @@ -43,7 +43,7 @@ class RezKonv(Integration): recipe.keywords.add(keyword) step = Step.objects.create( - instruction='\n'.join(directions) + '\n\n' + instruction='\n'.join(directions) + '\n\n', space=self.request.space, ) for ingredient in ingredients: @@ -52,7 +52,7 @@ class RezKonv(Integration): f = get_food(ingredient, self.request.space) u = get_unit(unit, self.request.space) step.ingredients.add(Ingredient.objects.create( - food=f, unit=u, amount=amount, note=note + food=f, unit=u, amount=amount, note=note, space=self.request.space, )) recipe.steps.add(step) diff --git a/cookbook/integration/safron.py b/cookbook/integration/safron.py index f3c439a3..b0a30be3 100644 --- a/cookbook/integration/safron.py +++ b/cookbook/integration/safron.py @@ -43,14 +43,14 @@ class Safron(Integration): recipe = Recipe.objects.create(name=title, description=description, created_by=self.request.user, internal=True, space=self.request.space, ) - step = Step.objects.create(instruction='\n'.join(directions)) + step = Step.objects.create(instruction='\n'.join(directions), space=self.request.space,) for ingredient in ingredients: amount, unit, ingredient, note = parse(ingredient) f = get_food(ingredient, self.request.space) u = get_unit(unit, self.request.space) step.ingredients.add(Ingredient.objects.create( - food=f, unit=u, amount=amount, note=note + food=f, unit=u, amount=amount, note=note, space=self.request.space, )) recipe.steps.add(step) diff --git a/cookbook/models.py b/cookbook/models.py index a9b9f04f..83e6496f 100644 --- a/cookbook/models.py +++ b/cookbook/models.py @@ -665,6 +665,10 @@ class ImportLog(models.Model, PermissionModelMixin): running = models.BooleanField(default=True) msg = models.TextField(default="") keyword = models.ForeignKey(Keyword, null=True, blank=True, on_delete=models.SET_NULL) + + total_recipes = models.IntegerField(default=0) + imported_recipes = models.IntegerField(default=0) + created_at = models.DateTimeField(auto_now_add=True) created_by = models.ForeignKey(User, on_delete=models.CASCADE) diff --git a/cookbook/serializer.py b/cookbook/serializer.py index 43c73b58..06560587 100644 --- a/cookbook/serializer.py +++ b/cookbook/serializer.py @@ -533,7 +533,7 @@ class ImportLogSerializer(serializers.ModelSerializer): class Meta: model = ImportLog - fields = ('id', 'type', 'msg', 'running', 'keyword', 'created_by', 'created_at') + fields = ('id', 'type', 'msg', 'running', 'keyword', 'total_recipes', 'imported_recipes', 'created_by', 'created_at') read_only_fields = ('created_by',) diff --git a/cookbook/views/data.py b/cookbook/views/data.py index 04f65f6f..a1a0f455 100644 --- a/cookbook/views/data.py +++ b/cookbook/views/data.py @@ -143,7 +143,7 @@ def import_url(request): ) step = Step.objects.create( - instruction=data['recipeInstructions'], + instruction=data['recipeInstructions'], space=request.space, ) recipe.steps.add(step) @@ -156,7 +156,7 @@ def import_url(request): recipe.keywords.add(k) for ing in data['recipeIngredient']: - ingredient = Ingredient() + ingredient = Ingredient(space=request.space,) if ing['ingredient']['text'] != '': ingredient.food, f_created = Food.objects.get_or_create(