diff --git a/cookbook/helper/automation_helper.py b/cookbook/helper/automation_helper.py index 442b1c6f..a86d405b 100644 --- a/cookbook/helper/automation_helper.py +++ b/cookbook/helper/automation_helper.py @@ -20,7 +20,7 @@ class AutomationEngine: Automation.INSTRUCTION_REPLACE: None, Automation.FOOD_REPLACE: None, Automation.UNIT_REPLACE: None, - Automation.TITLE_REPLACE: None, + Automation.NAME_REPLACE: None, } def __init__(self, request, use_cache=True, source=None): @@ -191,7 +191,7 @@ class AutomationEngine: Automation.INSTRUCTION_REPLACE Automation.FOOD_REPLACE Automation.UNIT_REPLACE - Automation.TITLE_REPLACE + Automation.NAME_REPLACE regex replacment utilized the following fields from the Automation model :param 1: source that should apply the automation in regex format ('.*' for all) @@ -218,10 +218,10 @@ class AutomationEngine: if self.regex_replace[automation_type]: for rule in self.regex_replace[automation_type].values(): if re.match(rule[0], (self.source)[:512]): - string = re.sub(rule[1], rule[2], string) + string = re.sub(rule[1], rule[2], string, flags=re.IGNORECASE) else: for rule in Automation.objects.filter(space=self.request.space, disabled=False, type=automation_type).only( 'param_1', 'param_2', 'param_3').order_by('order').all()[:512]: if re.match(rule.param_1, (self.source)[:512]): - string = re.sub(rule.param_2, rule.param_3, string) + string = re.sub(rule.param_2, rule.param_3, string, flags=re.IGNORECASE) return string diff --git a/cookbook/helper/ingredient_parser.py b/cookbook/helper/ingredient_parser.py index 320a3e17..dd3bfb99 100644 --- a/cookbook/helper/ingredient_parser.py +++ b/cookbook/helper/ingredient_parser.py @@ -339,7 +339,7 @@ class IngredientParser: ingredient = re.sub(r'(?<=([a-z])|\d)(?=(?(1)\d|[a-z]))', ' ', ingredient) if not self.ignore_rules: - ingredient = self.automation.apply_transpose_words_automations(ingredient) + ingredient = self.automation.apply_transpose_automation(ingredient) tokens = ingredient.split() # split at each space into tokens if len(tokens) == 1: diff --git a/cookbook/helper/recipe_url_import.py b/cookbook/helper/recipe_url_import.py index fc7a3e7a..ee4c9e70 100644 --- a/cookbook/helper/recipe_url_import.py +++ b/cookbook/helper/recipe_url_import.py @@ -38,7 +38,7 @@ def get_from_scraper(scrape, request): except Exception: recipe_json['source_url'] = '' - automation_engine = AutomationEngine(request, source=recipe_json['source_url']) + automation_engine = AutomationEngine(request, source=recipe_json.get('source_url')) # assign recipe name try: recipe_json['name'] = parse_name(scrape.title()[:128] or None) @@ -53,6 +53,8 @@ def get_from_scraper(scrape, request): if isinstance(recipe_json['name'], list) and len(recipe_json['name']) > 0: recipe_json['name'] = recipe_json['name'][0] + recipe_json['name'] = automation_engine.apply_regex_replace_automation(recipe_json['name'], Automation.NAME_REPLACE) + # assign recipe description # TODO notify user about limit if reached - >256 description will be truncated try: diff --git a/cookbook/migrations/0199_alter_automation_type.py b/cookbook/migrations/0199_alter_automation_type.py index 56da9d2a..47a42123 100644 --- a/cookbook/migrations/0199_alter_automation_type.py +++ b/cookbook/migrations/0199_alter_automation_type.py @@ -15,20 +15,15 @@ class Migration(migrations.Migration): name='type', field=models.CharField( choices=[ - ('FOOD_ALIAS', - 'Food Alias'), - ('UNIT_ALIAS', - 'Unit Alias'), - ('KEYWORD_ALIAS', - 'Keyword Alias'), - ('DESCRIPTION_REPLACE', - 'Description Replace'), - ('INSTRUCTION_REPLACE', - 'Instruction Replace'), - ('NEVER_UNIT', - 'Never Unit'), - ('TRANSPOSE_WORDS', - 'Transpose Words')], + ('FOOD_ALIAS', 'Food Alias'), + ('UNIT_ALIAS', 'Unit Alias'), + ('KEYWORD_ALIAS', 'Keyword Alias'), + ('DESCRIPTION_REPLACE', 'Description Replace'), + ('INSTRUCTION_REPLACE', 'Instruction Replace'), + ('NEVER_UNIT', 'Never Unit'), + ('TRANSPOSE_WORDS', 'Transpose Words'), + ('NAME_REPLACE', 'Name Replace') + ], max_length=128), ), ] diff --git a/cookbook/models.py b/cookbook/models.py index a6259a7a..f2869d5f 100644 --- a/cookbook/models.py +++ b/cookbook/models.py @@ -1316,7 +1316,7 @@ class Automation(ExportModelOperationsMixin('automations'), models.Model, Permis TRANSPOSE_WORDS = 'TRANSPOSE_WORDS' FOOD_REPLACE = 'FOOD_REPLACE' UNIT_REPLACE = 'UNIT_REPLACE' - TITLE_REPLACE = 'TITLE_REPLACE' + NAME_REPLACE = 'NAME_REPLACE' type = models.CharField(max_length=128, choices=( @@ -1329,7 +1329,7 @@ class Automation(ExportModelOperationsMixin('automations'), models.Model, Permis (TRANSPOSE_WORDS, _('Transpose Words')), (FOOD_REPLACE, _('Food Replace')), (UNIT_REPLACE, _('Unit Replace')), - (TITLE_REPLACE, _('Title Replace')), + (NAME_REPLACE, _('Name Replace')), )) name = models.CharField(max_length=128, default='') description = models.TextField(blank=True, null=True) diff --git a/cookbook/tests/other/test_automations.py b/cookbook/tests/other/test_automations.py index 92af4800..e10275cd 100644 --- a/cookbook/tests/other/test_automations.py +++ b/cookbook/tests/other/test_automations.py @@ -1,11 +1,17 @@ +import os + import pytest from django.contrib import auth from django.test import RequestFactory from django_scopes import scope from cookbook.helper.automation_helper import AutomationEngine +from cookbook.helper.recipe_url_import import get_from_scraper +from cookbook.helper.scrapers.scrapers import text_scraper from cookbook.models import Automation +DATA_DIR = "cookbook/tests/other/test_data/" + @pytest.mark.parametrize("arg", [ ['Match', True], @@ -94,7 +100,7 @@ def test_never_unit_automation(u1_s1, arg): @pytest.mark.parametrize("arg", [ [Automation.DESCRIPTION_REPLACE], [Automation.INSTRUCTION_REPLACE], - [Automation.TITLE_REPLACE], + [Automation.NAME_REPLACE], [Automation.FOOD_REPLACE], [Automation.UNIT_REPLACE], ]) @@ -135,6 +141,38 @@ def test_transpose_automation(u1_s1, arg): Automation.objects.get_or_create(name='transpose words test', type=Automation.TRANSPOSE_WORDS, param_1='second', param_2='first', created_by=user, space=space) assert automation.apply_transpose_automation(arg[0]) == arg[1] + +def test_url_import_regex_replace(u1_s1): + # TODO this does not test import with multiple steps - do any sites import with this pattern? It doesn't look like the url_importer supports it + user = auth.get_user(u1_s1) + space = user.userspace_set.first().space + request = RequestFactory() + request.user = user + request.space = space + recipe = 'regex_recipe.html' + types = [Automation.DESCRIPTION_REPLACE, Automation.INSTRUCTION_REPLACE, Automation.NAME_REPLACE, Automation.FOOD_REPLACE, Automation.UNIT_REPLACE] + find_text = "_remove" + target_text = "Test" + + if 'cookbook' in os.getcwd(): + test_file = os.path.join(os.getcwd(), 'other', 'test_data', recipe) + else: + test_file = os.path.join(os.getcwd(), 'cookbook', 'tests', 'other', 'test_data', recipe) + with open(test_file, 'r', encoding='UTF-8') as d: + scrape = text_scraper(text=d.read(), url="https://www.allrecipes.com") + with scope(space=space): + for t in types: + Automation.objects.get_or_create(name=t, type=t, param_1='.*', param_2=find_text, param_3='', created_by=user, space=space) + recipe_json = get_from_scraper(scrape, request) + assert recipe_json['name'] == target_text + assert recipe_json['description'] == target_text + assert recipe_json['steps'][0]['instruction'] == target_text + assert recipe_json['steps'][0]['ingredients'][0]['food']['name'] == target_text + assert recipe_json['steps'][0]['ingredients'][0]['food']['name'] == target_text + assert recipe_json['steps'][0]['ingredients'][1]['unit']['name'] == target_text + assert recipe_json['steps'][0]['ingredients'][1]['unit']['name'] == target_text + + # # for some reason this tests cant run due to some kind of encoding issue, needs to be fixed # # def test_description_replace_automation(u1_s1, space_1): # # if 'cookbook' in os.getcwd(): diff --git a/cookbook/tests/other/test_data/regex_recipe.html b/cookbook/tests/other/test_data/regex_recipe.html new file mode 100644 index 00000000..b19fcb7f --- /dev/null +++ b/cookbook/tests/other/test_data/regex_recipe.html @@ -0,0 +1,38 @@ + + + + + + Test_Remove_ + + + + + + + + diff --git a/cookbook/tests/other/test_url_import.py b/cookbook/tests/other/test_url_import.py index 9f57386a..ae4677c0 100644 --- a/cookbook/tests/other/test_url_import.py +++ b/cookbook/tests/other/test_url_import.py @@ -2,16 +2,13 @@ import json import os import pytest -from django.contrib import auth from django.urls import reverse -from django_scopes import scopes_disabled from cookbook.tests.conftest import validate_recipe from ._recipes import (ALLRECIPES, AMERICAS_TEST_KITCHEN, CHEF_KOCH, CHEF_KOCH2, COOKPAD, COOKS_COUNTRY, DELISH, FOOD_NETWORK, GIALLOZAFFERANO, JOURNAL_DES_FEMMES, MADAME_DESSERT, MARMITON, TASTE_OF_HOME, THE_SPRUCE_EATS, TUDOGOSTOSO) -from ...models import Automation IMPORT_SOURCE_URL = 'api_recipe_from_source' DATA_DIR = "cookbook/tests/other/test_data/" @@ -75,5 +72,3 @@ def test_recipe_import(arg, u1_s1): content_type='application/json') recipe = json.loads(response.content)['recipe_json'] validate_recipe(arg, recipe) - - diff --git a/docs/features/automation.md b/docs/features/automation.md index f857a09d..b89a4a10 100644 --- a/docs/features/automation.md +++ b/docs/features/automation.md @@ -31,7 +31,7 @@ This automation is a bit more complicated than the alias rules. It is run when i from a website. It uses Regular Expressions (RegEx) to determine if a description should be altered, what exactly to remove -and what to replace it with. +and what to replace it with. The search string ignores case, the replacement string respects case. - **Parameter 1**: pattern of which sites to match (e.g. `.*.chefkoch.de.*`, `.*`) - **Parameter 2**: pattern of what to replace (e.g. `.*`) diff --git a/vue/src/locales/en.json b/vue/src/locales/en.json index 869fba11..dd58d8b8 100644 --- a/vue/src/locales/en.json +++ b/vue/src/locales/en.json @@ -530,7 +530,7 @@ "Import Recipe": "Import Recipe", "Never_Unit": "Never Unit", "Transpose_Words": "Transpose Words", - "Title_Replace":"Title Replace", + "Name_Replace":"Name Replace", "Food_Replace":"Food Replace", "Unit_Replace":"Unit Replace" } diff --git a/vue/src/utils/models.js b/vue/src/utils/models.js index 2008a320..28ec0e46 100644 --- a/vue/src/utils/models.js +++ b/vue/src/utils/models.js @@ -543,9 +543,9 @@ export class Models { { value: "FOOD_ALIAS", text: "Food_Alias" }, { value: "UNIT_ALIAS", text: "Unit_Alias" }, { value: "KEYWORD_ALIAS", text: "Keyword_Alias" }, + { value: "NAME_REPLACE", text: "Name_Replace" }, { value: "DESCRIPTION_REPLACE", text: "Description_Replace" }, { value: "INSTRUCTION_REPLACE", text: "Instruction_Replace" }, - { value: "TITLE_REPLACE", text: "Title_Replace" }, { value: "FOOD_REPLACE", text: "Food_Replace" }, { value: "UNIT_REPLACE", text: "Unit_Replace" }, { value: "NEVER_UNIT", text: "Never_Unit" },