From b8317c2c299a27aca449b1af803c66ef8e55ff46 Mon Sep 17 00:00:00 2001 From: smilerz Date: Wed, 30 Aug 2023 20:44:40 -0500 Subject: [PATCH] move transpose words to AutomationEngine create tests for transpose words --- cookbook/helper/automation_helper.py | 35 +++- cookbook/helper/ingredient_parser.py | 149 +++++++++--------- cookbook/helper/recipe_url_import.py | 1 - cookbook/tests/other/test_automations.py | 30 ++-- .../tests/other/test_ingredient_parser.py | 1 - 5 files changed, 124 insertions(+), 92 deletions(-) diff --git a/cookbook/helper/automation_helper.py b/cookbook/helper/automation_helper.py index cdc05241..65cc2a88 100644 --- a/cookbook/helper/automation_helper.py +++ b/cookbook/helper/automation_helper.py @@ -1,5 +1,6 @@ +import re + from django.core.cache import caches -from django.db.models import Q from django.db.models.functions import Lower from cookbook.models import Automation @@ -136,6 +137,38 @@ class AutomationEngine: return tokens def apply_transpose_automation(self, string): + """ + If two words (param_1 & param_2) are detected in sequence, swap their position in the ingredient string + :param 1: first word to detect + :param 2: second word to detect + return: new ingredient string + """ + if self.use_cache and self.transpose_words is None: + self.transpose_words = {} + TRANSPOSE_WORDS_CACHE_KEY = f'automation_transpose_words_{self.request.space.pk}' + if c := caches['default'].get(TRANSPOSE_WORDS_CACHE_KEY, None): + self.transpose_words = c + caches['default'].touch(TRANSPOSE_WORDS_CACHE_KEY, 30) + else: + i = 0 + for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.TRANSPOSE_WORDS).only('param_1', 'param_2').order_by('order').all(): + self.transpose_words[i] = [a.param_1.lower(), a.param_2.lower()] + i += 1 + caches['default'].set(TRANSPOSE_WORDS_CACHE_KEY, self.transpose_words, 30) + else: + self.transpose_words = {} + + tokens = [x.lower() for x in string.replace(',', ' ').split()] + if self.transpose_words: + for key, value in self.transpose_words.items(): + if value[0] in tokens and value[1] in tokens: + string = re.sub(rf"\b({value[0]})\W*({value[1]})\b", r"\2 \1", string, flags=re.IGNORECASE) + else: + for rule in Automation.objects.filter(space=self.request.space, type=Automation.TRANSPOSE_WORDS, disabled=False) \ + .annotate(param_1_lower=Lower('param_1'), param_2_lower=Lower('param_2')) \ + .filter(param_1_lower__in=tokens, param_2_lower__in=tokens).order_by('order'): + if rule.param_1 in tokens and rule.param_2 in tokens: + string = re.sub(rf"\b({rule.param_1})\W*({rule.param_2})\b", r"\2 \1", string, flags=re.IGNORECASE) return string def apply_regex_replace_automation(self, string): diff --git a/cookbook/helper/ingredient_parser.py b/cookbook/helper/ingredient_parser.py index 74734c4b..53f967f4 100644 --- a/cookbook/helper/ingredient_parser.py +++ b/cookbook/helper/ingredient_parser.py @@ -2,12 +2,8 @@ import re import string import unicodedata -from django.core.cache import caches -from django.db.models import Q -from django.db.models.functions import Lower - from cookbook.helper.automation_helper import AutomationEngine -from cookbook.models import Automation, Food, Ingredient, Unit +from cookbook.models import Food, Ingredient, Unit class IngredientParser: @@ -16,7 +12,7 @@ class IngredientParser: # food_aliases = {} # unit_aliases = {} # never_unit = {} - transpose_words = {} + # transpose_words = {} automation = None def __init__(self, request, cache_mode=True, ignore_automations=False): @@ -29,51 +25,51 @@ class IngredientParser: self.request = request self.ignore_rules = ignore_automations self.automation = AutomationEngine(self.request, use_cache=cache_mode) - if cache_mode: - # FOOD_CACHE_KEY = f'automation_food_alias_{self.request.space.pk}' - # if c := caches['default'].get(FOOD_CACHE_KEY, None): - # self.food_aliases = c - # caches['default'].touch(FOOD_CACHE_KEY, 30) - # else: - # for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.FOOD_ALIAS).only('param_1', 'param_2').order_by('order').all(): - # self.food_aliases[a.param_1.lower()] = a.param_2 - # caches['default'].set(FOOD_CACHE_KEY, self.food_aliases, 30) + # if cache_mode: + # FOOD_CACHE_KEY = f'automation_food_alias_{self.request.space.pk}' + # if c := caches['default'].get(FOOD_CACHE_KEY, None): + # self.food_aliases = c + # caches['default'].touch(FOOD_CACHE_KEY, 30) + # else: + # for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.FOOD_ALIAS).only('param_1', 'param_2').order_by('order').all(): + # self.food_aliases[a.param_1.lower()] = a.param_2 + # caches['default'].set(FOOD_CACHE_KEY, self.food_aliases, 30) - # UNIT_CACHE_KEY = f'automation_unit_alias_{self.request.space.pk}' - # if c := caches['default'].get(UNIT_CACHE_KEY, None): - # self.unit_aliases = c - # caches['default'].touch(UNIT_CACHE_KEY, 30) - # else: - # for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.UNIT_ALIAS).only('param_1', 'param_2').order_by('order').all(): - # self.unit_aliases[a.param_1.lower()] = a.param_2 - # caches['default'].set(UNIT_CACHE_KEY, self.unit_aliases, 30) + # UNIT_CACHE_KEY = f'automation_unit_alias_{self.request.space.pk}' + # if c := caches['default'].get(UNIT_CACHE_KEY, None): + # self.unit_aliases = c + # caches['default'].touch(UNIT_CACHE_KEY, 30) + # else: + # for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.UNIT_ALIAS).only('param_1', 'param_2').order_by('order').all(): + # self.unit_aliases[a.param_1.lower()] = a.param_2 + # caches['default'].set(UNIT_CACHE_KEY, self.unit_aliases, 30) - # TODO migrated to automation engine - # NEVER_UNIT_CACHE_KEY = f'automation_never_unit_{self.request.space.pk}' - # if c := caches['default'].get(NEVER_UNIT_CACHE_KEY, None): - # self.never_unit = c - # caches['default'].touch(NEVER_UNIT_CACHE_KEY, 30) - # else: - # for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.NEVER_UNIT).only('param_1', 'param_2').order_by('order').all(): - # self.never_unit[a.param_1.lower()] = a.param_2 - # caches['default'].set(NEVER_UNIT_CACHE_KEY, self.never_unit, 30) + # TODO migrated to automation engine + # NEVER_UNIT_CACHE_KEY = f'automation_never_unit_{self.request.space.pk}' + # if c := caches['default'].get(NEVER_UNIT_CACHE_KEY, None): + # self.never_unit = c + # caches['default'].touch(NEVER_UNIT_CACHE_KEY, 30) + # else: + # for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.NEVER_UNIT).only('param_1', 'param_2').order_by('order').all(): + # self.never_unit[a.param_1.lower()] = a.param_2 + # caches['default'].set(NEVER_UNIT_CACHE_KEY, self.never_unit, 30) - # TODO migrated to automation engine - TRANSPOSE_WORDS_CACHE_KEY = f'automation_transpose_words_{self.request.space.pk}' - if c := caches['default'].get(TRANSPOSE_WORDS_CACHE_KEY, None): - self.transpose_words = c - caches['default'].touch(TRANSPOSE_WORDS_CACHE_KEY, 30) - else: - i = 0 - for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.TRANSPOSE_WORDS).only('param_1', 'param_2').order_by('order').all(): - self.transpose_words[i] = [a.param_1.lower(), a.param_2.lower()] - i += 1 - caches['default'].set(TRANSPOSE_WORDS_CACHE_KEY, self.transpose_words, 30) - else: - # self.food_aliases = {} - # self.unit_aliases = {} - # self.never_unit = {} - self.transpose_words = {} + # TODO migrated to automation engine + # TRANSPOSE_WORDS_CACHE_KEY = f'automation_transpose_words_{self.request.space.pk}' + # if c := caches['default'].get(TRANSPOSE_WORDS_CACHE_KEY, None): + # self.transpose_words = c + # caches['default'].touch(TRANSPOSE_WORDS_CACHE_KEY, 30) + # else: + # i = 0 + # for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.TRANSPOSE_WORDS).only('param_1', 'param_2').order_by('order').all(): + # self.transpose_words[i] = [a.param_1.lower(), a.param_2.lower()] + # i += 1 + # caches['default'].set(TRANSPOSE_WORDS_CACHE_KEY, self.transpose_words, 30) + # else: + # self.food_aliases = {} + # self.unit_aliases = {} + # self.never_unit = {} + # self.transpose_words = {} # def apply_food_automation(self, food): # """ @@ -277,33 +273,32 @@ class IngredientParser: # return tokens - # TODO migrated to automation engine - def apply_transpose_words_automations(self, ingredient): - """ - If two words (param_1 & param_2) are detected in sequence, swap their position in the ingredient string - :param 1: first word to detect - :param 2: second word to detect - return: new ingredient string - """ + # def apply_transpose_words_automations(self, ingredient): + # """ + # If two words (param_1 & param_2) are detected in sequence, swap their position in the ingredient string + # :param 1: first word to detect + # :param 2: second word to detect + # return: new ingredient string + # """ - if self.ignore_rules: - return ingredient + # if self.ignore_rules: + # return ingredient - else: - tokens = [x.lower() for x in ingredient.replace(',', ' ').split()] - if self.transpose_words: - filtered_rules = {} - for key, value in self.transpose_words.items(): - if value[0] in tokens and value[1] in tokens: - filtered_rules[key] = value - for k, v in filtered_rules.items(): - ingredient = re.sub(rf"\b({v[0]})\W*({v[1]})\b", r"\2 \1", ingredient, flags=re.IGNORECASE) - else: - for rule in Automation.objects.filter(space=self.request.space, type=Automation.TRANSPOSE_WORDS, disabled=False) \ - .annotate(param_1_lower=Lower('param_1'), param_2_lower=Lower('param_2')) \ - .filter(Q(Q(param_1_lower__in=tokens) | Q(param_2_lower__in=tokens))).order_by('order'): - ingredient = re.sub(rf"\b({rule.param_1})\W*({rule.param_1})\b", r"\2 \1", ingredient, flags=re.IGNORECASE) - return ingredient + # else: + # tokens = [x.lower() for x in ingredient.replace(',', ' ').split()] + # if self.transpose_words: + # filtered_rules = {} + # for key, value in self.transpose_words.items(): + # if value[0] in tokens and value[1] in tokens: + # filtered_rules[key] = value + # for k, v in filtered_rules.items(): + # ingredient = re.sub(rf"\b({v[0]})\W*({v[1]})\b", r"\2 \1", ingredient, flags=re.IGNORECASE) + # else: + # for rule in Automation.objects.filter(space=self.request.space, type=Automation.TRANSPOSE_WORDS, disabled=False) \ + # .annotate(param_1_lower=Lower('param_1'), param_2_lower=Lower('param_2')) \ + # .filter(Q(Q(param_1_lower__in=tokens) | Q(param_2_lower__in=tokens))).order_by('order'): + # ingredient = re.sub(rf"\b({rule.param_1})\W*({rule.param_1})\b", r"\2 \1", ingredient, flags=re.IGNORECASE) + # return ingredient def parse(self, ingredient): """ @@ -345,8 +340,8 @@ class IngredientParser: if re.match('([0-9])+([A-z])+\\s', ingredient): ingredient = re.sub(r'(?<=([a-z])|\d)(?=(?(1)\d|[a-z]))', ' ', ingredient) - # TODO migrated to automation engine - ingredient = self.apply_transpose_words_automations(ingredient) + if not self.ignore_rules: + ingredient = self.automation.apply_transpose_words_automations(ingredient) tokens = ingredient.split() # split at each space into tokens if len(tokens) == 1: @@ -360,8 +355,8 @@ class IngredientParser: # three arguments if it already has a unit there can't be # a fraction for the amount if len(tokens) > 2: - # TODO migrated to automation engine - tokens = self.automation.apply_never_unit_automation(tokens) + if not self.ignore_rules: + tokens = self.automation.apply_never_unit_automation(tokens) try: if unit is not None: # a unit is already found, no need to try the second argument for a fraction diff --git a/cookbook/helper/recipe_url_import.py b/cookbook/helper/recipe_url_import.py index cb7d4c73..27c69157 100644 --- a/cookbook/helper/recipe_url_import.py +++ b/cookbook/helper/recipe_url_import.py @@ -2,7 +2,6 @@ import re import traceback from html import unescape -from django.core.cache import caches from django.utils.dateparse import parse_duration from django.utils.translation import gettext as _ from isodate import parse_duration as iso_parse_duration diff --git a/cookbook/tests/other/test_automations.py b/cookbook/tests/other/test_automations.py index 7789df30..0247ba6b 100644 --- a/cookbook/tests/other/test_automations.py +++ b/cookbook/tests/other/test_automations.py @@ -6,16 +6,6 @@ from django_scopes import scope from cookbook.helper.automation_helper import AutomationEngine from cookbook.models import Automation -# TODO test case sensitive match, assert update value -# TODO test case insensitive match, assert update value -# TODO test no match, assert not update value -# TODO test accent insensitive match, assert not update value - - -# @pytest.fixture() -# def automation_food(space_1): -# return Keyword.objects.get_or_create(name='test_1', space=space_1)[0] - @pytest.mark.parametrize("arg", [ ['Match', True], @@ -103,8 +93,24 @@ def test_never_unit_automation(u1_s1, arg): assert automation.apply_never_unit_automation(arg[0]) == arg[2] -# def test_transpose_automation(): -# assert True == True +@pytest.mark.parametrize("arg", [ + ['second first', 'first second'], + ['longer string second first longer string', 'longer string first second longer string'], + ['second fails first', 'second fails first'], +]) +def test_transpose_automation(u1_s1, arg): + user = auth.get_user(u1_s1) + space = user.userspace_set.first().space + request = RequestFactory() + request.user = user + request.space = space + automation = AutomationEngine(request, False) + + with scope(space=space): + Automation.objects.get_or_create(name='transpose words test', type=Automation.TRANSPOSE_WORDS, param_1='second', param_2='first', created_by=user, space=space) + assert automation.apply_transpose_automation(arg[0]) == arg[1] + + assert True == True # # for some reason this tests cant run due to some kind of encoding issue, needs to be fixed # # def test_description_replace_automation(u1_s1, space_1): # # if 'cookbook' in os.getcwd(): diff --git a/cookbook/tests/other/test_ingredient_parser.py b/cookbook/tests/other/test_ingredient_parser.py index 15ecd73c..30c6eb6a 100644 --- a/cookbook/tests/other/test_ingredient_parser.py +++ b/cookbook/tests/other/test_ingredient_parser.py @@ -1,4 +1,3 @@ -import pytest from django.contrib import auth from django.test import RequestFactory from django_scopes import scope