test for automations applied during url import
renamed TITLE_REPLACE to NAME_REPLACE
This commit is contained in:
parent
2679a22464
commit
9b5e39415e
@ -20,7 +20,7 @@ class AutomationEngine:
|
|||||||
Automation.INSTRUCTION_REPLACE: None,
|
Automation.INSTRUCTION_REPLACE: None,
|
||||||
Automation.FOOD_REPLACE: None,
|
Automation.FOOD_REPLACE: None,
|
||||||
Automation.UNIT_REPLACE: None,
|
Automation.UNIT_REPLACE: None,
|
||||||
Automation.TITLE_REPLACE: None,
|
Automation.NAME_REPLACE: None,
|
||||||
}
|
}
|
||||||
|
|
||||||
def __init__(self, request, use_cache=True, source=None):
|
def __init__(self, request, use_cache=True, source=None):
|
||||||
@ -191,7 +191,7 @@ class AutomationEngine:
|
|||||||
Automation.INSTRUCTION_REPLACE
|
Automation.INSTRUCTION_REPLACE
|
||||||
Automation.FOOD_REPLACE
|
Automation.FOOD_REPLACE
|
||||||
Automation.UNIT_REPLACE
|
Automation.UNIT_REPLACE
|
||||||
Automation.TITLE_REPLACE
|
Automation.NAME_REPLACE
|
||||||
|
|
||||||
regex replacment utilized the following fields from the Automation model
|
regex replacment utilized the following fields from the Automation model
|
||||||
:param 1: source that should apply the automation in regex format ('.*' for all)
|
:param 1: source that should apply the automation in regex format ('.*' for all)
|
||||||
@ -218,10 +218,10 @@ class AutomationEngine:
|
|||||||
if self.regex_replace[automation_type]:
|
if self.regex_replace[automation_type]:
|
||||||
for rule in self.regex_replace[automation_type].values():
|
for rule in self.regex_replace[automation_type].values():
|
||||||
if re.match(rule[0], (self.source)[:512]):
|
if re.match(rule[0], (self.source)[:512]):
|
||||||
string = re.sub(rule[1], rule[2], string)
|
string = re.sub(rule[1], rule[2], string, flags=re.IGNORECASE)
|
||||||
else:
|
else:
|
||||||
for rule in Automation.objects.filter(space=self.request.space, disabled=False, type=automation_type).only(
|
for rule in Automation.objects.filter(space=self.request.space, disabled=False, type=automation_type).only(
|
||||||
'param_1', 'param_2', 'param_3').order_by('order').all()[:512]:
|
'param_1', 'param_2', 'param_3').order_by('order').all()[:512]:
|
||||||
if re.match(rule.param_1, (self.source)[:512]):
|
if re.match(rule.param_1, (self.source)[:512]):
|
||||||
string = re.sub(rule.param_2, rule.param_3, string)
|
string = re.sub(rule.param_2, rule.param_3, string, flags=re.IGNORECASE)
|
||||||
return string
|
return string
|
||||||
|
@ -339,7 +339,7 @@ class IngredientParser:
|
|||||||
ingredient = re.sub(r'(?<=([a-z])|\d)(?=(?(1)\d|[a-z]))', ' ', ingredient)
|
ingredient = re.sub(r'(?<=([a-z])|\d)(?=(?(1)\d|[a-z]))', ' ', ingredient)
|
||||||
|
|
||||||
if not self.ignore_rules:
|
if not self.ignore_rules:
|
||||||
ingredient = self.automation.apply_transpose_words_automations(ingredient)
|
ingredient = self.automation.apply_transpose_automation(ingredient)
|
||||||
|
|
||||||
tokens = ingredient.split() # split at each space into tokens
|
tokens = ingredient.split() # split at each space into tokens
|
||||||
if len(tokens) == 1:
|
if len(tokens) == 1:
|
||||||
|
@ -38,7 +38,7 @@ def get_from_scraper(scrape, request):
|
|||||||
except Exception:
|
except Exception:
|
||||||
recipe_json['source_url'] = ''
|
recipe_json['source_url'] = ''
|
||||||
|
|
||||||
automation_engine = AutomationEngine(request, source=recipe_json['source_url'])
|
automation_engine = AutomationEngine(request, source=recipe_json.get('source_url'))
|
||||||
# assign recipe name
|
# assign recipe name
|
||||||
try:
|
try:
|
||||||
recipe_json['name'] = parse_name(scrape.title()[:128] or None)
|
recipe_json['name'] = parse_name(scrape.title()[:128] or None)
|
||||||
@ -53,6 +53,8 @@ def get_from_scraper(scrape, request):
|
|||||||
if isinstance(recipe_json['name'], list) and len(recipe_json['name']) > 0:
|
if isinstance(recipe_json['name'], list) and len(recipe_json['name']) > 0:
|
||||||
recipe_json['name'] = recipe_json['name'][0]
|
recipe_json['name'] = recipe_json['name'][0]
|
||||||
|
|
||||||
|
recipe_json['name'] = automation_engine.apply_regex_replace_automation(recipe_json['name'], Automation.NAME_REPLACE)
|
||||||
|
|
||||||
# assign recipe description
|
# assign recipe description
|
||||||
# TODO notify user about limit if reached - >256 description will be truncated
|
# TODO notify user about limit if reached - >256 description will be truncated
|
||||||
try:
|
try:
|
||||||
|
@ -15,20 +15,15 @@ class Migration(migrations.Migration):
|
|||||||
name='type',
|
name='type',
|
||||||
field=models.CharField(
|
field=models.CharField(
|
||||||
choices=[
|
choices=[
|
||||||
('FOOD_ALIAS',
|
('FOOD_ALIAS', 'Food Alias'),
|
||||||
'Food Alias'),
|
('UNIT_ALIAS', 'Unit Alias'),
|
||||||
('UNIT_ALIAS',
|
('KEYWORD_ALIAS', 'Keyword Alias'),
|
||||||
'Unit Alias'),
|
('DESCRIPTION_REPLACE', 'Description Replace'),
|
||||||
('KEYWORD_ALIAS',
|
('INSTRUCTION_REPLACE', 'Instruction Replace'),
|
||||||
'Keyword Alias'),
|
('NEVER_UNIT', 'Never Unit'),
|
||||||
('DESCRIPTION_REPLACE',
|
('TRANSPOSE_WORDS', 'Transpose Words'),
|
||||||
'Description Replace'),
|
('NAME_REPLACE', 'Name Replace')
|
||||||
('INSTRUCTION_REPLACE',
|
],
|
||||||
'Instruction Replace'),
|
|
||||||
('NEVER_UNIT',
|
|
||||||
'Never Unit'),
|
|
||||||
('TRANSPOSE_WORDS',
|
|
||||||
'Transpose Words')],
|
|
||||||
max_length=128),
|
max_length=128),
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
|
@ -1316,7 +1316,7 @@ class Automation(ExportModelOperationsMixin('automations'), models.Model, Permis
|
|||||||
TRANSPOSE_WORDS = 'TRANSPOSE_WORDS'
|
TRANSPOSE_WORDS = 'TRANSPOSE_WORDS'
|
||||||
FOOD_REPLACE = 'FOOD_REPLACE'
|
FOOD_REPLACE = 'FOOD_REPLACE'
|
||||||
UNIT_REPLACE = 'UNIT_REPLACE'
|
UNIT_REPLACE = 'UNIT_REPLACE'
|
||||||
TITLE_REPLACE = 'TITLE_REPLACE'
|
NAME_REPLACE = 'NAME_REPLACE'
|
||||||
|
|
||||||
type = models.CharField(max_length=128,
|
type = models.CharField(max_length=128,
|
||||||
choices=(
|
choices=(
|
||||||
@ -1329,7 +1329,7 @@ class Automation(ExportModelOperationsMixin('automations'), models.Model, Permis
|
|||||||
(TRANSPOSE_WORDS, _('Transpose Words')),
|
(TRANSPOSE_WORDS, _('Transpose Words')),
|
||||||
(FOOD_REPLACE, _('Food Replace')),
|
(FOOD_REPLACE, _('Food Replace')),
|
||||||
(UNIT_REPLACE, _('Unit Replace')),
|
(UNIT_REPLACE, _('Unit Replace')),
|
||||||
(TITLE_REPLACE, _('Title Replace')),
|
(NAME_REPLACE, _('Name Replace')),
|
||||||
))
|
))
|
||||||
name = models.CharField(max_length=128, default='')
|
name = models.CharField(max_length=128, default='')
|
||||||
description = models.TextField(blank=True, null=True)
|
description = models.TextField(blank=True, null=True)
|
||||||
|
@ -1,11 +1,17 @@
|
|||||||
|
import os
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from django.contrib import auth
|
from django.contrib import auth
|
||||||
from django.test import RequestFactory
|
from django.test import RequestFactory
|
||||||
from django_scopes import scope
|
from django_scopes import scope
|
||||||
|
|
||||||
from cookbook.helper.automation_helper import AutomationEngine
|
from cookbook.helper.automation_helper import AutomationEngine
|
||||||
|
from cookbook.helper.recipe_url_import import get_from_scraper
|
||||||
|
from cookbook.helper.scrapers.scrapers import text_scraper
|
||||||
from cookbook.models import Automation
|
from cookbook.models import Automation
|
||||||
|
|
||||||
|
DATA_DIR = "cookbook/tests/other/test_data/"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("arg", [
|
@pytest.mark.parametrize("arg", [
|
||||||
['Match', True],
|
['Match', True],
|
||||||
@ -94,7 +100,7 @@ def test_never_unit_automation(u1_s1, arg):
|
|||||||
@pytest.mark.parametrize("arg", [
|
@pytest.mark.parametrize("arg", [
|
||||||
[Automation.DESCRIPTION_REPLACE],
|
[Automation.DESCRIPTION_REPLACE],
|
||||||
[Automation.INSTRUCTION_REPLACE],
|
[Automation.INSTRUCTION_REPLACE],
|
||||||
[Automation.TITLE_REPLACE],
|
[Automation.NAME_REPLACE],
|
||||||
[Automation.FOOD_REPLACE],
|
[Automation.FOOD_REPLACE],
|
||||||
[Automation.UNIT_REPLACE],
|
[Automation.UNIT_REPLACE],
|
||||||
])
|
])
|
||||||
@ -135,6 +141,38 @@ def test_transpose_automation(u1_s1, arg):
|
|||||||
Automation.objects.get_or_create(name='transpose words test', type=Automation.TRANSPOSE_WORDS, param_1='second', param_2='first', created_by=user, space=space)
|
Automation.objects.get_or_create(name='transpose words test', type=Automation.TRANSPOSE_WORDS, param_1='second', param_2='first', created_by=user, space=space)
|
||||||
assert automation.apply_transpose_automation(arg[0]) == arg[1]
|
assert automation.apply_transpose_automation(arg[0]) == arg[1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_url_import_regex_replace(u1_s1):
|
||||||
|
# TODO this does not test import with multiple steps - do any sites import with this pattern? It doesn't look like the url_importer supports it
|
||||||
|
user = auth.get_user(u1_s1)
|
||||||
|
space = user.userspace_set.first().space
|
||||||
|
request = RequestFactory()
|
||||||
|
request.user = user
|
||||||
|
request.space = space
|
||||||
|
recipe = 'regex_recipe.html'
|
||||||
|
types = [Automation.DESCRIPTION_REPLACE, Automation.INSTRUCTION_REPLACE, Automation.NAME_REPLACE, Automation.FOOD_REPLACE, Automation.UNIT_REPLACE]
|
||||||
|
find_text = "_remove"
|
||||||
|
target_text = "Test"
|
||||||
|
|
||||||
|
if 'cookbook' in os.getcwd():
|
||||||
|
test_file = os.path.join(os.getcwd(), 'other', 'test_data', recipe)
|
||||||
|
else:
|
||||||
|
test_file = os.path.join(os.getcwd(), 'cookbook', 'tests', 'other', 'test_data', recipe)
|
||||||
|
with open(test_file, 'r', encoding='UTF-8') as d:
|
||||||
|
scrape = text_scraper(text=d.read(), url="https://www.allrecipes.com")
|
||||||
|
with scope(space=space):
|
||||||
|
for t in types:
|
||||||
|
Automation.objects.get_or_create(name=t, type=t, param_1='.*', param_2=find_text, param_3='', created_by=user, space=space)
|
||||||
|
recipe_json = get_from_scraper(scrape, request)
|
||||||
|
assert recipe_json['name'] == target_text
|
||||||
|
assert recipe_json['description'] == target_text
|
||||||
|
assert recipe_json['steps'][0]['instruction'] == target_text
|
||||||
|
assert recipe_json['steps'][0]['ingredients'][0]['food']['name'] == target_text
|
||||||
|
assert recipe_json['steps'][0]['ingredients'][0]['food']['name'] == target_text
|
||||||
|
assert recipe_json['steps'][0]['ingredients'][1]['unit']['name'] == target_text
|
||||||
|
assert recipe_json['steps'][0]['ingredients'][1]['unit']['name'] == target_text
|
||||||
|
|
||||||
|
|
||||||
# # for some reason this tests cant run due to some kind of encoding issue, needs to be fixed
|
# # for some reason this tests cant run due to some kind of encoding issue, needs to be fixed
|
||||||
# # def test_description_replace_automation(u1_s1, space_1):
|
# # def test_description_replace_automation(u1_s1, space_1):
|
||||||
# # if 'cookbook' in os.getcwd():
|
# # if 'cookbook' in os.getcwd():
|
||||||
|
38
cookbook/tests/other/test_data/regex_recipe.html
Normal file
38
cookbook/tests/other/test_data/regex_recipe.html
Normal file
@ -0,0 +1,38 @@
|
|||||||
|
|
||||||
|
<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
|
||||||
|
<head>
|
||||||
|
<title>Test_Remove_</title>
|
||||||
|
<script type="application/ld+json">
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"@context": "http://schema.org",
|
||||||
|
"@type": "Recipe",
|
||||||
|
"mainEntityOfPage": "https://www.allrecipes.com/recipe/24010/easy-chicken-marsala/",
|
||||||
|
"name": "Test_Remove",
|
||||||
|
"datePublished": "2020-06-19T03:05:13.000Z",
|
||||||
|
"description": "Test_Remove",
|
||||||
|
"recipeIngredient": [
|
||||||
|
"1 Test_Remove Test_Remove",
|
||||||
|
"1 Test_Remove Test_Remove",
|
||||||
|
],
|
||||||
|
"recipeInstructions": [
|
||||||
|
{
|
||||||
|
"@type": "HowToStep",
|
||||||
|
"text": "Test_Remove"
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"recipeCategory": [
|
||||||
|
"Test_Remove",
|
||||||
|
],
|
||||||
|
}
|
||||||
|
]
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<body>
|
||||||
|
|
||||||
|
|
||||||
|
</body>
|
||||||
|
|
||||||
|
</html>
|
@ -2,16 +2,13 @@ import json
|
|||||||
import os
|
import os
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from django.contrib import auth
|
|
||||||
from django.urls import reverse
|
from django.urls import reverse
|
||||||
from django_scopes import scopes_disabled
|
|
||||||
|
|
||||||
from cookbook.tests.conftest import validate_recipe
|
from cookbook.tests.conftest import validate_recipe
|
||||||
|
|
||||||
from ._recipes import (ALLRECIPES, AMERICAS_TEST_KITCHEN, CHEF_KOCH, CHEF_KOCH2, COOKPAD,
|
from ._recipes import (ALLRECIPES, AMERICAS_TEST_KITCHEN, CHEF_KOCH, CHEF_KOCH2, COOKPAD,
|
||||||
COOKS_COUNTRY, DELISH, FOOD_NETWORK, GIALLOZAFFERANO, JOURNAL_DES_FEMMES,
|
COOKS_COUNTRY, DELISH, FOOD_NETWORK, GIALLOZAFFERANO, JOURNAL_DES_FEMMES,
|
||||||
MADAME_DESSERT, MARMITON, TASTE_OF_HOME, THE_SPRUCE_EATS, TUDOGOSTOSO)
|
MADAME_DESSERT, MARMITON, TASTE_OF_HOME, THE_SPRUCE_EATS, TUDOGOSTOSO)
|
||||||
from ...models import Automation
|
|
||||||
|
|
||||||
IMPORT_SOURCE_URL = 'api_recipe_from_source'
|
IMPORT_SOURCE_URL = 'api_recipe_from_source'
|
||||||
DATA_DIR = "cookbook/tests/other/test_data/"
|
DATA_DIR = "cookbook/tests/other/test_data/"
|
||||||
@ -75,5 +72,3 @@ def test_recipe_import(arg, u1_s1):
|
|||||||
content_type='application/json')
|
content_type='application/json')
|
||||||
recipe = json.loads(response.content)['recipe_json']
|
recipe = json.loads(response.content)['recipe_json']
|
||||||
validate_recipe(arg, recipe)
|
validate_recipe(arg, recipe)
|
||||||
|
|
||||||
|
|
||||||
|
@ -31,7 +31,7 @@ This automation is a bit more complicated than the alias rules. It is run when i
|
|||||||
from a website.
|
from a website.
|
||||||
|
|
||||||
It uses Regular Expressions (RegEx) to determine if a description should be altered, what exactly to remove
|
It uses Regular Expressions (RegEx) to determine if a description should be altered, what exactly to remove
|
||||||
and what to replace it with.
|
and what to replace it with. The search string ignores case, the replacement string respects case.
|
||||||
|
|
||||||
- **Parameter 1**: pattern of which sites to match (e.g. `.*.chefkoch.de.*`, `.*`)
|
- **Parameter 1**: pattern of which sites to match (e.g. `.*.chefkoch.de.*`, `.*`)
|
||||||
- **Parameter 2**: pattern of what to replace (e.g. `.*`)
|
- **Parameter 2**: pattern of what to replace (e.g. `.*`)
|
||||||
|
@ -530,7 +530,7 @@
|
|||||||
"Import Recipe": "Import Recipe",
|
"Import Recipe": "Import Recipe",
|
||||||
"Never_Unit": "Never Unit",
|
"Never_Unit": "Never Unit",
|
||||||
"Transpose_Words": "Transpose Words",
|
"Transpose_Words": "Transpose Words",
|
||||||
"Title_Replace":"Title Replace",
|
"Name_Replace":"Name Replace",
|
||||||
"Food_Replace":"Food Replace",
|
"Food_Replace":"Food Replace",
|
||||||
"Unit_Replace":"Unit Replace"
|
"Unit_Replace":"Unit Replace"
|
||||||
}
|
}
|
||||||
|
@ -543,9 +543,9 @@ export class Models {
|
|||||||
{ value: "FOOD_ALIAS", text: "Food_Alias" },
|
{ value: "FOOD_ALIAS", text: "Food_Alias" },
|
||||||
{ value: "UNIT_ALIAS", text: "Unit_Alias" },
|
{ value: "UNIT_ALIAS", text: "Unit_Alias" },
|
||||||
{ value: "KEYWORD_ALIAS", text: "Keyword_Alias" },
|
{ value: "KEYWORD_ALIAS", text: "Keyword_Alias" },
|
||||||
|
{ value: "NAME_REPLACE", text: "Name_Replace" },
|
||||||
{ value: "DESCRIPTION_REPLACE", text: "Description_Replace" },
|
{ value: "DESCRIPTION_REPLACE", text: "Description_Replace" },
|
||||||
{ value: "INSTRUCTION_REPLACE", text: "Instruction_Replace" },
|
{ value: "INSTRUCTION_REPLACE", text: "Instruction_Replace" },
|
||||||
{ value: "TITLE_REPLACE", text: "Title_Replace" },
|
|
||||||
{ value: "FOOD_REPLACE", text: "Food_Replace" },
|
{ value: "FOOD_REPLACE", text: "Food_Replace" },
|
||||||
{ value: "UNIT_REPLACE", text: "Unit_Replace" },
|
{ value: "UNIT_REPLACE", text: "Unit_Replace" },
|
||||||
{ value: "NEVER_UNIT", text: "Never_Unit" },
|
{ value: "NEVER_UNIT", text: "Never_Unit" },
|
||||||
|
Loading…
Reference in New Issue
Block a user