cgiacofei/TandoorRecipes
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

bug fix url import

Browse Source
This commit is contained in:
smilerz 2021-04-05 21:41:00 -05:00
parent c50bd039ef
commit 4bc4ce0d7c
6 changed files with 63 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cookbook/helper/recipe_html_import.py
View File

@ -82,7 +82,12 @@ def get_recipe_from_source(text, url, space):
html_data = get_from_html(soup)
images += get_images_from_source(soup, url)
for el in soup.find_all('script', type='application/ld+json'):
parse_list.append(remove_graph(el))
el = remove_graph(el)
if type(el) == list:
for l in el:
parse_list.append(l)
else:
parse_list.append(el)
for el in soup.find_all(type='application/json'):
parse_list.append(remove_graph(el))

8
cookbook/helper/recipe_url_import.py
View File

@ -39,8 +39,9 @@ def get_from_scraper(scrape, space):
pass
try:
recipe_json['image'] = scrape.image()
except AttributeError:
recipe_json['image'] = parse_image(scrape.image())
except (AttributeError, TypeError):
recipe_json['image'] = ''
pass
keywords = []
@ -283,7 +284,8 @@ def parse_keywords(keyword_json, space):
for kw in keyword_json:
kw = normalize_string(kw)
if k := Keyword.objects.filter(name=kw, space=space).first():
keywords.append({'id': str(k.id), 'text': str(k)})
if len (k['text']) > 0:
keywords.append({'id': str(k.id), 'text': str(k)})
else:
keywords.append({'id': random.randrange(1111111, 9999999, 1), 'text': kw})

1
cookbook/helper/scrapers/cooksillustrated.py
View File

@ -57,7 +57,6 @@ class CooksIllustrated(AbstractScraper):
raise NotImplementedError("This should be implemented.")
def get_recipe(self):
# TODO add missing data to schema.data
j = json.loads(self.soup.find(type='application/json').string)
name = list(j['props']['initialState']['content']['documents'])[0]
self.recipe = j['props']['initialState']['content']['documents'][name]

33
cookbook/tests/other/_recipes.py
View File

@ -128,8 +128,10 @@ ALLRECIPES = {
}
AMERICAS_TEST_KITCHEN = {
"file": ['americastestkitchen.html'],
'url': 'http://www.americastestkitchen.com/recipes/1771-pasta-with-chicken-broccoli-and-sun-dried-tomatoes',
"file": [
'americastestkitchen.html',
'cookscountry.html'
],
"name": "Pasta with Chicken, Broccoli, and Sun-dried Tomatoes",
"description": "This restaurant-chain classic can be as off-putting as a bad horror movie: drab colors, tough meat, and a main character—the pasta—with no bite.",
"servings": 4,
@ -138,12 +140,13 @@ AMERICAS_TEST_KITCHEN = {
"image": "https://res.cloudinary.com/hksqkdlah/image/upload/ar_1:1,c_fill,dpr_2.0,f_auto,fl_lossy.progressive.strip_profile,g_faces:auto,q_auto:low,w_150/33255_sfs-pasta-with-chicken-broccoli-sun-dried-tomatoes-15",
"keywords": [
{
"id": "21",
"text": "main courses"
}
],
"recipeIngredient": [
{
"amount": 4.0,
"amount": 4,
"unit": {
"text": "tablespoons"
},
@ -154,7 +157,7 @@ AMERICAS_TEST_KITCHEN = {
"original": "4 tablespoons unsalted butter"
},
{
"amount": 1.0,
"amount": 1,
"unit": {
"text": "pound"
},
@ -165,7 +168,7 @@ AMERICAS_TEST_KITCHEN = {
"original": "1 pound boneless, skinless chicken breast, trimmed of fat and cut crosswise into 1/4-inch slices"
},
{
"amount": 1.0,
"amount": 1,
"unit": {
"text": "small"
},
@ -187,7 +190,7 @@ AMERICAS_TEST_KITCHEN = {
"original": " Table salt"
},
{
"amount": 6.0,
"amount": 6,
"unit": {
"text": "medium"
},
@ -209,7 +212,7 @@ AMERICAS_TEST_KITCHEN = {
"original": "1/4 teaspoon red pepper flakes"
},
{
"amount": 2.0,
"amount": 2,
"unit": {
"text": "teaspoons"
},
@ -220,7 +223,7 @@ AMERICAS_TEST_KITCHEN = {
"original": "2 teaspoons chopped fresh thyme leaves"
},
{
"amount": 2.0,
"amount": 2,
"unit": {
"text": "teaspoons"
},
@ -231,7 +234,7 @@ AMERICAS_TEST_KITCHEN = {
"original": "2 teaspoons all-purpose flour"
},
{
"amount": 1.0,
"amount": 1,
"unit": {
"text": "cup"
},
@ -242,7 +245,7 @@ AMERICAS_TEST_KITCHEN = {
"original": "1 cup dry white wine"
},
{
"amount": 2.0,
"amount": 2,
"unit": {
"text": "cups"
},
@ -253,7 +256,7 @@ AMERICAS_TEST_KITCHEN = {
"original": "2 cups low-sodium chicken broth"
},
{
"amount": 1.0,
"amount": 1,
"unit": {
"text": "bunch"
},
@ -275,7 +278,7 @@ AMERICAS_TEST_KITCHEN = {
"original": "1/2 pound penne pasta, ziti, cavatappi, or campanelle"
},
{
"amount": 2.0,
"amount": 2,
"unit": {
"text": "ounces"
},
@ -286,7 +289,7 @@ AMERICAS_TEST_KITCHEN = {
"original": "2 ounces grated Asiago cheese(1 cup), plus extra for serving"
},
{
"amount": 1.0,
"amount": 1,
"unit": {
"text": "jar"
},
@ -297,7 +300,7 @@ AMERICAS_TEST_KITCHEN = {
"original": "1 jar oil-packed sun-dried tomato(7 to 8 1/2 ounces), rinsed, patted dry, and cut into 1/4-inch strips (about 1 cup)"
},
{
"amount": 1.0,
"amount": 1,
"unit": {
"text": "tablespoon"
},
@ -319,7 +322,7 @@ AMERICAS_TEST_KITCHEN = {
"original": " Ground black pepper"
}
],
"recipeInstructions": "<p>Lightly browning chicken breast strips in butter started building flavor into our pasta with chicken recipe. We kept the chicken tender and added more flavor by letting the strips finish cooking in the sauce, and we kept the broccoli fresh and crisp by blanching it in the boiling pasta water and then putting it aside until the dish was assembled. But our real breakthrough in developing this pasta and chicken recipe was to eliminate the cream typically used in this dish and to create instead a broth-based sauce, which we rounded out with a few tablespoons of butter, a handful of Asiago cheese, and some sun-dried tomatoes.</p>\nBring 4 quarts water to rolling boil, covered, in stockpot.\nMeanwhile, heat 1 tablespoon butter in 12-inch nonstick skillet over high heat until just beginning to brown, about 1 minute. Add chicken in single layer; cook for 1 minute without stirring, then stir chicken and continue to cook until most, but not all, of pink color has disappeared and chicken is lightly browned around the edges, about 2 minutes longer. Transfer chicken to clean bowl; set aside.\nReturn skillet to high heat and add 1 tablespoon butter; add onion and 1/4 teaspoon salt and cook, stirring occasionally, until browned about edges, 2 to 3 minutes. Stir in garlic, red pepper flakes, thyme, and flour; cook, stirring constantly, until fragrant, about 30 seconds. Add wine and chicken broth; bring to simmer, then reduce heat to medium and continue to simmer, stirring occasionally, until sauce has thickened slightly and reduced to 1 1/4 cups, about 15 minutes.\nWhile sauce simmers, add 1 tablespoon salt and broccoli to boiling water; cook until broccoli is tender but still crisp at center, about 2 minutes. Using slotted spoon, transfer broccoli to large paper towel-lined plate. Return water to boil; stir in pasta and cook until al dente. Drain, reserving 1/2 cup pasta cooking water; return pasta to pot.\nStir remaining 2 tablespoons butter, Asiago, sun-dried tomatoes, parsley, and chicken into sauce in skillet; cook until chicken is hot and cooked through, about 1 minute. Off heat, season to taste with pepper. Pour chicken/sauce mixture over pasta and add broccoli; toss gently to combine, adding pasta cooking water as needed to adjust sauce consistency. Serve immediately, passing additional Asiago and the lemon wedges (if using) separately.\n\nImported from http://www.americastestkitchen.com/recipes/1771-pasta-with-chicken-broccoli-and-sun-dried-tomatoes"
"recipeInstructions": "<p>Lightly browning chicken breast strips in butter started building flavor into our pasta with chicken recipe. We kept the chicken tender and added more flavor by letting the strips finish cooking in the sauce, and we kept the broccoli fresh and crisp by blanching it in the boiling pasta water and then putting it aside until the dish was assembled. But our real breakthrough in developing this pasta and chicken recipe was to eliminate the cream typically used in this dish and to create instead a broth-based sauce, which we rounded out with a few tablespoons of butter, a handful of Asiago cheese, and some sun-dried tomatoes.</p>\nBring 4 quarts water to rolling boil, covered, in stockpot.\nMeanwhile, heat 1 tablespoon butter in 12-inch nonstick skillet over high heat until just beginning to brown, about 1 minute. Add chicken in single layer; cook for 1 minute without stirring, then stir chicken and continue to cook until most, but not all, of pink color has disappeared and chicken is lightly browned around the edges, about 2 minutes longer. Transfer chicken to clean bowl; set aside.\nReturn skillet to high heat and add 1 tablespoon butter; add onion and 1/4 teaspoon salt and cook, stirring occasionally, until browned about edges, 2 to 3 minutes. Stir in garlic, red pepper flakes, thyme, and flour; cook, stirring constantly, until fragrant, about 30 seconds. Add wine and chicken broth; bring to simmer, then reduce heat to medium and continue to simmer, stirring occasionally, until sauce has thickened slightly and reduced to 1 1/4 cups, about 15 minutes.\nWhile sauce simmers, add 1 tablespoon salt and broccoli to boiling water; cook until broccoli is tender but still crisp at center, about 2 minutes. Using slotted spoon, transfer broccoli to large paper towel-lined plate. Return water to boil; stir in pasta and cook until al dente. Drain, reserving 1/2 cup pasta cooking water; return pasta to pot.\nStir remaining 2 tablespoons butter, Asiago, sun-dried tomatoes, parsley, and chicken into sauce in skillet; cook until chicken is hot and cooked through, about 1 minute. Off heat, season to taste with pepper. Pour chicken/sauce mixture over pasta and add broccoli; toss gently to combine, adding pasta cooking water as needed to adjust sauce consistency. Serve immediately, passing additional Asiago and the lemon wedges (if using) separately.\n\nImported from http://www.americastestkitchen.com/recipes/1771-pasta-with-chicken-broccoli-and-sun-dried-tomatoes?extcode=MCSAD10L0&ref=new_search_experience_1"
}
CHEF_KOCH = {

106
cookbook/tests/other/test_url_import.py
View File

@ -1,88 +1,50 @@
import json
import pytest
from django.urls import reverse
from ._recipes import (
ALLRECIPES, AMERICAS_TEST_KITCHEN, CHEF_KOCH, COOKPAD,
COOKS_COUNTRY, DELISH, FOOD_NETWORK, GIALLOZAFFERANO, JOURNAL_DES_FEMMES,
MADAME_DESSERT, MARMITON, TASTE_OF_HOME, THE_SPRUCE_EATS, TUDOGOSTOSO)
from ._recipes import (AMERICAS_TEST_KITCHEN)
IMPORT_SOURCE_URL = 'api_recipe_from_source'
DATA_DIR = "cookbook/tests/other/test_data/"
IMPORT_SOURCE_URL = 'data_import_source'
# These were chosen arbitrarily from:
# Top 10 recipe websites listed here https://www.similarweb.com/top-websites/category/food-and-drink/cooking-and-recipes/
# plus the test that previously existed
# plus the custom scraper that was created
# custom scraper that was created
# TODO thoughtfully add recipes that test specific scenerios
@pytest.mark.parametrize("arg", [
['a_u', 302],
['g1_s1', 302],
['u1_s1', 400],
['a1_s1', 400],
['a_u', 403],
['g1_s1', 404],
['u1_s1', 200],
['a1_s1', 404],
['g1_s2', 404],
['u1_s2', 404],
['a1_s2', 404],
])
def test_import_permission(arg, request):
c = request.getfixturevalue(arg[0])
assert c.get(reverse(IMPORT_SOURCE_URL)).status_code == arg[1]
@pytest.mark.parametrize("arg", [
ALLRECIPES,
# test of custom scraper ATK
AMERICAS_TEST_KITCHEN,
CHEF_KOCH,
COOKPAD,
# test of custom scraper ATK
COOKS_COUNTRY,
DELISH,
FOOD_NETWORK,
GIALLOZAFFERANO,
JOURNAL_DES_FEMMES,
# example of recipes_scraper in with wildmode
# example of json only source
MADAME_DESSERT,
MARMITON,
TASTE_OF_HOME,
# example of non-json recipes_scraper
THE_SPRUCE_EATS,
TUDOGOSTOSO
])
def test_recipe_import(arg, u1_s1):
for f in arg['file']:
with open(DATA_DIR + f, 'r', encoding='UTF-8') as d:
response = u1_s1.post(
reverse(IMPORT_SOURCE_URL),
{
'data': d.read(),
'url': arg['url'],
'mode': 'source'
},
files={'foo': 'bar'}
)
recipe = json.loads(response.content)['recipe_json']
for key in list(set(arg) - set(['file', 'url'])):
if type(arg[key]) == list:
assert len(recipe[key]) == len(arg[key])
if key == 'keywords':
valid_keywords = [i['text'] for i in arg[key]]
for k in recipe[key]:
assert k['text'] in valid_keywords
elif key == 'recipeIngredient':
valid_ing = ["{:g}{}{}{}{}".format(
i['amount'],
i['unit']['text'],
i['ingredient']['text'],
i['note'],
i['original'])
for i in arg[key]]
for i in recipe[key]:
assert "{:g}{}{}{}{}".format(
i['amount'],
i['unit']['text'],
i['ingredient']['text'],
i['note'],
i['original']) in valid_ing
else:
assert recipe[key] == arg[key]
# TODO this test is really bad, need to find a better solution, also pytest does not like those paths
# def test_ld_json():
# with scopes_disabled():
# test_list = [
# {'file': 'resources/websites/ld_json_1.html', 'result_length': 3237},
# {'file': 'resources/websites/ld_json_2.html', 'result_length': 1525},
# {'file': 'resources/websites/ld_json_3.html', 'result_length': 1644},
# {'file': 'resources/websites/ld_json_4.html', 'result_length': 1744},
# {'file': 'resources/websites/ld_json_itemList.html', 'result_length': 3222},
# {'file': 'resources/websites/ld_json_multiple.html', 'result_length': 1621},
# {'file': 'resources/websites/micro_data_1.html', 'result_length': 1094},
# {'file': 'resources/websites/micro_data_2.html', 'result_length': 1453},
# {'file': 'resources/websites/micro_data_3.html', 'result_length': 1163},
# {'file': 'resources/websites/micro_data_4.html', 'result_length': 4411},
# ]
#
# for test in test_list:
# with open(test['file'], 'rb') as file:
# print(f'Testing {test["file"]} expecting length {test["result_length"]}')
# parsed_content = json.loads(get_from_html(file.read(), 'test_url', None).content)
# assert len(str(parsed_content)) == test['result_length']
# file.close()

25
cookbook/views/api.py
View File

@ -614,31 +614,6 @@ def get_plan_ical(request, from_date, to_date):
return response
@group_required('user')
def recipe_from_json(request):
mjson = request.POST['json']
md_json = json.loads(mjson)
for ld_json_item in md_json:
# recipes type might be wrapped in @graph type
if '@graph' in ld_json_item:
for x in md_json['@graph']:
if '@type' in x and x['@type'] == 'Recipe':
md_json = x
if ('@type' in md_json
and md_json['@type'] == 'Recipe'):
return JsonResponse(find_recipe_json(md_json, '', request.space))
return JsonResponse(
{
'error': True,
'msg': _('Could not parse correctly...')
},
status=400
)
@group_required('user')
def recipe_from_source(request):
url = request.POST.get('url', None)