cgiacofei/TandoorRecipes
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

bug fix url import

Browse Source
This commit is contained in:
smilerz 2021-04-05 21:41:00 -05:00
parent 747d146389
commit 2fe6788ce5
6 changed files with 54 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cookbook/helper/recipe_html_import.py
View File

@ -82,7 +82,12 @@ def get_recipe_from_source(text, url, space):
html_data = get_from_html(soup)
images += get_images_from_source(soup, url)
for el in soup.find_all('script', type='application/ld+json'):
parse_list.append(remove_graph(el))
el = remove_graph(el)
if type(el) == list:
for l in el:
parse_list.append(l)
else:
parse_list.append(el)
for el in soup.find_all(type='application/json'):
parse_list.append(remove_graph(el))

8
cookbook/helper/recipe_url_import.py
View File

@ -39,8 +39,9 @@ def get_from_scraper(scrape, space):
pass
try:
recipe_json['image'] = scrape.image()
except AttributeError:
recipe_json['image'] = parse_image(scrape.image())
except (AttributeError, TypeError):
recipe_json['image'] = ''
pass
keywords = []
@ -283,7 +284,8 @@ def parse_keywords(keyword_json, space):
for kw in keyword_json:
kw = normalize_string(kw)
if k := Keyword.objects.filter(name=kw, space=space).first():
keywords.append({'id': str(k.id), 'text': str(k)})
if len (k['text']) > 0:
keywords.append({'id': str(k.id), 'text': str(k)})
else:
keywords.append({'id': random.randrange(1111111, 9999999, 1), 'text': kw})

1
cookbook/helper/scrapers/cooksillustrated.py
View File

@ -55,7 +55,6 @@ class CooksIllustrated(AbstractScraper):
raise NotImplementedError("This should be implemented.")
def get_recipe(self):
# TODO add missing data to schema.data
j = json.loads(self.soup.find(type='application/json').string)
name = list(j['props']['initialState']['content']['documents'])[0]
self.recipe = j['props']['initialState']['content']['documents'][name]

33
cookbook/tests/other/_recipes.py
View File

@ -128,8 +128,10 @@ ALLRECIPES = {
}
AMERICAS_TEST_KITCHEN = {
"file": ['americastestkitchen.html'],
'url': 'http://www.americastestkitchen.com/recipes/1771-pasta-with-chicken-broccoli-and-sun-dried-tomatoes',
"file": [
'americastestkitchen.html',
'cookscountry.html'
],
"name": "Pasta with Chicken, Broccoli, and Sun-dried Tomatoes",
"description": "This restaurant-chain classic can be as off-putting as a bad horror movie: drab colors, tough meat, and a main character—the pasta—with no bite.",
"servings": 4,
@ -138,12 +140,13 @@ AMERICAS_TEST_KITCHEN = {
"image": "https://res.cloudinary.com/hksqkdlah/image/upload/ar_1:1,c_fill,dpr_2.0,f_auto,fl_lossy.progressive.strip_profile,g_faces:auto,q_auto:low,w_150/33255_sfs-pasta-with-chicken-broccoli-sun-dried-tomatoes-15",
"keywords": [
{
"id": "21",
"text": "main courses"
}
],
"recipeIngredient": [
{
"amount": 4.0,
"amount": 4,
"unit": {
"text": "tablespoons"
},
@ -154,7 +157,7 @@ AMERICAS_TEST_KITCHEN = {
"original": "4 tablespoons unsalted butter"
},
{
"amount": 1.0,
"amount": 1,
"unit": {
"text": "pound"
},
@ -165,7 +168,7 @@ AMERICAS_TEST_KITCHEN = {
"original": "1 pound boneless, skinless chicken breast, trimmed of fat and cut crosswise into 1/4-inch slices"
},
{
"amount": 1.0,
"amount": 1,
"unit": {
"text": "small"
},
@ -187,7 +190,7 @@ AMERICAS_TEST_KITCHEN = {
"original": " Table salt"
},
{
"amount": 6.0,
"amount": 6,
"unit": {
"text": "medium"
},
@ -209,7 +212,7 @@ AMERICAS_TEST_KITCHEN = {
"original": "1/4 teaspoon red pepper flakes"
},
{
"amount": 2.0,
"amount": 2,
"unit": {
"text": "teaspoons"
},
@ -220,7 +223,7 @@ AMERICAS_TEST_KITCHEN = {
"original": "2 teaspoons chopped fresh thyme leaves"
},
{
"amount": 2.0,
"amount": 2,
"unit": {
"text": "teaspoons"
},
@ -231,7 +234,7 @@ AMERICAS_TEST_KITCHEN = {
"original": "2 teaspoons all-purpose flour"
},
{
"amount": 1.0,
"amount": 1,
"unit": {
"text": "cup"
},
@ -242,7 +245,7 @@ AMERICAS_TEST_KITCHEN = {
"original": "1 cup dry white wine"
},
{
"amount": 2.0,
"amount": 2,
"unit": {
"text": "cups"
},
@ -253,7 +256,7 @@ AMERICAS_TEST_KITCHEN = {
"original": "2 cups low-sodium chicken broth"
},
{
"amount": 1.0,
"amount": 1,
"unit": {
"text": "bunch"
},
@ -275,7 +278,7 @@ AMERICAS_TEST_KITCHEN = {
"original": "1/2 pound penne pasta, ziti, cavatappi, or campanelle"
},
{
"amount": 2.0,
"amount": 2,
"unit": {
"text": "ounces"
},
@ -286,7 +289,7 @@ AMERICAS_TEST_KITCHEN = {
"original": "2 ounces grated Asiago cheese(1 cup), plus extra for serving"
},
{
"amount": 1.0,
"amount": 1,
"unit": {
"text": "jar"
},
@ -297,7 +300,7 @@ AMERICAS_TEST_KITCHEN = {
"original": "1 jar oil-packed sun-dried tomato(7 to 8 1/2 ounces), rinsed, patted dry, and cut into 1/4-inch strips (about 1 cup)"
},
{
"amount": 1.0,
"amount": 1,
"unit": {
"text": "tablespoon"
},
@ -319,7 +322,7 @@ AMERICAS_TEST_KITCHEN = {
"original": " Ground black pepper"
}
],
"recipeInstructions": "<p>Lightly browning chicken breast strips in butter started building flavor into our pasta with chicken recipe. We kept the chicken tender and added more flavor by letting the strips finish cooking in the sauce, and we kept the broccoli fresh and crisp by blanching it in the boiling pasta water and then putting it aside until the dish was assembled. But our real breakthrough in developing this pasta and chicken recipe was to eliminate the cream typically used in this dish and to create instead a broth-based sauce, which we rounded out with a few tablespoons of butter, a handful of Asiago cheese, and some sun-dried tomatoes.</p>\nBring 4 quarts water to rolling boil, covered, in stockpot.\nMeanwhile, heat 1 tablespoon butter in 12-inch nonstick skillet over high heat until just beginning to brown, about 1 minute. Add chicken in single layer; cook for 1 minute without stirring, then stir chicken and continue to cook until most, but not all, of pink color has disappeared and chicken is lightly browned around the edges, about 2 minutes longer. Transfer chicken to clean bowl; set aside.\nReturn skillet to high heat and add 1 tablespoon butter; add onion and 1/4 teaspoon salt and cook, stirring occasionally, until browned about edges, 2 to 3 minutes. Stir in garlic, red pepper flakes, thyme, and flour; cook, stirring constantly, until fragrant, about 30 seconds. Add wine and chicken broth; bring to simmer, then reduce heat to medium and continue to simmer, stirring occasionally, until sauce has thickened slightly and reduced to 1 1/4 cups, about 15 minutes.\nWhile sauce simmers, add 1 tablespoon salt and broccoli to boiling water; cook until broccoli is tender but still crisp at center, about 2 minutes. Using slotted spoon, transfer broccoli to large paper towel-lined plate. Return water to boil; stir in pasta and cook until al dente. Drain, reserving 1/2 cup pasta cooking water; return pasta to pot.\nStir remaining 2 tablespoons butter, Asiago, sun-dried tomatoes, parsley, and chicken into sauce in skillet; cook until chicken is hot and cooked through, about 1 minute. Off heat, season to taste with pepper. Pour chicken/sauce mixture over pasta and add broccoli; toss gently to combine, adding pasta cooking water as needed to adjust sauce consistency. Serve immediately, passing additional Asiago and the lemon wedges (if using) separately.\n\nImported from http://www.americastestkitchen.com/recipes/1771-pasta-with-chicken-broccoli-and-sun-dried-tomatoes"
"recipeInstructions": "<p>Lightly browning chicken breast strips in butter started building flavor into our pasta with chicken recipe. We kept the chicken tender and added more flavor by letting the strips finish cooking in the sauce, and we kept the broccoli fresh and crisp by blanching it in the boiling pasta water and then putting it aside until the dish was assembled. But our real breakthrough in developing this pasta and chicken recipe was to eliminate the cream typically used in this dish and to create instead a broth-based sauce, which we rounded out with a few tablespoons of butter, a handful of Asiago cheese, and some sun-dried tomatoes.</p>\nBring 4 quarts water to rolling boil, covered, in stockpot.\nMeanwhile, heat 1 tablespoon butter in 12-inch nonstick skillet over high heat until just beginning to brown, about 1 minute. Add chicken in single layer; cook for 1 minute without stirring, then stir chicken and continue to cook until most, but not all, of pink color has disappeared and chicken is lightly browned around the edges, about 2 minutes longer. Transfer chicken to clean bowl; set aside.\nReturn skillet to high heat and add 1 tablespoon butter; add onion and 1/4 teaspoon salt and cook, stirring occasionally, until browned about edges, 2 to 3 minutes. Stir in garlic, red pepper flakes, thyme, and flour; cook, stirring constantly, until fragrant, about 30 seconds. Add wine and chicken broth; bring to simmer, then reduce heat to medium and continue to simmer, stirring occasionally, until sauce has thickened slightly and reduced to 1 1/4 cups, about 15 minutes.\nWhile sauce simmers, add 1 tablespoon salt and broccoli to boiling water; cook until broccoli is tender but still crisp at center, about 2 minutes. Using slotted spoon, transfer broccoli to large paper towel-lined plate. Return water to boil; stir in pasta and cook until al dente. Drain, reserving 1/2 cup pasta cooking water; return pasta to pot.\nStir remaining 2 tablespoons butter, Asiago, sun-dried tomatoes, parsley, and chicken into sauce in skillet; cook until chicken is hot and cooked through, about 1 minute. Off heat, season to taste with pepper. Pour chicken/sauce mixture over pasta and add broccoli; toss gently to combine, adding pasta cooking water as needed to adjust sauce consistency. Serve immediately, passing additional Asiago and the lemon wedges (if using) separately.\n\nImported from http://www.americastestkitchen.com/recipes/1771-pasta-with-chicken-broccoli-and-sun-dried-tomatoes?extcode=MCSAD10L0&ref=new_search_experience_1"
}
CHEF_KOCH = {

26
cookbook/tests/other/test_url_import.py
View File

@ -1,9 +1,31 @@
import json
import pytest
from django.urls import reverse
from django_scopes import scopes_disabled
from ._recipes import (AMERICAS_TEST_KITCHEN)
from cookbook.helper.recipe_url_import import get_from_html
IMPORT_SOURCE_URL = 'data_import_source'
# These were chosen arbitrarily from:
# Top 10 recipe websites listed here https://www.similarweb.com/top-websites/category/food-and-drink/cooking-and-recipes/
# plus the test that previously existed
# custom scraper that was created
# TODO thoughtfully add recipes that test specific scenerios
@pytest.mark.parametrize("arg", [
['a_u', 403],
['g1_s1', 404],
['u1_s1', 200],
['a1_s1', 404],
['g1_s2', 404],
['u1_s2', 404],
['a1_s2', 404],
])
def test_import_permission(arg, request):
c = request.getfixturevalue(arg[0])
assert c.get(reverse(IMPORT_SOURCE_URL)).status_code == arg[1]
# TODO this test is really bad, need to find a better solution, also pytest does not like those paths
# def test_ld_json():
# with scopes_disabled():

2
cookbook/views/api.py
View File

@ -593,7 +593,7 @@ def recipe_from_source(request):
},
status=400)
else:
return JsonResponse(get_from_scraper(scrape, request.space))
return JsonResponse({"recipe_json": get_from_scraper(scrape, request.space)})
else:
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Safari/537.36' # noqa: E501