manually parse json
This commit is contained in:
smilerz
@ -1,16 +1,16 @@
|
||||
import json
|
||||
import re
|
||||
from json.decoder import JSONDecodeError
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
from bs4.element import Tag
|
||||
# from cookbook.helper.ingredient_parser import parse as parse_ingredient
|
||||
from cookbook.helper import recipe_url_import as helper
|
||||
from cookbook.helper.scrapers.scrapers import text_scraper
|
||||
from json import JSONDecodeError
|
||||
from recipe_scrapers._utils import get_host_name, normalize_string
|
||||
from urllib.parse import unquote
|
||||
|
||||
|
||||
def get_recipe_from_source(text, url, space):
|
||||
# %%
|
||||
|
||||
# %%
|
||||
def get_from_raw(text):
|
||||
def build_node(k, v):
|
||||
if isinstance(v, dict):
|
||||
node = {
|
||||
@ -26,8 +26,8 @@ def get_recipe_from_source(text, url, space):
|
||||
}
|
||||
else:
|
||||
node = {
|
||||
'name': k + ": " + normalize_string(str(v)),
|
||||
'value': normalize_string(str(v))
|
||||
'name': k + ": " + str(v),
|
||||
'value': str(v)
|
||||
}
|
||||
return node
|
||||
|
||||
@ -52,14 +52,13 @@ def get_recipe_from_source(text, url, space):
|
||||
kid_list.append(build_node(k, v))
|
||||
else:
|
||||
kid_list.append({
|
||||
'name': normalize_string(str(kid)),
|
||||
'value': normalize_string(str(kid))
|
||||
'name': kid,
|
||||
'value': kid
|
||||
})
|
||||
return kid_list
|
||||
|
||||
recipe_json = {
|
||||
'name': '',
|
||||
'url': '',
|
||||
'description': '',
|
||||
'image': '',
|
||||
'keywords': [],
|
||||
@ -68,51 +67,26 @@ def get_recipe_from_source(text, url, space):
|
||||
'servings': '',
|
||||
'prepTime': '',
|
||||
'cookTime': ''
|
||||
}
|
||||
}
|
||||
recipe_tree = []
|
||||
temp_tree = []
|
||||
parse_list = []
|
||||
html_data = []
|
||||
images = []
|
||||
text = unquote(text)
|
||||
|
||||
try:
|
||||
parse_list.append(remove_graph(json.loads(text)))
|
||||
if not url and 'url' in parse_list[0]:
|
||||
url = parse_list[0]['url']
|
||||
scrape = text_scraper("<script type='application/ld+json'>" + text + "</script>", url=url)
|
||||
|
||||
parse_list.append(json.loads(text))
|
||||
except JSONDecodeError:
|
||||
soup = BeautifulSoup(text, "html.parser")
|
||||
html_data = get_from_html(soup)
|
||||
images += get_images_from_source(soup, url)
|
||||
for el in soup.find_all('script', type='application/ld+json'):
|
||||
el = remove_graph(el)
|
||||
if not url and 'url' in el:
|
||||
url = el['url']
|
||||
if type(el) == list:
|
||||
for le in el:
|
||||
parse_list.append(le)
|
||||
elif type(el) == dict:
|
||||
parse_list.append(el)
|
||||
parse_list.append(el)
|
||||
for el in soup.find_all(type='application/json'):
|
||||
el = remove_graph(el)
|
||||
if type(el) == list:
|
||||
for le in el:
|
||||
parse_list.append(le)
|
||||
elif type(el) == dict:
|
||||
parse_list.append(el)
|
||||
scrape = text_scraper(text, url=url)
|
||||
|
||||
recipe_json = helper.get_from_scraper(scrape, space)
|
||||
parse_list.append(el)
|
||||
|
||||
# first try finding ld+json as its most common
|
||||
for el in parse_list:
|
||||
temp_tree = []
|
||||
if isinstance(el, Tag):
|
||||
try:
|
||||
el = json.loads(el.string)
|
||||
except TypeError:
|
||||
continue
|
||||
|
||||
if isinstance(el, Tag):
|
||||
el = json.loads(el.string)
|
||||
|
||||
for k, v in el.items():
|
||||
if isinstance(v, dict):
|
||||
node = {
|
||||
@ -128,66 +102,22 @@ def get_recipe_from_source(text, url, space):
|
||||
}
|
||||
else:
|
||||
node = {
|
||||
'name': k + ": " + normalize_string(str(v)),
|
||||
'value': normalize_string(str(v))
|
||||
'name': k + ": " + str(v),
|
||||
'value': str(v)
|
||||
}
|
||||
temp_tree.append(node)
|
||||
|
||||
if '@type' in el and el['@type'] == 'Recipe':
|
||||
if ('@type' in el and el['@type'] == 'Recipe'):
|
||||
recipe_json = helper.find_recipe_json(el, None)
|
||||
recipe_tree += [{'name': 'ld+json', 'children': temp_tree}]
|
||||
else:
|
||||
recipe_tree += [{'name': 'json', 'children': temp_tree}]
|
||||
|
||||
return recipe_json, recipe_tree, html_data, images
|
||||
temp_tree = []
|
||||
|
||||
# overide keyword structure from dict to list
|
||||
kws = []
|
||||
for kw in recipe_json['keywords']:
|
||||
kws.append(kw['text'])
|
||||
recipe_json['keywords'] = kws
|
||||
|
||||
def get_from_html(soup):
|
||||
INVISIBLE_ELEMS = ('style', 'script', 'head', 'title')
|
||||
html = []
|
||||
for s in soup.strings:
|
||||
if ((s.parent.name not in INVISIBLE_ELEMS) and (len(s.strip()) > 0)):
|
||||
html.append(s)
|
||||
return html
|
||||
|
||||
|
||||
def get_images_from_source(soup, url):
|
||||
sources = ['src', 'srcset', 'data-src']
|
||||
images = []
|
||||
img_tags = soup.find_all('img')
|
||||
if url:
|
||||
site = get_host_name(url)
|
||||
prot = url.split(':')[0]
|
||||
|
||||
urls = []
|
||||
for img in img_tags:
|
||||
for src in sources:
|
||||
try:
|
||||
urls.append(img[src])
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
for u in urls:
|
||||
u = u.split('?')[0]
|
||||
filename = re.search(r'/([\w_-]+[.](jpg|jpeg|gif|png))$', u)
|
||||
if filename:
|
||||
if (('http' not in u) and (url)):
|
||||
# sometimes an image source can be relative
|
||||
# if it is provide the base url
|
||||
u = '{}://{}{}'.format(prot, site, u)
|
||||
if 'http' in u:
|
||||
images.append(u)
|
||||
return images
|
||||
|
||||
|
||||
def remove_graph(el):
|
||||
# recipes type might be wrapped in @graph type
|
||||
if isinstance(el, Tag):
|
||||
try:
|
||||
el = json.loads(el.string)
|
||||
if '@graph' in el:
|
||||
for x in el['@graph']:
|
||||
if '@type' in x and x['@type'] == 'Recipe':
|
||||
el = x
|
||||
except TypeError:
|
||||
pass
|
||||
return el
|
||||
return recipe_json, recipe_tree
|
||||
|
||||
Reference in New Issue
Block a user