cgiacofei/TandoorRecipes
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

recipe_scrapers deprectation fix

Browse Source
This commit is contained in:
vabene1111 2021-05-01 22:56:00 +02:00
parent ff43492265
commit d1b9d15816
1 changed files with 6 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
cookbook/helper/scrapers/scrapers.py
View File

@ -1,6 +1,6 @@
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from json import JSONDecodeError from json import JSONDecodeError
from recipe_scrapers import SCRAPERS, get_domain, _exception_handling from recipe_scrapers import SCRAPERS, get_host_name
from recipe_scrapers._factory import SchemaScraperFactory from recipe_scrapers._factory import SchemaScraperFactory
from recipe_scrapers._schemaorg import SchemaOrg from recipe_scrapers._schemaorg import SchemaOrg
@ -17,7 +17,7 @@ SCRAPERS.update(CUSTOM_SCRAPERS)
def text_scraper(text, url=None): def text_scraper(text, url=None):
domain = None domain = None
if url: if url:
domain = get_domain(url) domain = get_host_name(url)
if domain in SCRAPERS: if domain in SCRAPERS:
scraper_class = SCRAPERS[domain] scraper_class = SCRAPERS[domain]
else: else:
@ -25,12 +25,12 @@ def text_scraper(text, url=None):
class TextScraper(scraper_class): class TextScraper(scraper_class):
def __init__( def __init__(
self, self,
page_data, page_data,
url=None url=None
): ):
self.wild_mode = False self.wild_mode = False
self.exception_handling = _exception_handling self.exception_handling = None # TODO add new method here, old one was deprecated
self.meta_http_equiv = False self.meta_http_equiv = False
self.soup = BeautifulSoup(page_data, "html.parser") self.soup = BeautifulSoup(page_data, "html.parser")
self.url = url self.url = url