From d36274066ae81a96c3f11b8dd0b054aa1b0241a5 Mon Sep 17 00:00:00 2001 From: vabene1111 Date: Sun, 2 May 2021 13:42:14 +0200 Subject: [PATCH 1/3] some fixed for the new importer + bumped python to 3.9 --- .idea/recipes.iml | 2 +- cookbook/helper/recipe_url_import.py | 21 +++++++++++---------- cookbook/views/api.py | 12 ++---------- docs/install/manual.md | 10 +++++----- docs/install/other.md | 2 +- requirements.txt | 2 +- 6 files changed, 21 insertions(+), 28 deletions(-) diff --git a/.idea/recipes.iml b/.idea/recipes.iml index e1dd7064..6d3000d1 100644 --- a/.idea/recipes.iml +++ b/.idea/recipes.iml @@ -18,7 +18,7 @@ - + diff --git a/cookbook/helper/recipe_url_import.py b/cookbook/helper/recipe_url_import.py index 263a1585..50813775 100644 --- a/cookbook/helper/recipe_url_import.py +++ b/cookbook/helper/recipe_url_import.py @@ -2,6 +2,7 @@ import random import re from isodate import parse_duration as iso_parse_duration from isodate.isoerror import ISO8601Error +from recipe_scrapers._exceptions import ElementNotFoundInHtml from cookbook.helper.ingredient_parser import parse as parse_single_ingredient from cookbook.models import Keyword @@ -17,12 +18,12 @@ def get_from_scraper(scrape, space): recipe_json = {} try: recipe_json['name'] = parse_name(scrape.title() or scrape.schema.data.get('name') or '') - except (TypeError, AttributeError): + except (TypeError, AttributeError,ElementNotFoundInHtml): recipe_json['name'] = '' try: description = scrape.schema.data.get("description") or '' - except AttributeError: + except (AttributeError,ElementNotFoundInHtml): description = '' recipe_json['description'] = parse_description(description) @@ -30,27 +31,27 @@ def get_from_scraper(scrape, space): try: servings = scrape.yields() servings = int(re.findall(r'\b\d+\b', servings)[0]) - except (AttributeError, ValueError, IndexError): + except (AttributeError,ElementNotFoundInHtml, ValueError, IndexError): servings = 1 recipe_json['servings'] = servings try: recipe_json['prepTime'] = get_minutes(scrape.schema.data.get("prepTime")) or 0 - except AttributeError: + except (AttributeError, ElementNotFoundInHtml): recipe_json['prepTime'] = 0 try: recipe_json['cookTime'] = get_minutes(scrape.schema.data.get("cookTime")) or 0 - except AttributeError: + except (AttributeError, ElementNotFoundInHtml): recipe_json['cookTime'] = 0 if recipe_json['cookTime'] + recipe_json['prepTime'] == 0: try: recipe_json['prepTime'] = get_minutes(scrape.total_time()) or 0 - except AttributeError: + except (AttributeError,ElementNotFoundInHtml): pass try: recipe_json['image'] = parse_image(scrape.image()) or '' - except (AttributeError, TypeError, SchemaOrgException): + except (AttributeError,ElementNotFoundInHtml, TypeError, SchemaOrgException): recipe_json['image'] = '' keywords = [] @@ -62,7 +63,7 @@ def get_from_scraper(scrape, space): if scrape.schema.data.get('recipeCuisine'): keywords += listify_keywords(scrape.schema.data.get("recipeCuisine")) recipe_json['keywords'] = parse_keywords(list(set(map(str.casefold, keywords))), space) - except AttributeError: + except (AttributeError,ElementNotFoundInHtml): recipe_json['keywords'] = keywords try: @@ -103,12 +104,12 @@ def get_from_scraper(scrape, space): } ) recipe_json['recipeIngredient'] = ingredients - except AttributeError: + except (AttributeError,ElementNotFoundInHtml): recipe_json['recipeIngredient'] = ingredients try: recipe_json['recipeInstructions'] = parse_instructions(scrape.instructions()) - except AttributeError: + except (AttributeError,ElementNotFoundInHtml): recipe_json['recipeInstructions'] = "" if scrape.url: diff --git a/cookbook/views/api.py b/cookbook/views/api.py index f7f90eef..b2c84f61 100644 --- a/cookbook/views/api.py +++ b/cookbook/views/api.py @@ -16,10 +16,6 @@ from django.http import FileResponse, HttpResponse, JsonResponse from django.shortcuts import redirect, get_object_or_404 from django.utils.translation import gettext as _ from icalendar import Calendar, Event - -from rest_framework import decorators, viewsets -from rest_framework.exceptions import APIException, PermissionDenied - from recipe_scrapers import scrape_me, WebsiteNotImplementedError, NoSchemaFoundInWildMode from rest_framework import decorators, viewsets from rest_framework.exceptions import APIException, PermissionDenied @@ -35,14 +31,10 @@ from cookbook.helper.permission_helper import (CustomIsAdmin, CustomIsGuest, CustomIsOwner, CustomIsShare, CustomIsShared, CustomIsUser, - group_required, share_link_valid) -from cookbook.helper.recipe_html_import import get_recipe_from_source -from cookbook.helper.recipe_url_import import get_from_scraper - group_required) +from cookbook.helper.recipe_html_import import get_recipe_from_source from cookbook.helper.recipe_search import search_recipes -from cookbook.helper.recipe_url_import import get_from_html, get_from_scraper, find_recipe_json - +from cookbook.helper.recipe_url_import import get_from_scraper from cookbook.models import (CookLog, Food, Ingredient, Keyword, MealPlan, MealType, Recipe, RecipeBook, ShoppingList, ShoppingListEntry, ShoppingListRecipe, Step, diff --git a/docs/install/manual.md b/docs/install/manual.md index 7e5b0fcd..7ebd3cf1 100644 --- a/docs/install/manual.md +++ b/docs/install/manual.md @@ -3,7 +3,7 @@ These intructions are inspired from a standard django/gunicorn/postgresql instructions ([for example](https://www.digitalocean.com/community/tutorials/how-to-set-up-django-with-postgres-nginx-and-gunicorn-on-ubuntu-16-04)) !!! warning - Be sure to use pyton3.8 and pip related to python 3.8. Depending on your distribution calling `python` or `pip` will use python2 instead of pyton 3.8. + Be sure to use pyton3.9 and pip related to python 3.9. Depending on your distribution calling `python` or `pip` will use python2 instead of pyton 3.9. ## Prerequisites @@ -12,7 +12,7 @@ These intructions are inspired from a standard django/gunicorn/postgresql instru Get the last version from the repository: `git clone https://github.com/vabene1111/recipes.git -b master` Install postgresql requirements: `sudo apt install libpq-dev postgresql` -Install project requirements: `pip3.8 install -r requirements.txt` +Install project requirements: `pip3.9 install -r requirements.txt` ## Setup postgresql @@ -44,11 +44,11 @@ wget https://raw.githubusercontent.com/vabene1111/recipes/develop/.env.template Execute `export $(cat .env |grep "^[^#]" | xargs)` to load variables from `.env` -Execute `/python3.8 manage.py migrate` +Execute `/python3.9 manage.py migrate` and revert superuser from postgres: `sudo -u postgres psql` and `ALTER USER djangouser WITH NOSUPERUSER;` -Generate static files: `python3.8 manage.py collectstatic` and remember the folder where files have been copied. +Generate static files: `python3.9 manage.py collectstatic` and remember the folder where files have been copied. ## Setup web services @@ -70,7 +70,7 @@ RestartSec=3 Group=www-data WorkingDirectory=/media/data/recipes EnvironmentFile=/media/data/recipes/.env -ExecStart=/opt/.pyenv/versions/3.8.5/bin/gunicorn --error-logfile /tmp/gunicorn_err.log --log-level debug --capture-output --bind unix:/media/data/recipes/recipes.sock recipes.wsgi:application +ExecStart=/opt/.pyenv/versions/3.9/bin/gunicorn --error-logfile /tmp/gunicorn_err.log --log-level debug --capture-output --bind unix:/media/data/recipes/recipes.sock recipes.wsgi:application [Install] WantedBy=multi-user.target diff --git a/docs/install/other.md b/docs/install/other.md index 1221b209..4b42eb09 100644 --- a/docs/install/other.md +++ b/docs/install/other.md @@ -1,7 +1,7 @@ !!! info "Community Contributed" The examples in this section were contributed by members of the community. This page especially contains some setups that might help you if you really want to go down a certain path but none - of the examples are supported (as i simply am not able to give you support for them). + of the examples are supported (as I simply am not able to give you support for them). ## Apache + Traefik + Sub-Path diff --git a/requirements.txt b/requirements.txt index ed4de89c..54913d05 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,7 +31,7 @@ Jinja2==2.11.3 django-webpack-loader==0.7.0 django-js-reverse==0.9.1 django-allauth==0.44.0 -recipe-scrapers==13.1.1 +recipe-scrapers==13.2.1 django-scopes==1.2.0 pytest==6.2.3 pytest-django==4.2.0 From 373df5d99ff5761341e6320271bd3de5257bb108 Mon Sep 17 00:00:00 2001 From: vabene1111 Date: Sun, 2 May 2021 13:44:59 +0200 Subject: [PATCH 2/3] updated python in ci --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a113ead1..1aa68a27 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,14 +9,14 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: [3.8] + python-version: [3.9] steps: - uses: actions/checkout@v1 - - name: Set up Python 3.8 + - name: Set up Python 3.9 uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: 3.9 - name: Install dependencies run: | python -m pip install --upgrade pip From 3baa03396c6fbf858d1b8ff2c59775e1c362778d Mon Sep 17 00:00:00 2001 From: vabene1111 Date: Sun, 2 May 2021 17:33:52 +0200 Subject: [PATCH 3/3] tweaks to the importer --- cookbook/templates/url_import.html | 57 ++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 19 deletions(-) diff --git a/cookbook/templates/url_import.html b/cookbook/templates/url_import.html index 858e0958..b4d11f4f 100644 --- a/cookbook/templates/url_import.html +++ b/cookbook/templates/url_import.html @@ -75,6 +75,25 @@ + + + + + + + @@ -266,18 +285,18 @@
- -
-
+
+

{% trans 'Discovered Attributes' %}

- {% trans 'Drag recipe attributes from below into the appropriate box on the left. Click any node to display its full properties.' %} + {% trans 'Drag recipe attributes from below into the appropriate box on the left. Click any node to display its full properties.' %}
@@ -291,8 +310,8 @@ images
-
@@ -304,9 +323,9 @@
{% trans 'Items dragged to Blank Field will be appended.'%}
-
@@ -324,7 +343,7 @@
-
@@ -335,17 +354,17 @@
- +
    -
    - [[txt]] + style="display:flex; justify-content:space-between;"> + [[txt]]
@@ -353,10 +372,10 @@
    -
    + style="display:flex; justify-content:space-between;"> Image