cgiacofei/TandoorRecipes
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1300 from TandoorRecipes/performance_refactor

Browse Source 
Performance refactor
This commit is contained in:
vabene1111 2022-01-13 17:06:04 +01:00 committed by GitHub
commit a2dc8d8988
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 537 additions and 225 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cookbook/helper/HelperFunctions.py
View File

@ -7,7 +7,7 @@ class Round(Func):
def str2bool(v):
if type(v) == bool:
if type(v) == bool or v is None:
return v
else:
return v.lower() in ("yes", "true", "1")

562
cookbook/helper/recipe_search.py
View File

@ -3,8 +3,8 @@ from datetime import timedelta
from django.contrib.postgres.search import SearchQuery, SearchRank, TrigramSimilarity
from django.core.cache import caches
from django.db.models import Avg, Case, Count, Func, Max, Q, Subquery, Value, When
from django.db.models.functions import Coalesce
from django.db.models import Avg, Case, Count, Func, Max, OuterRef, Q, Subquery, Value, When
from django.db.models.functions import Coalesce, Substr
from django.utils import timezone, translation
from cookbook.filters import RecipeFilter
@ -145,6 +145,8 @@ def search_recipes(request, queryset, params):
# TODO creating setting to include descendants of keywords a setting
# for kw in Keyword.objects.filter(pk__in=search_keywords):
# search_keywords += list(kw.get_descendants().values_list('pk', flat=True))
for kw in Keyword.objects.filter(pk__in=search_keywords):
search_keywords = [*search_keywords, *list(kw.get_descendants_and_self().values_list('pk', flat=True))]
queryset = queryset.filter(keywords__id__in=search_keywords)
else:
# when performing an 'and' search returned recipes should include a parent OR any of its descedants
@ -155,6 +157,8 @@ def search_recipes(request, queryset, params):
if len(search_foods) > 0:
if search_foods_or:
# TODO creating setting to include descendants of food a setting
for fd in Food.objects.filter(pk__in=search_foods):
search_foods = [*search_foods, *list(fd.get_descendants_and_self().values_list('pk', flat=True))]
queryset = queryset.filter(steps__ingredients__food__id__in=search_foods)
else:
# when performing an 'and' search returned recipes should include a parent OR any of its descedants
@ -170,6 +174,7 @@ def search_recipes(request, queryset, params):
queryset = queryset.filter(recipebookentry__book__id=k)
if search_rating:
# TODO make ratings a settings user-only vs all-users
queryset = queryset.annotate(rating=Round(Avg(Case(When(cooklog__created_by=request.user, then='cooklog__rating'), default=Value(0)))))
if search_rating == -1:
queryset = queryset.filter(rating=0)
@ -196,189 +201,414 @@ def search_recipes(request, queryset, params):
return queryset
# TODO: This might be faster https://github.com/django-treebeard/django-treebeard/issues/115
def get_facet(qs=None, request=None, use_cache=True, hash_key=None):
"""
Gets an annotated list from a queryset.
:param qs:
recipe queryset to build facets from
:param request:
the web request that contains the necessary query parameters
:param use_cache:
will find results in cache, if any, and return them or empty list.
will save the list of recipes IDs in the cache for future processing
:param hash_key:
the cache key of the recipe list to process
only evaluated if the use_cache parameter is false
"""
facets = {}
recipe_list = []
cache_timeout = 600
class RecipeFacet():
class CacheEmpty(Exception):
pass
if use_cache:
qs_hash = hash(frozenset(qs.values_list('pk')))
facets['cache_key'] = str(qs_hash)
SEARCH_CACHE_KEY = f"recipes_filter_{qs_hash}"
if c := caches['default'].get(SEARCH_CACHE_KEY, None):
facets['Keywords'] = c['Keywords'] or []
facets['Foods'] = c['Foods'] or []
facets['Books'] = c['Books'] or []
facets['Ratings'] = c['Ratings'] or []
facets['Recent'] = c['Recent'] or []
else:
facets['Keywords'] = []
facets['Foods'] = []
facets['Books'] = []
rating_qs = qs.annotate(rating=Round(Avg(Case(When(cooklog__created_by=request.user, then='cooklog__rating'), default=Value(0)))))
facets['Ratings'] = dict(Counter(r.rating for r in rating_qs))
facets['Recent'] = ViewLog.objects.filter(
created_by=request.user, space=request.space,
created_at__gte=timezone.now() - timedelta(days=14) # TODO make days of recent recipe a setting
).values_list('recipe__pk', flat=True)
def __init__(self, request, queryset=None, hash_key=None, cache_timeout=3600):
if hash_key is None and queryset is None:
raise ValueError(_("One of queryset or hash_key must be provided"))
self._request = request
self._queryset = queryset
self.hash_key = hash_key or str(hash(frozenset(self._queryset.values_list('pk'))))
self._SEARCH_CACHE_KEY = f"recipes_filter_{self.hash_key}"
self._cache_timeout = cache_timeout
self._cache = caches['default'].get(self._SEARCH_CACHE_KEY, {})
if self._cache is None and self._queryset is None:
raise self.CacheEmpty("No queryset provided and cache empty")
self.Keywords = self._cache.get('Keywords', None)
self.Foods = self._cache.get('Foods', None)
self.Books = self._cache.get('Books', None)
self.Ratings = self._cache.get('Ratings', None)
self.Recent = self._cache.get('Recent', None)
if self._queryset is not None:
self._recipe_list = list(self._queryset.values_list('id', flat=True))
self._search_params = {
'keyword_list': self._request.query_params.getlist('keywords', []),
'food_list': self._request.query_params.getlist('foods', []),
'book_list': self._request.query_params.getlist('book', []),
'search_keywords_or': str2bool(self._request.query_params.get('keywords_or', True)),
'search_foods_or': str2bool(self._request.query_params.get('foods_or', True)),
'search_books_or': str2bool(self._request.query_params.get('books_or', True)),
'space': self._request.space,
}
elif self.hash_key is not None:
self._recipe_list = self._cache.get('recipe_list', None)
self._search_params = {
'keyword_list': self._cache.get('keyword_list', None),
'food_list': self._cache.get('food_list', None),
'book_list': self._cache.get('book_list', None),
'search_keywords_or': self._cache.get('search_keywords_or', None),
'search_foods_or': self._cache.get('search_foods_or', None),
'search_books_or': self._cache.get('search_books_or', None),
'space': self._cache.get('space', None),
}
self._cache = {
**self._search_params,
'recipe_list': self._recipe_list,
'Ratings': self.Ratings,
'Recent': self.Recent,
'Keywords': self.Keywords,
'Foods': self.Foods,
'Books': self.Books
cached_search = {
'recipe_list': list(qs.values_list('id', flat=True)),
'keyword_list': request.query_params.getlist('keywords', []),
'food_list': request.query_params.getlist('foods', []),
'book_list': request.query_params.getlist('book', []),
'search_keywords_or': str2bool(request.query_params.get('keywords_or', True)),
'search_foods_or': str2bool(request.query_params.get('foods_or', True)),
'search_books_or': str2bool(request.query_params.get('books_or', True)),
'space': request.space,
'Ratings': facets['Ratings'],
'Recent': facets['Recent'],
'Keywords': facets['Keywords'],
'Foods': facets['Foods'],
'Books': facets['Books']
}
caches['default'].set(SEARCH_CACHE_KEY, cached_search, cache_timeout)
return facets
caches['default'].set(self._SEARCH_CACHE_KEY, self._cache, self._cache_timeout)
SEARCH_CACHE_KEY = f'recipes_filter_{hash_key}'
if c := caches['default'].get(SEARCH_CACHE_KEY, None):
recipe_list = c['recipe_list']
keyword_list = c['keyword_list']
food_list = c['food_list']
book_list = c['book_list']
search_keywords_or = c['search_keywords_or']
search_foods_or = c['search_foods_or']
search_books_or = c['search_books_or']
else:
return {}
def get_facets(self):
if self._cache is None:
pass
return {
'cache_key': self.hash_key,
'Ratings': self.get_ratings(),
'Recent': self.get_recent(),
'Keywords': self.get_keywords(),
'Foods': self.get_foods(),
'Books': self.get_books()
}
# if using an OR search, will annotate all keywords, otherwise, just those that appear in results
if search_keywords_or:
keywords = Keyword.objects.filter(space=request.space).annotate(recipe_count=Count('recipe'))
else:
keywords = Keyword.objects.filter(recipe__in=recipe_list, space=request.space).annotate(recipe_count=Count('recipe'))
# custom django-tree function annotates a queryset to make building a tree easier.
# see https://django-treebeard.readthedocs.io/en/latest/api.html#treebeard.models.Node.get_annotated_list_qs for details
kw_a = annotated_qs(keywords, root=True, fill=True)
def set_cache(self, key, value):
self._cache = {**self._cache, key: value}
caches['default'].set(
self._SEARCH_CACHE_KEY,
self._cache,
self._cache_timeout
)
# # if using an OR search, will annotate all keywords, otherwise, just those that appear in results
if search_foods_or:
foods = Food.objects.filter(space=request.space).annotate(recipe_count=Count('ingredient'))
else:
foods = Food.objects.filter(ingredient__step__recipe__in=recipe_list, space=request.space).annotate(recipe_count=Count('ingredient'))
food_a = annotated_qs(foods, root=True, fill=True)
def get_books(self):
if self.Books is None:
self.Books = []
return self.Books
facets['Keywords'] = fill_annotated_parents(kw_a, keyword_list)
facets['Foods'] = fill_annotated_parents(food_a, food_list)
# TODO add book facet
facets['Books'] = []
c['Keywords'] = facets['Keywords']
c['Foods'] = facets['Foods']
c['Books'] = facets['Books']
caches['default'].set(SEARCH_CACHE_KEY, c, cache_timeout)
return facets
def get_keywords(self):
if self.Keywords is None:
if self._search_params['search_keywords_or']:
keywords = Keyword.objects.filter(space=self._request.space).distinct()
else:
keywords = Keyword.objects.filter(Q(recipe__in=self._recipe_list) | Q(depth=1)).filter(space=self._request.space).distinct()
# set keywords to root objects only
keywords = self._keyword_queryset(keywords)
self.Keywords = [{**x, 'children': None} if x['numchild'] > 0 else x for x in list(keywords)]
self.set_cache('Keywords', self.Keywords)
return self.Keywords
def get_foods(self):
if self.Foods is None:
# # if using an OR search, will annotate all keywords, otherwise, just those that appear in results
if self._search_params['search_foods_or']:
foods = Food.objects.filter(space=self._request.space).distinct()
else:
foods = Food.objects.filter(Q(ingredient__step__recipe__in=self._recipe_list) | Q(depth=1)).filter(space=self._request.space).distinct()
# set keywords to root objects only
foods = self._food_queryset(foods)
self.Foods = [{**x, 'children': None} if x['numchild'] > 0 else x for x in list(foods)]
self.set_cache('Foods', self.Foods)
return self.Foods
def get_books(self):
if self.Books is None:
self.Books = []
return self.Books
def get_ratings(self):
if self.Ratings is None:
if self._queryset is None:
self._queryset = Recipe.objects.filter(id__in=self._recipe_list)
rating_qs = self._queryset.annotate(rating=Round(Avg(Case(When(cooklog__created_by=self._request.user, then='cooklog__rating'), default=Value(0)))))
self.Ratings = dict(Counter(r.rating for r in rating_qs))
self.set_cache('Ratings', self.Ratings)
return self.Ratings
def get_recent(self):
if self.Recent is None:
# TODO make days of recent recipe a setting
recent_recipes = ViewLog.objects.filter(created_by=self._request.user, space=self._request.space, created_at__gte=timezone.now() - timedelta(days=14)
).values_list('recipe__pk', flat=True)
self.Recent = list(recent_recipes)
self.set_cache('Recent', self.Recent)
return self.Recent
def add_food_children(self, id):
try:
food = Food.objects.get(id=id)
nodes = food.get_ancestors()
except Food.DoesNotExist:
return self.get_facets()
foods = self._food_queryset(Food.objects.filter(path__startswith=food.path, depth=food.depth+1), food)
deep_search = self.Foods
for node in nodes:
index = next((i for i, x in enumerate(deep_search) if x["id"] == node.id), None)
deep_search = deep_search[index]['children']
index = next((i for i, x in enumerate(deep_search) if x["id"] == food.id), None)
deep_search[index]['children'] = [{**x, 'children': None} if x['numchild'] > 0 else x for x in list(foods)]
self.set_cache('Foods', self.Foods)
return self.get_facets()
def add_keyword_children(self, id):
try:
keyword = Keyword.objects.get(id=id)
nodes = keyword.get_ancestors()
except Keyword.DoesNotExist:
return self.get_facets()
keywords = self._keyword_queryset(Keyword.objects.filter(path__startswith=keyword.path, depth=keyword.depth+1), keyword)
deep_search = self.Keywords
for node in nodes:
index = next((i for i, x in enumerate(deep_search) if x["id"] == node.id), None)
deep_search = deep_search[index]['children']
index = next((i for i, x in enumerate(deep_search) if x["id"] == keyword.id), None)
deep_search[index]['children'] = [{**x, 'children': None} if x['numchild'] > 0 else x for x in list(keywords)]
self.set_cache('Keywords', self.Keywords)
return self.get_facets()
def _recipe_count_queryset(self, field, depth=1, steplen=4):
return Recipe.objects.filter(**{f'{field}__path__startswith': OuterRef('path')}, id__in=self._recipe_list, space=self._request.space
).values(child=Substr(f'{field}__path', 1, steplen)
).annotate(count=Count('pk', distinct=True)).values('count')
def _keyword_queryset(self, queryset, keyword=None):
depth = getattr(keyword, 'depth', 0) + 1
steplen = depth * Keyword.steplen
return queryset.annotate(count=Coalesce(Subquery(self._recipe_count_queryset('keywords', depth, steplen)), 0)
).filter(depth=depth, count__gt=0
).values('id', 'name', 'count', 'numchild').order_by('name')
def _food_queryset(self, queryset, food=None):
depth = getattr(food, 'depth', 0) + 1
steplen = depth * Food.steplen
return queryset.annotate(count=Coalesce(Subquery(self._recipe_count_queryset('steps__ingredients__food', depth, steplen)), 0)
).filter(depth__lte=depth, count__gt=0
).values('id', 'name', 'count', 'numchild').order_by('name')
def fill_annotated_parents(annotation, filters):
tree_list = []
parent = []
i = 0
level = -1
for r in annotation:
expand = False
# # TODO: This might be faster https://github.com/django-treebeard/django-treebeard/issues/115
# def get_facet(qs=None, request=None, use_cache=True, hash_key=None, food=None, keyword=None):
# """
# Gets an annotated list from a queryset.
# :param qs:
# recipe queryset to build facets from
# :param request:
# the web request that contains the necessary query parameters
# :param use_cache:
# will find results in cache, if any, and return them or empty list.
# will save the list of recipes IDs in the cache for future processing
# :param hash_key:
# the cache key of the recipe list to process
# only evaluated if the use_cache parameter is false
# :param food:
# return children facets of food
# only evaluated if the use_cache parameter is false
# :param keyword:
# return children facets of keyword
# only evaluated if the use_cache parameter is false
# """
# facets = {}
# recipe_list = []
# cache_timeout = 600
annotation[i][1]['id'] = r[0].id
annotation[i][1]['name'] = r[0].name
annotation[i][1]['count'] = getattr(r[0], 'recipe_count', 0)
annotation[i][1]['isDefaultExpanded'] = False
# # return cached values
# if use_cache:
# qs_hash = hash(frozenset(qs.values_list('pk')))
# facets['cache_key'] = str(qs_hash)
# SEARCH_CACHE_KEY = f"recipes_filter_{qs_hash}"
# if c := caches['default'].get(SEARCH_CACHE_KEY, None):
# facets['Keywords'] = c['Keywords'] or []
# facets['Foods'] = c['Foods'] or []
# facets['Books'] = c['Books'] or []
# facets['Ratings'] = c['Ratings'] or []
# facets['Recent'] = c['Recent'] or []
# else:
# facets['Keywords'] = []
# facets['Foods'] = []
# facets['Books'] = []
# # TODO make ratings a settings user-only vs all-users
# rating_qs = qs.annotate(rating=Round(Avg(Case(When(cooklog__created_by=request.user, then='cooklog__rating'), default=Value(0)))))
# facets['Ratings'] = dict(Counter(r.rating for r in rating_qs))
# facets['Recent'] = ViewLog.objects.filter(
# created_by=request.user, space=request.space,
# created_at__gte=timezone.now() - timedelta(days=14) # TODO make days of recent recipe a setting
# ).values_list('recipe__pk', flat=True)
if str(r[0].id) in filters:
expand = True
if r[1]['level'] < level:
parent = parent[:r[1]['level'] - level]
parent[-1] = i
level = r[1]['level']
elif r[1]['level'] > level:
parent.extend([i])
level = r[1]['level']
else:
parent[-1] = i
j = 0
# cached_search = {
# 'recipe_list': list(qs.values_list('id', flat=True)),
# 'keyword_list': request.query_params.getlist('keywords', []),
# 'food_list': request.query_params.getlist('foods', []),
# 'book_list': request.query_params.getlist('book', []),
# 'search_keywords_or': str2bool(request.query_params.get('keywords_or', True)),
# 'search_foods_or': str2bool(request.query_params.get('foods_or', True)),
# 'search_books_or': str2bool(request.query_params.get('books_or', True)),
# 'space': request.space,
# 'Ratings': facets['Ratings'],
# 'Recent': facets['Recent'],
# 'Keywords': facets['Keywords'],
# 'Foods': facets['Foods'],
# 'Books': facets['Books']
# }
# caches['default'].set(SEARCH_CACHE_KEY, cached_search, cache_timeout)
# return facets
while j < level:
# this causes some double counting when a recipe has both a child and an ancestor
annotation[parent[j]][1]['count'] += getattr(r[0], 'recipe_count', 0)
if expand:
annotation[parent[j]][1]['isDefaultExpanded'] = True
j += 1
if level == 0:
tree_list.append(annotation[i][1])
elif level > 0:
annotation[parent[level - 1]][1].setdefault('children', []).append(annotation[i][1])
i += 1
return tree_list
# # construct and cache new values by retrieving search parameters from the cache
# SEARCH_CACHE_KEY = f'recipes_filter_{hash_key}'
# if c := caches['default'].get(SEARCH_CACHE_KEY, None):
# recipe_list = c['recipe_list']
# keyword_list = c['keyword_list']
# food_list = c['food_list']
# book_list = c['book_list']
# search_keywords_or = c['search_keywords_or']
# search_foods_or = c['search_foods_or']
# search_books_or = c['search_books_or']
# else:
# return {}
# # if using an OR search, will annotate all keywords, otherwise, just those that appear in results
# if search_keywords_or:
# keywords = Keyword.objects.filter(space=request.space).distinct()
# else:
# keywords = Keyword.objects.filter(Q(recipe__in=recipe_list) | Q(depth=1)).filter(space=request.space).distinct()
# # Subquery that counts recipes for keyword including children
# kw_recipe_count = Recipe.objects.filter(**{'keywords__path__startswith': OuterRef('path')}, id__in=recipe_list, space=request.space
# ).values(kw=Substr('keywords__path', 1, Keyword.steplen)
# ).annotate(count=Count('pk', distinct=True)).values('count')
# # set keywords to root objects only
# keywords = keywords.annotate(count=Coalesce(Subquery(kw_recipe_count), 0)
# ).filter(depth=1, count__gt=0
# ).values('id', 'name', 'count', 'numchild'
# ).order_by('name')
# if keyword:
# facets['Keywords'] = list(keywords)
# return facets
# # custom django-tree function annotates a queryset to make building a tree easier.
# # see https://django-treebeard.readthedocs.io/en/latest/api.html#treebeard.models.Node.get_annotated_list_qs for details
# # kw_a = annotated_qs(keywords, root=True, fill=True)
# # # if using an OR search, will annotate all keywords, otherwise, just those that appear in results
# if search_foods_or:
# foods = Food.objects.filter(space=request.space).distinct()
# else:
# foods = Food.objects.filter(Q(ingredient__step__recipe__in=recipe_list) | Q(depth=1)).filter(space=request.space).distinct()
# food_recipe_count = Recipe.objects.filter(**{'steps__ingredients__food__path__startswith': OuterRef('path')}, id__in=recipe_list, space=request.space
# ).values(kw=Substr('steps__ingredients__food__path', 1, Food.steplen * (1+getattr(food, 'depth', 0)))
# ).annotate(count=Count('pk', distinct=True)).values('count')
# # set keywords to root objects only
# foods = foods.annotate(count=Coalesce(Subquery(food_recipe_count), 0)
# ).filter(depth=(1+getattr(food, 'depth', 0)), count__gt=0
# ).values('id', 'name', 'count', 'numchild'
# ).order_by('name')
# if food:
# facets['Foods'] = list(foods)
# return facets
# # food_a = annotated_qs(foods, root=True, fill=True)
# # c['Keywords'] = facets['Keywords'] = fill_annotated_parents(kw_a, keyword_list)
# c['Keywords'] = facets['Keywords'] = list(keywords)
# # c['Foods'] = facets['Foods'] = fill_annotated_parents(food_a, food_list)
# c['Foods'] = facets['Foods'] = list(foods)
# # TODO add book facet
# c['Books'] = facets['Books'] = []
# caches['default'].set(SEARCH_CACHE_KEY, c, cache_timeout)
# return facets
def annotated_qs(qs, root=False, fill=False):
"""
Gets an annotated list from a queryset.
:param root:
# def fill_annotated_parents(annotation, filters):
# tree_list = []
# parent = []
# i = 0
# level = -1
# for r in annotation:
# expand = False
Will backfill in annotation to include all parents to root node.
# annotation[i][1]['id'] = r[0].id
# annotation[i][1]['name'] = r[0].name
# annotation[i][1]['count'] = getattr(r[0], 'recipe_count', 0)
# annotation[i][1]['isDefaultExpanded'] = False
:param fill:
Will fill in gaps in annotation where nodes between children
and ancestors are not included in the queryset.
"""
# if str(r[0].id) in filters:
# expand = True
# if r[1]['level'] < level:
# parent = parent[:r[1]['level'] - level]
# parent[-1] = i
# level = r[1]['level']
# elif r[1]['level'] > level:
# parent.extend([i])
# level = r[1]['level']
# else:
# parent[-1] = i
# j = 0
result, info = [], {}
start_depth, prev_depth = (None, None)
nodes_list = list(qs.values_list('pk', flat=True))
for node in qs.order_by('path'):
node_queue = [node]
while len(node_queue) > 0:
dirty = False
current_node = node_queue[-1]
depth = current_node.get_depth()
parent_id = current_node.parent
if root and depth > 1 and parent_id not in nodes_list:
parent_id = current_node.parent
nodes_list.append(parent_id)
node_queue.append(current_node.__class__.objects.get(pk=parent_id))
dirty = True
# while j < level:
# # this causes some double counting when a recipe has both a child and an ancestor
# annotation[parent[j]][1]['count'] += getattr(r[0], 'recipe_count', 0)
# if expand:
# annotation[parent[j]][1]['isDefaultExpanded'] = True
# j += 1
# if level == 0:
# tree_list.append(annotation[i][1])
# elif level > 0:
# annotation[parent[level - 1]][1].setdefault('children', []).append(annotation[i][1])
# i += 1
# return tree_list
if fill and depth > 1 and prev_depth and depth > prev_depth and parent_id not in nodes_list:
nodes_list.append(parent_id)
node_queue.append(current_node.__class__.objects.get(pk=parent_id))
dirty = True
if not dirty:
working_node = node_queue.pop()
if start_depth is None:
start_depth = depth
open = (depth and (prev_depth is None or depth > prev_depth))
if prev_depth is not None and depth < prev_depth:
info['close'] = list(range(0, prev_depth - depth))
info = {'open': open, 'close': [], 'level': depth - start_depth}
result.append((working_node, info,))
prev_depth = depth
if start_depth and start_depth > 0:
info['close'] = list(range(0, prev_depth - start_depth + 1))
return result
# def annotated_qs(qs, root=False, fill=False):
# """
# Gets an annotated list from a queryset.
# :param root:
# Will backfill in annotation to include all parents to root node.
# :param fill:
# Will fill in gaps in annotation where nodes between children
# and ancestors are not included in the queryset.
# """
# result, info = [], {}
# start_depth, prev_depth = (None, None)
# nodes_list = list(qs.values_list('pk', flat=True))
# for node in qs.order_by('path'):
# node_queue = [node]
# while len(node_queue) > 0:
# dirty = False
# current_node = node_queue[-1]
# depth = current_node.get_depth()
# parent_id = current_node.parent
# if root and depth > 1 and parent_id not in nodes_list:
# parent_id = current_node.parent
# nodes_list.append(parent_id)
# node_queue.append(current_node.__class__.objects.get(pk=parent_id))
# dirty = True
# if fill and depth > 1 and prev_depth and depth > prev_depth and parent_id not in nodes_list:
# nodes_list.append(parent_id)
# node_queue.append(current_node.__class__.objects.get(pk=parent_id))
# dirty = True
# if not dirty:
# working_node = node_queue.pop()
# if start_depth is None:
# start_depth = depth
# open = (depth and (prev_depth is None or depth > prev_depth))
# if prev_depth is not None and depth < prev_depth:
# info['close'] = list(range(0, prev_depth - depth))
# info = {'open': open, 'close': [], 'level': depth - start_depth}
# result.append((working_node, info,))
# prev_depth = depth
# if start_depth and start_depth > 0:
# info['close'] = list(range(0, prev_depth - start_depth + 1))
# return result
def old_search(request):

44
cookbook/serializer.py
View File

@ -12,6 +12,7 @@ from rest_framework import serializers
from rest_framework.exceptions import NotFound, ValidationError
from rest_framework.fields import empty
from cookbook.helper.HelperFunctions import str2bool
from cookbook.helper.shopping_helper import list_from_recipe
from cookbook.models import (Automation, BookmarkletImport, Comment, CookLog, Food,
FoodInheritField, ImportLog, Ingredient, Keyword, MealPlan, MealType,
@ -21,13 +22,18 @@ from cookbook.models import (Automation, BookmarkletImport, Comment, CookLog, Fo
SupermarketCategoryRelation, Sync, SyncLog, Unit, UserFile,
UserPreference, ViewLog)
from cookbook.templatetags.custom_tags import markdown
from recipes.settings import MEDIA_URL, SCRIPT_NAME
class ExtendedRecipeMixin(serializers.ModelSerializer):
# adds image and recipe count to serializer when query param extended=1
image = serializers.SerializerMethodField('get_image')
numrecipe = serializers.SerializerMethodField('count_recipes')
# ORM path to this object from Recipe
recipe_filter = None
# list of ORM paths to any image
images = None
image = serializers.SerializerMethodField('get_image')
numrecipe = serializers.ReadOnlyField(source='count_recipes_test')
def get_fields(self, *args, **kwargs):
fields = super().get_fields(*args, **kwargs)
@ -37,8 +43,7 @@ class ExtendedRecipeMixin(serializers.ModelSerializer):
api_serializer = None
# extended values are computationally expensive and not needed in normal circumstances
try:
if bool(int(
self.context['request'].query_params.get('extended', False))) and self.__class__ == api_serializer:
if str2bool(self.context['request'].query_params.get('extended', False)) and self.__class__ == api_serializer:
return fields
except (AttributeError, KeyError) as e:
pass
@ -50,21 +55,8 @@ class ExtendedRecipeMixin(serializers.ModelSerializer):
return fields
def get_image(self, obj):
# TODO add caching
recipes = Recipe.objects.filter(**{self.recipe_filter: obj}, space=obj.space).exclude(
image__isnull=True).exclude(image__exact='')
try:
if recipes.count() == 0 and obj.has_children():
obj__in = self.recipe_filter + '__in'
recipes = Recipe.objects.filter(**{obj__in: obj.get_descendants()}, space=obj.space).exclude(
image__isnull=True).exclude(image__exact='') # if no recipes found - check whole tree
except AttributeError:
# probably not a tree
pass
if recipes.count() != 0:
return random.choice(recipes).image.url
else:
return None
if obj.recipe_image:
return SCRIPT_NAME + MEDIA_URL + obj.recipe_image
def count_recipes(self, obj):
return Recipe.objects.filter(**{self.recipe_filter: obj}, space=obj.space).count()
@ -98,7 +90,11 @@ class CustomOnHandField(serializers.Field):
return instance
def to_representation(self, obj):
shared_users = [x.id for x in list(self.context['request'].user.get_shopping_share())] + [self.context['request'].user.id]
shared_users = None
if request := self.context.get('request', None):
shared_users = getattr(request, '_shared_users', None)
if shared_users is None:
shared_users = [x.id for x in list(self.context['request'].user.get_shopping_share())] + [self.context['request'].user.id]
return obj.onhand_users.filter(id__in=shared_users).exists()
def to_internal_value(self, data):
@ -379,14 +375,16 @@ class RecipeSimpleSerializer(serializers.ModelSerializer):
class FoodSerializer(UniqueFieldsMixin, WritableNestedModelSerializer, ExtendedRecipeMixin):
supermarket_category = SupermarketCategorySerializer(allow_null=True, required=False)
recipe = RecipeSimpleSerializer(allow_null=True, required=False)
shopping = serializers.SerializerMethodField('get_shopping_status')
# shopping = serializers.SerializerMethodField('get_shopping_status')
shopping = serializers.ReadOnlyField(source='shopping_status')
inherit_fields = FoodInheritFieldSerializer(many=True, allow_null=True, required=False)
food_onhand = CustomOnHandField(required=False, allow_null=True)
recipe_filter = 'steps__ingredients__food'
images = ['recipe__image']
def get_shopping_status(self, obj):
return ShoppingListEntry.objects.filter(space=obj.space, food=obj, checked=False).count() > 0
# def get_shopping_status(self, obj):
# return ShoppingListEntry.objects.filter(space=obj.space, food=obj, checked=False).count() > 0
def create(self, validated_data):
validated_data['name'] = validated_data['name'].strip()

86
cookbook/views/api.py
View File

@ -12,8 +12,9 @@ from django.contrib.auth.models import User
from django.contrib.postgres.search import TrigramSimilarity
from django.core.exceptions import FieldError, ValidationError
from django.core.files import File
from django.db.models import Case, ProtectedError, Q, Value, When
from django.db.models import Case, Count, Exists, OuterRef, ProtectedError, Q, Subquery, Value, When
from django.db.models.fields.related import ForeignObjectRel
from django.db.models.functions import Coalesce
from django.http import FileResponse, HttpResponse, JsonResponse
from django.shortcuts import get_object_or_404, redirect
from django.urls import reverse
@ -30,13 +31,14 @@ from rest_framework.response import Response
from rest_framework.viewsets import ViewSetMixin
from treebeard.exceptions import InvalidMoveToDescendant, InvalidPosition, PathOverflow
from cookbook.helper.HelperFunctions import str2bool
from cookbook.helper.image_processing import handle_image
from cookbook.helper.ingredient_parser import IngredientParser
from cookbook.helper.permission_helper import (CustomIsAdmin, CustomIsGuest, CustomIsOwner,
CustomIsShare, CustomIsShared, CustomIsUser,
group_required)
from cookbook.helper.recipe_html_import import get_recipe_from_source
from cookbook.helper.recipe_search import get_facet, old_search, search_recipes
from cookbook.helper.recipe_search import RecipeFacet, old_search, search_recipes
from cookbook.helper.recipe_url_import import get_from_scraper
from cookbook.helper.shopping_helper import list_from_recipe, shopping_helper
from cookbook.models import (Automation, BookmarkletImport, CookLog, Food, FoodInheritField,
@ -100,7 +102,38 @@ class DefaultPagination(PageNumberPagination):
max_page_size = 200
class FuzzyFilterMixin(ViewSetMixin):
class ExtendedRecipeMixin():
'''
ExtendedRecipe annotates a queryset with recipe_image and recipe_count values
'''
@classmethod
def annotate_recipe(self, queryset=None, request=None, serializer=None, tree=False):
extended = str2bool(request.query_params.get('extended', None))
if extended:
recipe_filter = serializer.recipe_filter
images = serializer.images
space = request.space
# add a recipe count annotation to the query
# explanation on construction https://stackoverflow.com/a/43771738/15762829
recipe_count = Recipe.objects.filter(**{recipe_filter: OuterRef('id')}, space=space).values(recipe_filter).annotate(count=Count('pk')).values('count')
queryset = queryset.annotate(recipe_count_test=Coalesce(Subquery(recipe_count), 0))
# add a recipe image annotation to the query
image_subquery = Recipe.objects.filter(**{recipe_filter: OuterRef('id')}, space=space).exclude(image__isnull=True).exclude(image__exact='').order_by("?").values('image')[:1]
if tree:
image_children_subquery = Recipe.objects.filter(**{f"{recipe_filter}__path__startswith": OuterRef('path')},
space=space).exclude(image__isnull=True).exclude(image__exact='').order_by("?").values('image')[:1]
else:
image_children_subquery = None
if images:
queryset = queryset.annotate(recipe_image=Coalesce(*images, image_subquery, image_children_subquery))
else:
queryset = queryset.annotate(recipe_image=Coalesce(image_subquery, image_children_subquery))
return queryset
class FuzzyFilterMixin(ViewSetMixin, ExtendedRecipeMixin):
schema = FilterSchema()
def get_queryset(self):
@ -141,12 +174,12 @@ class FuzzyFilterMixin(ViewSetMixin):
if random:
self.queryset = self.queryset.order_by("?")
self.queryset = self.queryset[:int(limit)]
return self.queryset
return self.annotate_recipe(queryset=self.queryset, request=self.request, serializer=self.serializer_class)
class MergeMixin(ViewSetMixin):
@decorators.action(detail=True, url_path='merge/(?P<target>[^/.]+)', methods=['PUT'], )
@decorators.renderer_classes((TemplateHTMLRenderer, JSONRenderer))
@ decorators.action(detail=True, url_path='merge/(?P<target>[^/.]+)', methods=['PUT'], )
@ decorators.renderer_classes((TemplateHTMLRenderer, JSONRenderer))
def merge(self, request, pk, target):
self.description = f"Merge {self.basename} onto target {self.basename} with ID of [int]."
@ -211,7 +244,7 @@ class MergeMixin(ViewSetMixin):
return Response(content, status=status.HTTP_400_BAD_REQUEST)
class TreeMixin(MergeMixin, FuzzyFilterMixin):
class TreeMixin(MergeMixin, FuzzyFilterMixin, ExtendedRecipeMixin):
schema = TreeSchema()
model = None
@ -237,11 +270,13 @@ class TreeMixin(MergeMixin, FuzzyFilterMixin):
except self.model.DoesNotExist:
self.queryset = self.model.objects.none()
else:
return super().get_queryset()
return self.queryset.filter(space=self.request.space).order_by('name')
return self.annotate_recipe(queryset=super().get_queryset(), request=self.request, serializer=self.serializer_class, tree=True)
self.queryset = self.queryset.filter(space=self.request.space).order_by('name')
@decorators.action(detail=True, url_path='move/(?P<parent>[^/.]+)', methods=['PUT'], )
@decorators.renderer_classes((TemplateHTMLRenderer, JSONRenderer))
return self.annotate_recipe(queryset=self.queryset, request=self.request, serializer=self.serializer_class, tree=True)
@ decorators.action(detail=True, url_path='move/(?P<parent>[^/.]+)', methods=['PUT'], )
@ decorators.renderer_classes((TemplateHTMLRenderer, JSONRenderer))
def move(self, request, pk, parent):
self.description = f"Move {self.basename} to be a child of {self.basename} with ID of [int]. Use ID: 0 to move {self.basename} to the root."
if self.model.node_order_by:
@ -413,7 +448,15 @@ class FoodViewSet(viewsets.ModelViewSet, TreeMixin):
permission_classes = [CustomIsUser]
pagination_class = DefaultPagination
@decorators.action(detail=True, methods=['PUT'], serializer_class=FoodShoppingUpdateSerializer,)
def get_queryset(self):
self.request._shared_users = [x.id for x in list(self.request.user.get_shopping_share())] + [self.request.user.id]
self.queryset = super().get_queryset()
shopping_status = ShoppingListEntry.objects.filter(space=self.request.space, food=OuterRef('id'), checked=False).values('id')
# onhand_status = self.queryset.annotate(onhand_status=Exists(onhand_users_set__in=[shared_users]))
return self.queryset.annotate(shopping_status=Exists(shopping_status)).prefetch_related('onhand_users', 'inherit_fields').select_related('recipe', 'supermarket_category')
@ decorators.action(detail=True, methods=['PUT'], serializer_class=FoodShoppingUpdateSerializer,)
# TODO DRF only allows one action in a decorator action without overriding get_operation_id_base() this should be PUT and DELETE probably
def shopping(self, request, pk):
if self.request.space.demo:
@ -561,7 +604,7 @@ class RecipePagination(PageNumberPagination):
max_page_size = 100
def paginate_queryset(self, queryset, request, view=None):
self.facets = get_facet(qs=queryset, request=request)
self.facets = RecipeFacet(request, queryset=queryset)
return super().paginate_queryset(queryset, request, view)
def get_paginated_response(self, data):
@ -570,7 +613,7 @@ class RecipePagination(PageNumberPagination):
('next', self.get_next_link()),
('previous', self.get_previous_link()),
('results', data),
('facets', self.facets)
('facets', self.facets.get_facets())
]))
@ -608,8 +651,7 @@ class RecipeViewSet(viewsets.ModelViewSet):
self.queryset = self.queryset.filter(space=self.request.space)
self.queryset = search_recipes(self.request, self.queryset, self.request.GET)
return super().get_queryset()
return super().get_queryset().prefetch_related('cooklog_set')
def list(self, request, *args, **kwargs):
if self.request.GET.get('debug', False):
@ -1089,10 +1131,20 @@ def ingredient_from_string(request):
@group_required('user')
def get_facets(request):
key = request.GET.get('hash', None)
food = request.GET.get('food', None)
keyword = request.GET.get('keyword', None)
facets = RecipeFacet(request, hash_key=key)
if food:
results = facets.add_food_children(food)
elif keyword:
results = facets.add_keyword_children(keyword)
else:
results = facets.get_facets()
return JsonResponse(
{
'facets': get_facet(request=request, use_cache=False, hash_key=key),
'facets': results,
},
status=200
)

8
recipes/middleware.py
View File

@ -1,3 +1,4 @@
import time
from os import getenv
from django.conf import settings
@ -13,19 +14,20 @@ class CustomRemoteUser(RemoteUserMiddleware):
Gist code by vstoykov, you can check his original gist at:
https://gist.github.com/vstoykov/1390853/5d2e8fac3ca2b2ada8c7de2fb70c021e50927375
Changes:
Ignoring static file requests and a certain useless admin request from triggering the logger.
Ignoring static file requests and a certain useless admin request from triggering the logger.
Updated statements to make it Python 3 friendly.
"""
def terminal_width():
"""
Function to compute the terminal width.
"""
width = 0
try:
import struct, fcntl, termios
import fcntl
import struct
import termios
s = struct.pack('HHHH', 0, 0, 0, 0)
x = fcntl.ioctl(1, termios.TIOCGWINSZ, s)
width = struct.unpack('HHHH', x)[1]

4
recipes/settings.py
View File

@ -371,10 +371,10 @@ LANGUAGES = [
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/2.0/howto/static-files/
SCRIPT_NAME = os.getenv('SCRIPT_NAME', '')
# path for django_js_reverse to generate the javascript file containing all urls. Only done because the default command (collectstatic_js_reverse) fails to update the manifest
JS_REVERSE_OUTPUT_PATH = os.path.join(BASE_DIR, "cookbook/static/django_js_reverse")
JS_REVERSE_SCRIPT_PREFIX = os.getenv('JS_REVERSE_SCRIPT_PREFIX', os.getenv('SCRIPT_NAME', ''))
JS_REVERSE_SCRIPT_PREFIX = os.getenv('JS_REVERSE_SCRIPT_PREFIX', SCRIPT_NAME)
STATIC_URL = os.getenv('STATIC_URL', '/static/')
STATIC_ROOT = os.path.join(BASE_DIR, "staticfiles")

54
vue/src/apps/RecipeSearchView/RecipeSearchView.vue
View File

@ -80,7 +80,7 @@
</div>
<div class="row" style="margin-top: 1vh">
<div class="col-12">
<a :href="resolveDjangoUrl('view_settings') + '#search'">{{ $t("Advanced Search Settings") }}</a>
<a :href="resolveDjangoUrl('view_settings') + '#search'">{{ $t("Search Settings") }}</a>
</div>
</div>
<div class="row" style="margin-top: 1vh">
@ -97,9 +97,10 @@
<treeselect
v-model="settings.search_keywords"
:options="facets.Keywords"
:load-options="loadKeywordChildren"
:flat="true"
searchNested
multiple
:multiple="true"
:placeholder="$t('Keywords')"
:normalizer="normalizer"
@input="refreshData(false)"
@ -124,9 +125,10 @@
<treeselect
v-model="settings.search_foods"
:options="facets.Foods"
:load-options="loadFoodChildren"
:flat="true"
searchNested
multiple
:multiple="true"
:placeholder="$t('Ingredients')"
:normalizer="normalizer"
@input="refreshData(false)"
@ -243,7 +245,7 @@ import LoadingSpinner from "@/components/LoadingSpinner" // TODO: is this deprec
import RecipeCard from "@/components/RecipeCard"
import GenericMultiselect from "@/components/GenericMultiselect"
import Treeselect from "@riophae/vue-treeselect"
import { Treeselect, LOAD_CHILDREN_OPTIONS } from "@riophae/vue-treeselect"
import "@riophae/vue-treeselect/dist/vue-treeselect.css"
import RecipeSwitcher from "@/components/Buttons/RecipeSwitcher"
@ -403,9 +405,9 @@ export default {
this.pagination_count = result.data.count
this.facets = result.data.facets
if (this.facets?.cache_key) {
this.getFacets(this.facets.cache_key)
}
// if (this.facets?.cache_key) {
// this.getFacets(this.facets.cache_key)
// }
this.recipes = this.removeDuplicates(result.data.results, (recipe) => recipe.id)
if (!this.searchFiltered) {
// if meal plans are being shown - filter out any meal plan recipes from the recipe list
@ -479,8 +481,12 @@ export default {
return [undefined, undefined]
}
},
getFacets: function (hash) {
this.genericGetAPI("api_get_facets", { hash: hash }).then((response) => {
getFacets: function (hash, facet, id) {
let params = { hash: hash }
if (facet) {
params[facet] = id
}
return this.genericGetAPI("api_get_facets", params).then((response) => {
this.facets = { ...this.facets, ...response.data.facets }
})
},
@ -508,9 +514,33 @@ export default {
} else {
params.options = { query: { debug: true } }
}
this.genericAPI(this.Models.RECIPE, this.Actions.LIST, params).then((result) => {
console.log(result.data)
})
this.genericAPI(this.Models.RECIPE, this.Actions.LIST, params).then((result) => {})
},
loadFoodChildren({ action, parentNode, callback }) {
// Typically, do the AJAX stuff here.
// Once the server has responded,
// assign children options to the parent node & call the callback.
if (action === LOAD_CHILDREN_OPTIONS) {
if (this.facets?.cache_key) {
this.getFacets(this.facets.cache_key, "food", parentNode.id).then(callback())
}
} else {
callback()
}
},
loadKeywordChildren({ action, parentNode, callback }) {
// Typically, do the AJAX stuff here.
// Once the server has responded,
// assign children options to the parent node & call the callback.
if (action === LOAD_CHILDREN_OPTIONS) {
if (this.facets?.cache_key) {
this.getFacets(this.facets.cache_key, "keyword", parentNode.id).then(callback())
}
} else {
callback()
}
},
},
}

2
vue/src/locales/en.json
View File

@ -116,7 +116,7 @@
"Information": "Information",
"Download": "Download",
"Create": "Create",
"Advanced Search Settings": "Advanced Search Settings",
"Search Settings": "Search Settings",
"View": "View",
"Recipes": "Recipes",
"Move": "Move",