Merge pull request #1300 from TandoorRecipes/performance_refactor

Performance refactor

cookbook/helper/HelperFunctions.py

@@ -7,7 +7,7 @@ class Round(Func):
 
 
 def str2bool(v):
-    if type(v) == bool:
+    if type(v) == bool or v is None:
         return v
     else:
         return v.lower() in ("yes", "true", "1")

cookbook/helper/recipe_search.py

@@ -3,8 +3,8 @@ from datetime import timedelta
 
 from django.contrib.postgres.search import SearchQuery, SearchRank, TrigramSimilarity
 from django.core.cache import caches
-from django.db.models import Avg, Case, Count, Func, Max, Q, Subquery, Value, When
+from django.db.models import Avg, Case, Count, Func, Max, OuterRef, Q, Subquery, Value, When
-from django.db.models.functions import Coalesce
+from django.db.models.functions import Coalesce, Substr
 from django.utils import timezone, translation
 
 from cookbook.filters import RecipeFilter
@@ -145,6 +145,8 @@ def search_recipes(request, queryset, params):
             # TODO creating setting to include descendants of keywords a setting
             # for kw in Keyword.objects.filter(pk__in=search_keywords):
             #     search_keywords += list(kw.get_descendants().values_list('pk', flat=True))
+            for kw in Keyword.objects.filter(pk__in=search_keywords):
+                search_keywords = [*search_keywords, *list(kw.get_descendants_and_self().values_list('pk', flat=True))]
             queryset = queryset.filter(keywords__id__in=search_keywords)
         else:
             # when performing an 'and' search returned recipes should include a parent OR any of its descedants
@@ -155,6 +157,8 @@ def search_recipes(request, queryset, params):
     if len(search_foods) > 0:
         if search_foods_or:
             # TODO creating setting to include descendants of food a setting
+            for fd in Food.objects.filter(pk__in=search_foods):
+                search_foods = [*search_foods, *list(fd.get_descendants_and_self().values_list('pk', flat=True))]
             queryset = queryset.filter(steps__ingredients__food__id__in=search_foods)
         else:
             # when performing an 'and' search returned recipes should include a parent OR any of its descedants
@@ -170,6 +174,7 @@ def search_recipes(request, queryset, params):
                 queryset = queryset.filter(recipebookentry__book__id=k)
 
     if search_rating:
+        # TODO make ratings a settings user-only vs all-users
         queryset = queryset.annotate(rating=Round(Avg(Case(When(cooklog__created_by=request.user, then='cooklog__rating'), default=Value(0)))))
         if search_rating == -1:
             queryset = queryset.filter(rating=0)
@@ -196,189 +201,414 @@ def search_recipes(request, queryset, params):
     return queryset
 
 
-# TODO:  This might be faster https://github.com/django-treebeard/django-treebeard/issues/115
+class RecipeFacet():
-def get_facet(qs=None, request=None, use_cache=True, hash_key=None):
+    class CacheEmpty(Exception):
-    """
+        pass
-    Gets an annotated list from a queryset.
-    :param qs:
-        recipe queryset to build facets from
-    :param request:
-        the web request that contains the necessary query parameters
-    :param use_cache:
-        will find results in cache, if any, and return them or empty list.
-        will save the list of recipes IDs in the cache for future processing
-    :param hash_key:
-        the cache key of the recipe list to process
-        only evaluated if the use_cache parameter is false
-    """
-    facets = {}
-    recipe_list = []
-    cache_timeout = 600
 
-    if use_cache:
+    def __init__(self, request, queryset=None, hash_key=None, cache_timeout=3600):
-        qs_hash = hash(frozenset(qs.values_list('pk')))
+        if hash_key is None and queryset is None:
-        facets['cache_key'] = str(qs_hash)
+            raise ValueError(_("One of queryset or hash_key must be provided"))
-        SEARCH_CACHE_KEY = f"recipes_filter_{qs_hash}"
-        if c := caches['default'].get(SEARCH_CACHE_KEY, None):
-            facets['Keywords'] = c['Keywords'] or []
-            facets['Foods'] = c['Foods'] or []
-            facets['Books'] = c['Books'] or []
-            facets['Ratings'] = c['Ratings'] or []
-            facets['Recent'] = c['Recent'] or []
-        else:
-            facets['Keywords'] = []
-            facets['Foods'] = []
-            facets['Books'] = []
-            rating_qs = qs.annotate(rating=Round(Avg(Case(When(cooklog__created_by=request.user, then='cooklog__rating'), default=Value(0)))))
-            facets['Ratings'] = dict(Counter(r.rating for r in rating_qs))
-            facets['Recent'] = ViewLog.objects.filter(
-                created_by=request.user, space=request.space,
-                created_at__gte=timezone.now() - timedelta(days=14)  # TODO make days of recent recipe a setting
-            ).values_list('recipe__pk', flat=True)
 
-        cached_search = {
+        self._request = request
-            'recipe_list': list(qs.values_list('id', flat=True)),
+        self._queryset = queryset
-            'keyword_list': request.query_params.getlist('keywords', []),
+        self.hash_key = hash_key or str(hash(frozenset(self._queryset.values_list('pk'))))
-            'food_list': request.query_params.getlist('foods', []),
+        self._SEARCH_CACHE_KEY = f"recipes_filter_{self.hash_key}"
-            'book_list': request.query_params.getlist('book', []),
+        self._cache_timeout = cache_timeout
-            'search_keywords_or': str2bool(request.query_params.get('keywords_or', True)),
+        self._cache = caches['default'].get(self._SEARCH_CACHE_KEY, {})
-            'search_foods_or': str2bool(request.query_params.get('foods_or', True)),
+        if self._cache is None and self._queryset is None:
-            'search_books_or': str2bool(request.query_params.get('books_or', True)),
+            raise self.CacheEmpty("No queryset provided and cache empty")
-            'space': request.space,
+
-            'Ratings': facets['Ratings'],
+        self.Keywords = self._cache.get('Keywords', None)
-            'Recent': facets['Recent'],
+        self.Foods = self._cache.get('Foods', None)
-            'Keywords': facets['Keywords'],
+        self.Books = self._cache.get('Books', None)
-            'Foods': facets['Foods'],
+        self.Ratings = self._cache.get('Ratings', None)
-            'Books': facets['Books']
+        self.Recent = self._cache.get('Recent', None)
+
+        if self._queryset is not None:
+            self._recipe_list = list(self._queryset.values_list('id', flat=True))
+            self._search_params = {
+                'keyword_list': self._request.query_params.getlist('keywords', []),
+                'food_list': self._request.query_params.getlist('foods', []),
+                'book_list': self._request.query_params.getlist('book', []),
+                'search_keywords_or': str2bool(self._request.query_params.get('keywords_or', True)),
+                'search_foods_or': str2bool(self._request.query_params.get('foods_or', True)),
+                'search_books_or': str2bool(self._request.query_params.get('books_or', True)),
+                'space': self._request.space,
+            }
+        elif self.hash_key is not None:
+            self._recipe_list = self._cache.get('recipe_list', None)
+            self._search_params = {
+                'keyword_list': self._cache.get('keyword_list', None),
+                'food_list': self._cache.get('food_list', None),
+                'book_list': self._cache.get('book_list', None),
+                'search_keywords_or': self._cache.get('search_keywords_or', None),
+                'search_foods_or': self._cache.get('search_foods_or', None),
+                'search_books_or': self._cache.get('search_books_or', None),
+                'space': self._cache.get('space', None),
             }
-        caches['default'].set(SEARCH_CACHE_KEY, cached_search, cache_timeout)
-        return facets
 
-    SEARCH_CACHE_KEY = f'recipes_filter_{hash_key}'
+        self._cache = {
-    if c := caches['default'].get(SEARCH_CACHE_KEY, None):
+            **self._search_params,
-        recipe_list = c['recipe_list']
+            'recipe_list': self._recipe_list,
-        keyword_list = c['keyword_list']
+            'Ratings': self.Ratings,
-        food_list = c['food_list']
+            'Recent': self.Recent,
-        book_list = c['book_list']
+            'Keywords': self.Keywords,
-        search_keywords_or = c['search_keywords_or']
+            'Foods': self.Foods,
-        search_foods_or = c['search_foods_or']
+            'Books': self.Books
-        search_books_or = c['search_books_or']
-    else:
-        return {}
 
-    # if using an OR search, will annotate all keywords, otherwise, just those that appear in results
+        }
-    if search_keywords_or:
+        caches['default'].set(self._SEARCH_CACHE_KEY, self._cache, self._cache_timeout)
-        keywords = Keyword.objects.filter(space=request.space).annotate(recipe_count=Count('recipe'))
+
+    def get_facets(self):
+        if self._cache is None:
+            pass
+        return {
+            'cache_key': self.hash_key,
+            'Ratings': self.get_ratings(),
+            'Recent': self.get_recent(),
+            'Keywords': self.get_keywords(),
+            'Foods': self.get_foods(),
+            'Books': self.get_books()
+        }
+
+    def set_cache(self, key, value):
+        self._cache = {**self._cache, key: value}
+        caches['default'].set(
+            self._SEARCH_CACHE_KEY,
+            self._cache,
+            self._cache_timeout
+        )
+
+    def get_books(self):
+        if self.Books is None:
+            self.Books = []
+        return self.Books
+
+    def get_keywords(self):
+        if self.Keywords is None:
+            if self._search_params['search_keywords_or']:
+                keywords = Keyword.objects.filter(space=self._request.space).distinct()
             else:
-        keywords = Keyword.objects.filter(recipe__in=recipe_list, space=request.space).annotate(recipe_count=Count('recipe'))
+                keywords = Keyword.objects.filter(Q(recipe__in=self._recipe_list) | Q(depth=1)).filter(space=self._request.space).distinct()
-    # custom django-tree function annotates a queryset to make building a tree easier.
+
-    # see https://django-treebeard.readthedocs.io/en/latest/api.html#treebeard.models.Node.get_annotated_list_qs for details
+            # set keywords to root objects only
-    kw_a = annotated_qs(keywords, root=True, fill=True)
+            keywords = self._keyword_queryset(keywords)
+            self.Keywords = [{**x, 'children': None} if x['numchild'] > 0 else x for x in list(keywords)]
+            self.set_cache('Keywords', self.Keywords)
+        return self.Keywords
+
+    def get_foods(self):
+        if self.Foods is None:
+            # # if using an OR search, will annotate all keywords, otherwise, just those that appear in results
+            if self._search_params['search_foods_or']:
+                foods = Food.objects.filter(space=self._request.space).distinct()
+            else:
+                foods = Food.objects.filter(Q(ingredient__step__recipe__in=self._recipe_list) | Q(depth=1)).filter(space=self._request.space).distinct()
+
+            # set keywords to root objects only
+            foods = self._food_queryset(foods)
+
+            self.Foods = [{**x, 'children': None} if x['numchild'] > 0 else x for x in list(foods)]
+            self.set_cache('Foods', self.Foods)
+        return self.Foods
+
+    def get_books(self):
+        if self.Books is None:
+            self.Books = []
+        return self.Books
+
+    def get_ratings(self):
+        if self.Ratings is None:
+            if self._queryset is None:
+                self._queryset = Recipe.objects.filter(id__in=self._recipe_list)
+            rating_qs = self._queryset.annotate(rating=Round(Avg(Case(When(cooklog__created_by=self._request.user, then='cooklog__rating'), default=Value(0)))))
+            self.Ratings = dict(Counter(r.rating for r in rating_qs))
+            self.set_cache('Ratings', self.Ratings)
+        return self.Ratings
+
+    def get_recent(self):
+        if self.Recent is None:
+            # TODO make days of recent recipe a setting
+            recent_recipes = ViewLog.objects.filter(created_by=self._request.user, space=self._request.space, created_at__gte=timezone.now() - timedelta(days=14)
+                                                    ).values_list('recipe__pk', flat=True)
+            self.Recent = list(recent_recipes)
+            self.set_cache('Recent', self.Recent)
+        return self.Recent
+
+    def add_food_children(self, id):
+        try:
+            food = Food.objects.get(id=id)
+            nodes = food.get_ancestors()
+        except Food.DoesNotExist:
+            return self.get_facets()
+        foods = self._food_queryset(Food.objects.filter(path__startswith=food.path, depth=food.depth+1), food)
+        deep_search = self.Foods
+        for node in nodes:
+            index = next((i for i, x in enumerate(deep_search) if x["id"] == node.id), None)
+            deep_search = deep_search[index]['children']
+        index = next((i for i, x in enumerate(deep_search) if x["id"] == food.id), None)
+        deep_search[index]['children'] = [{**x, 'children': None} if x['numchild'] > 0 else x for x in list(foods)]
+        self.set_cache('Foods', self.Foods)
+        return self.get_facets()
+
+    def add_keyword_children(self, id):
+        try:
+            keyword = Keyword.objects.get(id=id)
+            nodes = keyword.get_ancestors()
+        except Keyword.DoesNotExist:
+            return self.get_facets()
+        keywords = self._keyword_queryset(Keyword.objects.filter(path__startswith=keyword.path, depth=keyword.depth+1), keyword)
+        deep_search = self.Keywords
+        for node in nodes:
+            index = next((i for i, x in enumerate(deep_search) if x["id"] == node.id), None)
+            deep_search = deep_search[index]['children']
+        index = next((i for i, x in enumerate(deep_search) if x["id"] == keyword.id), None)
+        deep_search[index]['children'] = [{**x, 'children': None} if x['numchild'] > 0 else x for x in list(keywords)]
+        self.set_cache('Keywords', self.Keywords)
+        return self.get_facets()
+
+    def _recipe_count_queryset(self, field, depth=1, steplen=4):
+        return Recipe.objects.filter(**{f'{field}__path__startswith': OuterRef('path')}, id__in=self._recipe_list, space=self._request.space
+                                     ).values(child=Substr(f'{field}__path',  1, steplen)
+                                              ).annotate(count=Count('pk', distinct=True)).values('count')
+
+    def _keyword_queryset(self, queryset, keyword=None):
+        depth = getattr(keyword, 'depth', 0) + 1
+        steplen = depth * Keyword.steplen
+
+        return queryset.annotate(count=Coalesce(Subquery(self._recipe_count_queryset('keywords', depth, steplen)), 0)
+                                 ).filter(depth=depth, count__gt=0
+                                          ).values('id', 'name', 'count', 'numchild').order_by('name')
+
+    def _food_queryset(self, queryset, food=None):
+        depth = getattr(food, 'depth', 0) + 1
+        steplen = depth * Food.steplen
+
+        return queryset.annotate(count=Coalesce(Subquery(self._recipe_count_queryset('steps__ingredients__food', depth, steplen)), 0)
+                                 ).filter(depth__lte=depth, count__gt=0
+                                          ).values('id', 'name', 'count', 'numchild').order_by('name')
+
+
+# # TODO:  This might be faster https://github.com/django-treebeard/django-treebeard/issues/115
+# def get_facet(qs=None, request=None, use_cache=True, hash_key=None, food=None, keyword=None):
+#     """
+#     Gets an annotated list from a queryset.
+#     :param qs:
+#         recipe queryset to build facets from
+#     :param request:
+#         the web request that contains the necessary query parameters
+#     :param use_cache:
+#         will find results in cache, if any, and return them or empty list.
+#         will save the list of recipes IDs in the cache for future processing
+#     :param hash_key:
+#         the cache key of the recipe list to process
+#         only evaluated if the use_cache parameter is false
+#     :param food:
+#         return children facets of food
+#         only evaluated if the use_cache parameter is false
+#     :param keyword:
+#         return children facets of keyword
+#         only evaluated if the use_cache parameter is false
+#     """
+#     facets = {}
+#     recipe_list = []
+#     cache_timeout = 600
+
+#     # return cached values
+#     if use_cache:
+#         qs_hash = hash(frozenset(qs.values_list('pk')))
+#         facets['cache_key'] = str(qs_hash)
+#         SEARCH_CACHE_KEY = f"recipes_filter_{qs_hash}"
+#         if c := caches['default'].get(SEARCH_CACHE_KEY, None):
+#             facets['Keywords'] = c['Keywords'] or []
+#             facets['Foods'] = c['Foods'] or []
+#             facets['Books'] = c['Books'] or []
+#             facets['Ratings'] = c['Ratings'] or []
+#             facets['Recent'] = c['Recent'] or []
+#         else:
+#             facets['Keywords'] = []
+#             facets['Foods'] = []
+#             facets['Books'] = []
+#             # TODO make ratings a settings user-only vs all-users
+#             rating_qs = qs.annotate(rating=Round(Avg(Case(When(cooklog__created_by=request.user, then='cooklog__rating'), default=Value(0)))))
+#             facets['Ratings'] = dict(Counter(r.rating for r in rating_qs))
+#             facets['Recent'] = ViewLog.objects.filter(
+#                 created_by=request.user, space=request.space,
+#                 created_at__gte=timezone.now() - timedelta(days=14)  # TODO make days of recent recipe a setting
+#             ).values_list('recipe__pk', flat=True)
+
+#         cached_search = {
+#             'recipe_list': list(qs.values_list('id', flat=True)),
+#             'keyword_list': request.query_params.getlist('keywords', []),
+#             'food_list': request.query_params.getlist('foods', []),
+#             'book_list': request.query_params.getlist('book', []),
+#             'search_keywords_or': str2bool(request.query_params.get('keywords_or', True)),
+#             'search_foods_or': str2bool(request.query_params.get('foods_or', True)),
+#             'search_books_or': str2bool(request.query_params.get('books_or', True)),
+#             'space': request.space,
+#             'Ratings': facets['Ratings'],
+#             'Recent': facets['Recent'],
+#             'Keywords': facets['Keywords'],
+#             'Foods': facets['Foods'],
+#             'Books': facets['Books']
+#         }
+#         caches['default'].set(SEARCH_CACHE_KEY, cached_search, cache_timeout)
+#         return facets
+
+#     # construct and cache new values by retrieving search parameters from the cache
+#     SEARCH_CACHE_KEY = f'recipes_filter_{hash_key}'
+#     if c := caches['default'].get(SEARCH_CACHE_KEY, None):
+#         recipe_list = c['recipe_list']
+#         keyword_list = c['keyword_list']
+#         food_list = c['food_list']
+#         book_list = c['book_list']
+#         search_keywords_or = c['search_keywords_or']
+#         search_foods_or = c['search_foods_or']
+#         search_books_or = c['search_books_or']
+#     else:
+#         return {}
 
 #     # if using an OR search, will annotate all keywords, otherwise, just those that appear in results
-    if search_foods_or:
+#     if search_keywords_or:
-        foods = Food.objects.filter(space=request.space).annotate(recipe_count=Count('ingredient'))
+#         keywords = Keyword.objects.filter(space=request.space).distinct()
-    else:
+#     else:
-        foods = Food.objects.filter(ingredient__step__recipe__in=recipe_list, space=request.space).annotate(recipe_count=Count('ingredient'))
+#         keywords = Keyword.objects.filter(Q(recipe__in=recipe_list) | Q(depth=1)).filter(space=request.space).distinct()
-    food_a = annotated_qs(foods, root=True, fill=True)
 
-    facets['Keywords'] = fill_annotated_parents(kw_a, keyword_list)
+#     # Subquery that counts recipes for keyword including children
-    facets['Foods'] = fill_annotated_parents(food_a, food_list)
+#     kw_recipe_count = Recipe.objects.filter(**{'keywords__path__startswith': OuterRef('path')}, id__in=recipe_list, space=request.space
-    # TODO add book facet
+#                                             ).values(kw=Substr('keywords__path',  1, Keyword.steplen)
-    facets['Books'] = []
+#                                                      ).annotate(count=Count('pk', distinct=True)).values('count')
-    c['Keywords'] = facets['Keywords']
+
-    c['Foods'] = facets['Foods']
+#     # set keywords to root objects only
-    c['Books'] = facets['Books']
+#     keywords = keywords.annotate(count=Coalesce(Subquery(kw_recipe_count), 0)
-    caches['default'].set(SEARCH_CACHE_KEY, c, cache_timeout)
+#                                  ).filter(depth=1, count__gt=0
-    return facets
+#                                           ).values('id', 'name', 'count', 'numchild'
+#                                                    ).order_by('name')
+#     if keyword:
+#         facets['Keywords'] = list(keywords)
+#         return facets
+
+#     # custom django-tree function annotates a queryset to make building a tree easier.
+#     # see https://django-treebeard.readthedocs.io/en/latest/api.html#treebeard.models.Node.get_annotated_list_qs for details
+#     # kw_a = annotated_qs(keywords, root=True, fill=True)
+
+#     # # if using an OR search, will annotate all keywords, otherwise, just those that appear in results
+#     if search_foods_or:
+#         foods = Food.objects.filter(space=request.space).distinct()
+#     else:
+#         foods = Food.objects.filter(Q(ingredient__step__recipe__in=recipe_list) | Q(depth=1)).filter(space=request.space).distinct()
+
+#     food_recipe_count = Recipe.objects.filter(**{'steps__ingredients__food__path__startswith': OuterRef('path')}, id__in=recipe_list, space=request.space
+#                                               ).values(kw=Substr('steps__ingredients__food__path',  1, Food.steplen * (1+getattr(food, 'depth', 0)))
+#                                                        ).annotate(count=Count('pk', distinct=True)).values('count')
+
+#     # set keywords to root objects only
+#     foods = foods.annotate(count=Coalesce(Subquery(food_recipe_count), 0)
+#                            ).filter(depth=(1+getattr(food, 'depth', 0)), count__gt=0
+#                                     ).values('id', 'name', 'count', 'numchild'
+#                                              ).order_by('name')
+#     if food:
+#         facets['Foods'] = list(foods)
+#         return facets
+
+#     # food_a = annotated_qs(foods, root=True, fill=True)
+
+#     # c['Keywords'] = facets['Keywords'] = fill_annotated_parents(kw_a, keyword_list)
+#     c['Keywords'] = facets['Keywords'] = list(keywords)
+#     # c['Foods'] = facets['Foods'] = fill_annotated_parents(food_a, food_list)
+#     c['Foods'] = facets['Foods'] = list(foods)
+#     # TODO add book facet
+#     c['Books'] = facets['Books'] = []
+#     caches['default'].set(SEARCH_CACHE_KEY, c, cache_timeout)
+#     return facets
 
 
-def fill_annotated_parents(annotation, filters):
+# def fill_annotated_parents(annotation, filters):
-    tree_list = []
+#     tree_list = []
-    parent = []
+#     parent = []
-    i = 0
+#     i = 0
-    level = -1
+#     level = -1
-    for r in annotation:
+#     for r in annotation:
-        expand = False
+#         expand = False
 
-        annotation[i][1]['id'] = r[0].id
+#         annotation[i][1]['id'] = r[0].id
-        annotation[i][1]['name'] = r[0].name
+#         annotation[i][1]['name'] = r[0].name
-        annotation[i][1]['count'] = getattr(r[0], 'recipe_count', 0)
+#         annotation[i][1]['count'] = getattr(r[0], 'recipe_count', 0)
-        annotation[i][1]['isDefaultExpanded'] = False
+#         annotation[i][1]['isDefaultExpanded'] = False
 
-        if str(r[0].id) in filters:
+#         if str(r[0].id) in filters:
-            expand = True
+#             expand = True
-        if r[1]['level'] < level:
+#         if r[1]['level'] < level:
-            parent = parent[:r[1]['level'] - level]
+#             parent = parent[:r[1]['level'] - level]
-            parent[-1] = i
+#             parent[-1] = i
-            level = r[1]['level']
+#             level = r[1]['level']
-        elif r[1]['level'] > level:
+#         elif r[1]['level'] > level:
-            parent.extend([i])
+#             parent.extend([i])
-            level = r[1]['level']
+#             level = r[1]['level']
-        else:
+#         else:
-            parent[-1] = i
+#             parent[-1] = i
-        j = 0
+#         j = 0
 
-        while j < level:
+#         while j < level:
-            # this causes some double counting when a recipe has both a child and an ancestor
+#             # this causes some double counting when a recipe has both a child and an ancestor
-            annotation[parent[j]][1]['count'] += getattr(r[0], 'recipe_count', 0)
+#             annotation[parent[j]][1]['count'] += getattr(r[0], 'recipe_count', 0)
-            if expand:
+#             if expand:
-                annotation[parent[j]][1]['isDefaultExpanded'] = True
+#                 annotation[parent[j]][1]['isDefaultExpanded'] = True
-            j += 1
+#             j += 1
-        if level == 0:
+#         if level == 0:
-            tree_list.append(annotation[i][1])
+#             tree_list.append(annotation[i][1])
-        elif level > 0:
+#         elif level > 0:
-            annotation[parent[level - 1]][1].setdefault('children', []).append(annotation[i][1])
+#             annotation[parent[level - 1]][1].setdefault('children', []).append(annotation[i][1])
-        i += 1
+#         i += 1
-    return tree_list
+#     return tree_list
 
 
-def annotated_qs(qs, root=False, fill=False):
+# def annotated_qs(qs, root=False, fill=False):
-    """
+#     """
-    Gets an annotated list from a queryset.
+#     Gets an annotated list from a queryset.
-    :param root:
+#     :param root:
 
-        Will backfill in annotation to include all parents to root node.
+#         Will backfill in annotation to include all parents to root node.
 
-    :param fill:
+#     :param fill:
-        Will fill in gaps in annotation where nodes between children
+#         Will fill in gaps in annotation where nodes between children
-        and ancestors are not included in the queryset.
+#         and ancestors are not included in the queryset.
-    """
+#     """
 
-    result, info = [], {}
+#     result, info = [], {}
-    start_depth, prev_depth = (None, None)
+#     start_depth, prev_depth = (None, None)
-    nodes_list = list(qs.values_list('pk', flat=True))
+#     nodes_list = list(qs.values_list('pk', flat=True))
-    for node in qs.order_by('path'):
+#     for node in qs.order_by('path'):
-        node_queue = [node]
+#         node_queue = [node]
-        while len(node_queue) > 0:
+#         while len(node_queue) > 0:
-            dirty = False
+#             dirty = False
-            current_node = node_queue[-1]
+#             current_node = node_queue[-1]
-            depth = current_node.get_depth()
+#             depth = current_node.get_depth()
-            parent_id = current_node.parent
+#             parent_id = current_node.parent
-            if root and depth > 1 and parent_id not in nodes_list:
+#             if root and depth > 1 and parent_id not in nodes_list:
-                parent_id = current_node.parent
+#                 parent_id = current_node.parent
-                nodes_list.append(parent_id)
+#                 nodes_list.append(parent_id)
-                node_queue.append(current_node.__class__.objects.get(pk=parent_id))
+#                 node_queue.append(current_node.__class__.objects.get(pk=parent_id))
-                dirty = True
+#                 dirty = True
 
-            if fill and depth > 1 and prev_depth and depth > prev_depth and parent_id not in nodes_list:
+#             if fill and depth > 1 and prev_depth and depth > prev_depth and parent_id not in nodes_list:
-                nodes_list.append(parent_id)
+#                 nodes_list.append(parent_id)
-                node_queue.append(current_node.__class__.objects.get(pk=parent_id))
+#                 node_queue.append(current_node.__class__.objects.get(pk=parent_id))
-                dirty = True
+#                 dirty = True
 
-            if not dirty:
+#             if not dirty:
-                working_node = node_queue.pop()
+#                 working_node = node_queue.pop()
-                if start_depth is None:
+#                 if start_depth is None:
-                    start_depth = depth
+#                     start_depth = depth
-                open = (depth and (prev_depth is None or depth > prev_depth))
+#                 open = (depth and (prev_depth is None or depth > prev_depth))
-                if prev_depth is not None and depth < prev_depth:
+#                 if prev_depth is not None and depth < prev_depth:
-                    info['close'] = list(range(0, prev_depth - depth))
+#                     info['close'] = list(range(0, prev_depth - depth))
-                info = {'open': open, 'close': [], 'level': depth - start_depth}
+#                 info = {'open': open, 'close': [], 'level': depth - start_depth}
-                result.append((working_node, info,))
+#                 result.append((working_node, info,))
-                prev_depth = depth
+#                 prev_depth = depth
-    if start_depth and start_depth > 0:
+#     if start_depth and start_depth > 0:
-        info['close'] = list(range(0, prev_depth - start_depth + 1))
+#         info['close'] = list(range(0, prev_depth - start_depth + 1))
-    return result
+#     return result
 
 
 def old_search(request):

cookbook/serializer.py

@@ -12,6 +12,7 @@ from rest_framework import serializers
 from rest_framework.exceptions import NotFound, ValidationError
 from rest_framework.fields import empty
 
+from cookbook.helper.HelperFunctions import str2bool
 from cookbook.helper.shopping_helper import list_from_recipe
 from cookbook.models import (Automation, BookmarkletImport, Comment, CookLog, Food,
                              FoodInheritField, ImportLog, Ingredient, Keyword, MealPlan, MealType,
@@ -21,13 +22,18 @@ from cookbook.models import (Automation, BookmarkletImport, Comment, CookLog, Fo
                              SupermarketCategoryRelation, Sync, SyncLog, Unit, UserFile,
                              UserPreference, ViewLog)
 from cookbook.templatetags.custom_tags import markdown
+from recipes.settings import MEDIA_URL, SCRIPT_NAME
 
 
 class ExtendedRecipeMixin(serializers.ModelSerializer):
     # adds image and recipe count to serializer when query param extended=1
-    image = serializers.SerializerMethodField('get_image')
+    # ORM path to this object from Recipe
-    numrecipe = serializers.SerializerMethodField('count_recipes')
     recipe_filter = None
+    # list of ORM paths to any image
+    images = None
+
+    image = serializers.SerializerMethodField('get_image')
+    numrecipe = serializers.ReadOnlyField(source='count_recipes_test')
 
     def get_fields(self, *args, **kwargs):
         fields = super().get_fields(*args, **kwargs)
@@ -37,8 +43,7 @@ class ExtendedRecipeMixin(serializers.ModelSerializer):
             api_serializer = None
         # extended values are computationally expensive and not needed in normal circumstances
         try:
-            if bool(int(
+            if str2bool(self.context['request'].query_params.get('extended', False)) and self.__class__ == api_serializer:
-                    self.context['request'].query_params.get('extended', False))) and self.__class__ == api_serializer:
                 return fields
         except (AttributeError, KeyError) as e:
             pass
@@ -50,21 +55,8 @@ class ExtendedRecipeMixin(serializers.ModelSerializer):
         return fields
 
     def get_image(self, obj):
-        # TODO add caching
+        if obj.recipe_image:
-        recipes = Recipe.objects.filter(**{self.recipe_filter: obj}, space=obj.space).exclude(
+            return SCRIPT_NAME + MEDIA_URL + obj.recipe_image
-            image__isnull=True).exclude(image__exact='')
-        try:
-            if recipes.count() == 0 and obj.has_children():
-                obj__in = self.recipe_filter + '__in'
-                recipes = Recipe.objects.filter(**{obj__in: obj.get_descendants()}, space=obj.space).exclude(
-                    image__isnull=True).exclude(image__exact='')  # if no recipes found - check whole tree
-        except AttributeError:
-            # probably not a tree
-            pass
-        if recipes.count() != 0:
-            return random.choice(recipes).image.url
-        else:
-            return None
 
     def count_recipes(self, obj):
         return Recipe.objects.filter(**{self.recipe_filter: obj}, space=obj.space).count()
@@ -98,6 +90,10 @@ class CustomOnHandField(serializers.Field):
         return instance
 
     def to_representation(self, obj):
+        shared_users = None
+        if request := self.context.get('request', None):
+            shared_users = getattr(request, '_shared_users', None)
+        if shared_users is None:
             shared_users = [x.id for x in list(self.context['request'].user.get_shopping_share())] + [self.context['request'].user.id]
         return obj.onhand_users.filter(id__in=shared_users).exists()
 
@@ -379,14 +375,16 @@ class RecipeSimpleSerializer(serializers.ModelSerializer):
 class FoodSerializer(UniqueFieldsMixin, WritableNestedModelSerializer, ExtendedRecipeMixin):
     supermarket_category = SupermarketCategorySerializer(allow_null=True, required=False)
     recipe = RecipeSimpleSerializer(allow_null=True, required=False)
-    shopping = serializers.SerializerMethodField('get_shopping_status')
+    # shopping = serializers.SerializerMethodField('get_shopping_status')
+    shopping = serializers.ReadOnlyField(source='shopping_status')
     inherit_fields = FoodInheritFieldSerializer(many=True, allow_null=True, required=False)
     food_onhand = CustomOnHandField(required=False, allow_null=True)
 
     recipe_filter = 'steps__ingredients__food'
+    images = ['recipe__image']
 
-    def get_shopping_status(self, obj):
+    # def get_shopping_status(self, obj):
-        return ShoppingListEntry.objects.filter(space=obj.space, food=obj, checked=False).count() > 0
+    #     return ShoppingListEntry.objects.filter(space=obj.space, food=obj, checked=False).count() > 0
 
     def create(self, validated_data):
         validated_data['name'] = validated_data['name'].strip()

cookbook/views/api.py

@@ -12,8 +12,9 @@ from django.contrib.auth.models import User
 from django.contrib.postgres.search import TrigramSimilarity
 from django.core.exceptions import FieldError, ValidationError
 from django.core.files import File
-from django.db.models import Case, ProtectedError, Q, Value, When
+from django.db.models import Case, Count, Exists, OuterRef, ProtectedError, Q, Subquery, Value, When
 from django.db.models.fields.related import ForeignObjectRel
+from django.db.models.functions import Coalesce
 from django.http import FileResponse, HttpResponse, JsonResponse
 from django.shortcuts import get_object_or_404, redirect
 from django.urls import reverse
@@ -30,13 +31,14 @@ from rest_framework.response import Response
 from rest_framework.viewsets import ViewSetMixin
 from treebeard.exceptions import InvalidMoveToDescendant, InvalidPosition, PathOverflow
 
+from cookbook.helper.HelperFunctions import str2bool
 from cookbook.helper.image_processing import handle_image
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.helper.permission_helper import (CustomIsAdmin, CustomIsGuest, CustomIsOwner,
                                                CustomIsShare, CustomIsShared, CustomIsUser,
                                                group_required)
 from cookbook.helper.recipe_html_import import get_recipe_from_source
-from cookbook.helper.recipe_search import get_facet, old_search, search_recipes
+from cookbook.helper.recipe_search import RecipeFacet, old_search, search_recipes
 from cookbook.helper.recipe_url_import import get_from_scraper
 from cookbook.helper.shopping_helper import list_from_recipe, shopping_helper
 from cookbook.models import (Automation, BookmarkletImport, CookLog, Food, FoodInheritField,
@@ -100,7 +102,38 @@ class DefaultPagination(PageNumberPagination):
     max_page_size = 200
 
 
-class FuzzyFilterMixin(ViewSetMixin):
+class ExtendedRecipeMixin():
+    '''
+    ExtendedRecipe annotates a queryset with recipe_image and recipe_count values
+    '''
+    @classmethod
+    def annotate_recipe(self, queryset=None, request=None, serializer=None, tree=False):
+        extended = str2bool(request.query_params.get('extended', None))
+        if extended:
+            recipe_filter = serializer.recipe_filter
+            images = serializer.images
+            space = request.space
+
+            # add a recipe count annotation to the query
+            #  explanation on construction https://stackoverflow.com/a/43771738/15762829
+            recipe_count = Recipe.objects.filter(**{recipe_filter: OuterRef('id')}, space=space).values(recipe_filter).annotate(count=Count('pk')).values('count')
+            queryset = queryset.annotate(recipe_count_test=Coalesce(Subquery(recipe_count), 0))
+
+            # add a recipe image annotation to the query
+            image_subquery = Recipe.objects.filter(**{recipe_filter: OuterRef('id')}, space=space).exclude(image__isnull=True).exclude(image__exact='').order_by("?").values('image')[:1]
+            if tree:
+                image_children_subquery = Recipe.objects.filter(**{f"{recipe_filter}__path__startswith": OuterRef('path')},
+                                                                space=space).exclude(image__isnull=True).exclude(image__exact='').order_by("?").values('image')[:1]
+            else:
+                image_children_subquery = None
+            if images:
+                queryset = queryset.annotate(recipe_image=Coalesce(*images, image_subquery, image_children_subquery))
+            else:
+                queryset = queryset.annotate(recipe_image=Coalesce(image_subquery, image_children_subquery))
+        return queryset
+
+
+class FuzzyFilterMixin(ViewSetMixin, ExtendedRecipeMixin):
     schema = FilterSchema()
 
     def get_queryset(self):
@@ -141,7 +174,7 @@ class FuzzyFilterMixin(ViewSetMixin):
             if random:
                 self.queryset = self.queryset.order_by("?")
             self.queryset = self.queryset[:int(limit)]
-        return self.queryset
+        return self.annotate_recipe(queryset=self.queryset, request=self.request, serializer=self.serializer_class)
 
 
 class MergeMixin(ViewSetMixin):
@@ -211,7 +244,7 @@ class MergeMixin(ViewSetMixin):
                 return Response(content, status=status.HTTP_400_BAD_REQUEST)
 
 
-class TreeMixin(MergeMixin, FuzzyFilterMixin):
+class TreeMixin(MergeMixin, FuzzyFilterMixin, ExtendedRecipeMixin):
     schema = TreeSchema()
     model = None
 
@@ -237,8 +270,10 @@ class TreeMixin(MergeMixin, FuzzyFilterMixin):
                 except self.model.DoesNotExist:
                     self.queryset = self.model.objects.none()
         else:
-            return super().get_queryset()
+            return self.annotate_recipe(queryset=super().get_queryset(), request=self.request, serializer=self.serializer_class, tree=True)
-        return self.queryset.filter(space=self.request.space).order_by('name')
+        self.queryset = self.queryset.filter(space=self.request.space).order_by('name')
+
+        return self.annotate_recipe(queryset=self.queryset, request=self.request, serializer=self.serializer_class, tree=True)
 
     @ decorators.action(detail=True, url_path='move/(?P<parent>[^/.]+)', methods=['PUT'], )
     @ decorators.renderer_classes((TemplateHTMLRenderer, JSONRenderer))
@@ -413,6 +448,14 @@ class FoodViewSet(viewsets.ModelViewSet, TreeMixin):
     permission_classes = [CustomIsUser]
     pagination_class = DefaultPagination
 
+    def get_queryset(self):
+        self.request._shared_users = [x.id for x in list(self.request.user.get_shopping_share())] + [self.request.user.id]
+
+        self.queryset = super().get_queryset()
+        shopping_status = ShoppingListEntry.objects.filter(space=self.request.space, food=OuterRef('id'), checked=False).values('id')
+        # onhand_status = self.queryset.annotate(onhand_status=Exists(onhand_users_set__in=[shared_users]))
+        return self.queryset.annotate(shopping_status=Exists(shopping_status)).prefetch_related('onhand_users', 'inherit_fields').select_related('recipe', 'supermarket_category')
+
     @ decorators.action(detail=True,  methods=['PUT'], serializer_class=FoodShoppingUpdateSerializer,)
     # TODO DRF only allows one action in a decorator action without overriding get_operation_id_base() this should be PUT and DELETE probably
     def shopping(self, request, pk):
@@ -561,7 +604,7 @@ class RecipePagination(PageNumberPagination):
     max_page_size = 100
 
     def paginate_queryset(self, queryset, request, view=None):
-        self.facets = get_facet(qs=queryset, request=request)
+        self.facets = RecipeFacet(request, queryset=queryset)
         return super().paginate_queryset(queryset, request, view)
 
     def get_paginated_response(self, data):
@@ -570,7 +613,7 @@ class RecipePagination(PageNumberPagination):
             ('next', self.get_next_link()),
             ('previous', self.get_previous_link()),
             ('results', data),
-            ('facets', self.facets)
+            ('facets', self.facets.get_facets())
         ]))
 
 
@@ -608,8 +651,7 @@ class RecipeViewSet(viewsets.ModelViewSet):
             self.queryset = self.queryset.filter(space=self.request.space)
 
         self.queryset = search_recipes(self.request, self.queryset, self.request.GET)
-
+        return super().get_queryset().prefetch_related('cooklog_set')
-        return super().get_queryset()
 
     def list(self, request, *args, **kwargs):
         if self.request.GET.get('debug', False):
@@ -1089,10 +1131,20 @@ def ingredient_from_string(request):
 @group_required('user')
 def get_facets(request):
     key = request.GET.get('hash', None)
+    food = request.GET.get('food', None)
+    keyword = request.GET.get('keyword', None)
+    facets = RecipeFacet(request, hash_key=key)
+
+    if food:
+        results = facets.add_food_children(food)
+    elif keyword:
+        results = facets.add_keyword_children(keyword)
+    else:
+        results = facets.get_facets()
 
     return JsonResponse(
         {
-            'facets': get_facet(request=request, use_cache=False, hash_key=key),
+            'facets': results,
         },
         status=200
     )

recipes/middleware.py

@@ -1,3 +1,4 @@
+import time
 from os import getenv
 
 from django.conf import settings
@@ -18,14 +19,15 @@ Updated statements to make it Python 3 friendly.
 """
 
 
-
 def terminal_width():
     """
     Function to compute the terminal width.
     """
     width = 0
     try:
-        import struct, fcntl, termios
+        import fcntl
+        import struct
+        import termios
         s = struct.pack('HHHH', 0, 0, 0, 0)
         x = fcntl.ioctl(1, termios.TIOCGWINSZ, s)
         width = struct.unpack('HHHH', x)[1]

recipes/settings.py

@@ -371,10 +371,10 @@ LANGUAGES = [
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/2.0/howto/static-files/
 
+SCRIPT_NAME = os.getenv('SCRIPT_NAME', '')
 # path for django_js_reverse to generate the javascript file containing all urls. Only done because the default command (collectstatic_js_reverse) fails to update the manifest
 JS_REVERSE_OUTPUT_PATH = os.path.join(BASE_DIR, "cookbook/static/django_js_reverse")
-
+JS_REVERSE_SCRIPT_PREFIX = os.getenv('JS_REVERSE_SCRIPT_PREFIX', SCRIPT_NAME)
-JS_REVERSE_SCRIPT_PREFIX = os.getenv('JS_REVERSE_SCRIPT_PREFIX', os.getenv('SCRIPT_NAME', ''))
 
 STATIC_URL = os.getenv('STATIC_URL', '/static/')
 STATIC_ROOT = os.path.join(BASE_DIR, "staticfiles")

vue/src/apps/RecipeSearchView/RecipeSearchView.vue

@@ -80,7 +80,7 @@
                                         </div>
                                         <div class="row" style="margin-top: 1vh">
                                             <div class="col-12">
-                                                <a :href="resolveDjangoUrl('view_settings') + '#search'">{{ $t("Advanced Search Settings") }}</a>
+                                                <a :href="resolveDjangoUrl('view_settings') + '#search'">{{ $t("Search Settings") }}</a>
                                             </div>
                                         </div>
                                         <div class="row" style="margin-top: 1vh">
@@ -97,9 +97,10 @@
                                                 <treeselect
                                                     v-model="settings.search_keywords"
                                                     :options="facets.Keywords"
+                                                    :load-options="loadKeywordChildren"
                                                     :flat="true"
                                                     searchNested
-                                                    multiple
+                                                    :multiple="true"
                                                     :placeholder="$t('Keywords')"
                                                     :normalizer="normalizer"
                                                     @input="refreshData(false)"
@@ -124,9 +125,10 @@
                                                 <treeselect
                                                     v-model="settings.search_foods"
                                                     :options="facets.Foods"
+                                                    :load-options="loadFoodChildren"
                                                     :flat="true"
                                                     searchNested
-                                                    multiple
+                                                    :multiple="true"
                                                     :placeholder="$t('Ingredients')"
                                                     :normalizer="normalizer"
                                                     @input="refreshData(false)"
@@ -243,7 +245,7 @@ import LoadingSpinner from "@/components/LoadingSpinner" // TODO: is this deprec
 
 import RecipeCard from "@/components/RecipeCard"
 import GenericMultiselect from "@/components/GenericMultiselect"
-import Treeselect from "@riophae/vue-treeselect"
+import { Treeselect, LOAD_CHILDREN_OPTIONS } from "@riophae/vue-treeselect"
 import "@riophae/vue-treeselect/dist/vue-treeselect.css"
 import RecipeSwitcher from "@/components/Buttons/RecipeSwitcher"
 
@@ -403,9 +405,9 @@ export default {
                 this.pagination_count = result.data.count
 
                 this.facets = result.data.facets
-                if (this.facets?.cache_key) {
+                // if (this.facets?.cache_key) {
-                    this.getFacets(this.facets.cache_key)
+                //     this.getFacets(this.facets.cache_key)
-                }
+                // }
                 this.recipes = this.removeDuplicates(result.data.results, (recipe) => recipe.id)
                 if (!this.searchFiltered) {
                     // if meal plans are being shown - filter out any meal plan recipes from the recipe list
@@ -479,8 +481,12 @@ export default {
                 return [undefined, undefined]
             }
         },
-        getFacets: function (hash) {
+        getFacets: function (hash, facet, id) {
-            this.genericGetAPI("api_get_facets", { hash: hash }).then((response) => {
+            let params = { hash: hash }
+            if (facet) {
+                params[facet] = id
+            }
+            return this.genericGetAPI("api_get_facets", params).then((response) => {
                 this.facets = { ...this.facets, ...response.data.facets }
             })
         },
@@ -508,9 +514,33 @@ export default {
             } else {
                 params.options = { query: { debug: true } }
             }
-            this.genericAPI(this.Models.RECIPE, this.Actions.LIST, params).then((result) => {
+            this.genericAPI(this.Models.RECIPE, this.Actions.LIST, params).then((result) => {})
-                console.log(result.data)
+        },
-            })
+        loadFoodChildren({ action, parentNode, callback }) {
+            // Typically, do the AJAX stuff here.
+            // Once the server has responded,
+            // assign children options to the parent node & call the callback.
+
+            if (action === LOAD_CHILDREN_OPTIONS) {
+                if (this.facets?.cache_key) {
+                    this.getFacets(this.facets.cache_key, "food", parentNode.id).then(callback())
+                }
+            } else {
+                callback()
+            }
+        },
+        loadKeywordChildren({ action, parentNode, callback }) {
+            // Typically, do the AJAX stuff here.
+            // Once the server has responded,
+            // assign children options to the parent node & call the callback.
+
+            if (action === LOAD_CHILDREN_OPTIONS) {
+                if (this.facets?.cache_key) {
+                    this.getFacets(this.facets.cache_key, "keyword", parentNode.id).then(callback())
+                }
+            } else {
+                callback()
+            }
         },
     },
 }

vue/src/locales/en.json

@@ -116,7 +116,7 @@
     "Information": "Information",
     "Download": "Download",
     "Create": "Create",
-    "Advanced Search Settings": "Advanced Search Settings",
+    "Search Settings": "Search Settings",
     "View": "View",
     "Recipes": "Recipes",
     "Move": "Move",