cached facet results

This commit is contained in:
smilerz 2021-09-26 16:44:12 -05:00
parent b7be5cd325
commit b3cffa4a38
23 changed files with 336 additions and 171 deletions

View File

@ -1,6 +1,7 @@
# only set this to true when testing/debugging
# when unset: 1 (true) - dont unset this, just for development
DEBUG=0
SQL_DEBUG=0
# hosts the application can run under e.g. recipes.mydomain.com,cooking.mydomain.com,...
ALLOWED_HOSTS=*

View File

@ -5,6 +5,7 @@ from recipes import settings
from django.contrib.postgres.search import (
SearchQuery, SearchRank, TrigramSimilarity
)
from django.core.cache import caches
from django.db.models import Avg, Case, Count, Func, Max, Q, Subquery, Value, When
from django.utils import timezone, translation
@ -17,6 +18,13 @@ class Round(Func):
template = '%(function)s(%(expressions)s, 0)'
def str2bool(v):
if type(v) == bool:
return v
else:
return v.lower() in ("yes", "true", "1")
# TODO create extensive tests to make sure ORs ANDs and various filters, sorting, etc work as expected
# TODO consider creating a simpleListRecipe API that only includes minimum of recipe info and minimal filtering
def search_recipes(request, queryset, params):
@ -32,13 +40,13 @@ def search_recipes(request, queryset, params):
search_units = params.get('units', None)
# TODO I think default behavior should be 'AND' which is how most sites operate with facet/filters based on results
search_keywords_or = params.get('keywords_or', True)
search_foods_or = params.get('foods_or', True)
search_books_or = params.get('books_or', True)
search_keywords_or = str2bool(params.get('keywords_or', True))
search_foods_or = str2bool(params.get('foods_or', True))
search_books_or = str2bool(params.get('books_or', True))
search_internal = params.get('internal', None)
search_random = params.get('random', False)
search_new = params.get('new', False)
search_internal = str2bool(params.get('internal', None))
search_random = str2bool(params.get('random', False))
search_new = str2bool(params.get('new', False))
search_last_viewed = int(params.get('last_viewed', 0))
orderby = []
@ -58,7 +66,7 @@ def search_recipes(request, queryset, params):
# TODO create setting for default ordering - most cooked, rating,
# TODO create options for live sorting
# TODO make days of new recipe a setting
if search_new == 'true':
if search_new:
queryset = (
queryset.annotate(new_recipe=Case(
When(created_at__gte=(timezone.now() - timedelta(days=7)), then=('pk')), default=Value(0), ))
@ -144,7 +152,7 @@ def search_recipes(request, queryset, params):
queryset = queryset.filter(name__icontains=search_string)
if len(search_keywords) > 0:
if search_keywords_or == 'true':
if search_keywords_or:
# TODO creating setting to include descendants of keywords a setting
# for kw in Keyword.objects.filter(pk__in=search_keywords):
# search_keywords += list(kw.get_descendants().values_list('pk', flat=True))
@ -156,7 +164,7 @@ def search_recipes(request, queryset, params):
queryset = queryset.filter(keywords__id__in=list(kw.get_descendants_and_self().values_list('pk', flat=True)))
if len(search_foods) > 0:
if search_foods_or == 'true':
if search_foods_or:
# TODO creating setting to include descendants of food a setting
queryset = queryset.filter(steps__ingredients__food__id__in=search_foods)
else:
@ -166,7 +174,7 @@ def search_recipes(request, queryset, params):
queryset = queryset.filter(steps__ingredients__food__id__in=list(fd.get_descendants_and_self().values_list('pk', flat=True)))
if len(search_books) > 0:
if search_books_or == 'true':
if search_books_or:
queryset = queryset.filter(recipebookentry__book__id__in=search_books)
else:
for k in search_books:
@ -183,58 +191,119 @@ def search_recipes(request, queryset, params):
if search_units:
queryset = queryset.filter(steps__ingredients__unit__id=search_units)
if search_internal == 'true':
if search_internal:
queryset = queryset.filter(internal=True)
queryset = queryset.distinct()
if search_random == 'true':
if search_random:
queryset = queryset.order_by("?")
else:
# TODO add order by user settings
# orderby += ['name']
queryset = queryset.order_by(*orderby)
return queryset
def get_facet(qs, request):
# NOTE facet counts for tree models include self AND descendants
def get_facet(qs=None, request=None, use_cache=True, hash_key=None):
"""
Gets an annotated list from a queryset.
:param qs:
recipe queryset to build facets from
:param request:
the web request that contains the necessary query parameters
:param use_cache:
will find results in cache, if any, and return them or empty list.
will save the list of recipes IDs in the cache for future processing
:param hash_key:
the cache key of the recipe list to process
only evaluated if the use_cache parameter is false
"""
facets = {}
keyword_list = request.query_params.getlist('keywords', [])
food_list = request.query_params.getlist('foods', [])
book_list = request.query_params.getlist('book', [])
search_keywords_or = request.query_params.get('keywords_or', True)
search_foods_or = request.query_params.get('foods_or', True)
search_books_or = request.query_params.get('books_or', True)
recipe_list = []
cache_timeout = 600
if use_cache:
qs_hash = hash(frozenset(qs.values_list('pk')))
facets['cache_key'] = str(qs_hash)
SEARCH_CACHE_KEY = f"recipes_filter_{qs_hash}"
if c := caches['default'].get(SEARCH_CACHE_KEY, None):
facets['Keywords'] = c['Keywords'] or []
facets['Foods'] = c['Foods'] or []
facets['Books'] = c['Books'] or []
facets['Ratings'] = c['Ratings'] or []
facets['Recent'] = c['Recent'] or []
else:
facets['Keywords'] = []
facets['Foods'] = []
facets['Books'] = []
rating_qs = qs.annotate(rating=Round(Avg(Case(When(cooklog__created_by=request.user, then='cooklog__rating'), default=Value(0)))))
facets['Ratings'] = dict(Counter(r.rating for r in rating_qs))
facets['Recent'] = ViewLog.objects.filter(
created_by=request.user, space=request.space,
created_at__gte=timezone.now() - timedelta(days=14) # TODO make days of recent recipe a setting
).values_list('recipe__pk', flat=True)
cached_search = {
'recipe_list': list(qs.values_list('id', flat=True)),
'keyword_list': request.query_params.getlist('keywords', []),
'food_list': request.query_params.getlist('foods', []),
'book_list': request.query_params.getlist('book', []),
'search_keywords_or': str2bool(request.query_params.get('keywords_or', True)),
'search_foods_or': str2bool(request.query_params.get('foods_or', True)),
'search_books_or': str2bool(request.query_params.get('books_or', True)),
'space': request.space,
'Ratings': facets['Ratings'],
'Recent': facets['Recent'],
'Keywords': facets['Keywords'],
'Foods': facets['Foods'],
'Books': facets['Books']
}
caches['default'].set(SEARCH_CACHE_KEY, cached_search, cache_timeout)
return facets
SEARCH_CACHE_KEY = f'recipes_filter_{hash_key}'
if c := caches['default'].get(SEARCH_CACHE_KEY, None):
recipe_list = c['recipe_list']
keyword_list = c['keyword_list']
food_list = c['food_list']
book_list = c['book_list']
search_keywords_or = c['search_keywords_or']
search_foods_or = c['search_foods_or']
search_books_or = c['search_books_or']
else:
return {}
# if using an OR search, will annotate all keywords, otherwise, just those that appear in results
if search_keywords_or:
keywords = Keyword.objects.filter(space=request.space).annotate(recipe_count=Count('recipe'))
else:
keywords = Keyword.objects.filter(recipe__in=qs, space=request.space).annotate(recipe_count=Count('recipe'))
keywords = Keyword.objects.filter(recipe__in=recipe_list, space=request.space).annotate(recipe_count=Count('recipe'))
# custom django-tree function annotates a queryset to make building a tree easier.
# see https://django-treebeard.readthedocs.io/en/latest/api.html#treebeard.models.Node.get_annotated_list_qs for details
kw_a = annotated_qs(keywords, root=True, fill=True)
# if using an OR search, will annotate all keywords, otherwise, just those that appear in results
# # if using an OR search, will annotate all keywords, otherwise, just those that appear in results
if search_foods_or:
foods = Food.objects.filter(space=request.space).annotate(recipe_count=Count('ingredient'))
else:
foods = Food.objects.filter(ingredient__step__recipe__in=list(qs.values_list('id', flat=True)), space=request.space).annotate(recipe_count=Count('ingredient'))
foods = Food.objects.filter(ingredient__step__recipe__in=recipe_list, space=request.space).annotate(recipe_count=Count('ingredient'))
food_a = annotated_qs(foods, root=True, fill=True)
rating_qs = qs.annotate(rating=Round(Avg(Case(When(cooklog__created_by=request.user, then='cooklog__rating'), default=Value(0)))))
# TODO add rating facet
facets['Ratings'] = dict(Counter(r.rating for r in rating_qs))
facets['Keywords'] = fill_annotated_parents(kw_a, keyword_list)
facets['Foods'] = fill_annotated_parents(food_a, food_list)
# TODO add book facet
facets['Books'] = []
facets['Recent'] = ViewLog.objects.filter(
created_by=request.user, space=request.space,
created_at__gte=timezone.now() - timedelta(days=14) # TODO make days of recent recipe a setting
).values_list('recipe__pk', flat=True)
c['Keywords'] = facets['Keywords']
c['Foods'] = facets['Foods']
c['Books'] = facets['Books']
caches['default'].set(SEARCH_CACHE_KEY, c, cache_timeout)
return facets

View File

@ -385,7 +385,7 @@ class Food(ExportModelOperationsMixin('food'), TreeModel, PermissionModelMixin):
return self.name
def delete(self):
if len(self.ingredient_set.all().exclude(step=None)) > 0:
if self.ingredient_set.all().exclude(step=None).count() > 0:
raise ProtectedError(self.name + _(" is part of a recipe step and cannot be deleted"), self.ingredient_set.all().exclude(step=None))
else:
return super().delete()

View File

@ -216,9 +216,9 @@ class KeywordSerializer(UniqueFieldsMixin, serializers.ModelSerializer):
def get_image(self, obj):
recipes = obj.recipe_set.all().filter(space=obj.space).exclude(image__isnull=True).exclude(image__exact='')
if len(recipes) == 0 and obj.has_children():
if recipes.count() == 0 and obj.has_children():
recipes = Recipe.objects.filter(keywords__in=obj.get_descendants(), space=obj.space).exclude(image__isnull=True).exclude(image__exact='') # if no recipes found - check whole tree
if len(recipes) != 0:
if recipes.count() != 0:
return random.choice(recipes).image.url
else:
return None
@ -249,7 +249,7 @@ class UnitSerializer(UniqueFieldsMixin, serializers.ModelSerializer):
def get_image(self, obj):
recipes = Recipe.objects.filter(steps__ingredients__unit=obj, space=obj.space).exclude(image__isnull=True).exclude(image__exact='')
if len(recipes) != 0:
if recipes.count() != 0:
return random.choice(recipes).image.url
else:
return None
@ -330,10 +330,10 @@ class FoodSerializer(UniqueFieldsMixin, WritableNestedModelSerializer):
# if food is not also a recipe, look for recipe images that use the food
recipes = Recipe.objects.filter(steps__ingredients__food=obj, space=obj.space).exclude(image__isnull=True).exclude(image__exact='')
# if no recipes found - check whole tree
if len(recipes) == 0 and obj.has_children():
if recipes.count() == 0 and obj.has_children():
recipes = Recipe.objects.filter(steps__ingredients__food__in=obj.get_descendants(), space=obj.space).exclude(image__isnull=True).exclude(image__exact='')
if len(recipes) != 0:
if recipes.count() != 0:
return random.choice(recipes).image.url
else:
return None

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -109,6 +109,7 @@ urlpatterns = [
path('api/backup/', api.get_backup, name='api_backup'),
path('api/ingredient-from-string/', api.ingredient_from_string, name='api_ingredient_from_string'),
path('api/share-link/<int:pk>', api.share_link, name='api_share_link'),
path('api/get_facets/', api.get_facets, name='api_get_facets'),
path('dal/keyword/', dal.KeywordAutocomplete.as_view(), name='dal_keyword'), # TODO is this deprecated? not yet, some old forms remain, could likely be changed to generic API endpoints
path('dal/food/', dal.IngredientsAutocomplete.as_view(), name='dal_food'), # TODO is this deprecated?

View File

@ -499,7 +499,7 @@ class RecipePagination(PageNumberPagination):
max_page_size = 100
def paginate_queryset(self, queryset, request, view=None):
self.facets = get_facet(queryset, request)
self.facets = get_facet(qs=queryset, request=request)
return super().paginate_queryset(queryset, request, view)
def get_paginated_response(self, data):
@ -906,3 +906,15 @@ def ingredient_from_string(request):
},
status=200
)
@group_required('user')
def get_facets(request):
key = request.GET['hash']
return JsonResponse(
{
'facets': get_facet(request=request, use_cache=False, hash_key=key),
},
status=200
)

View File

@ -1,7 +1,71 @@
from os import getenv
from django.conf import settings
from django.contrib.auth.middleware import RemoteUserMiddleware
from django.db import connection
class CustomRemoteUser(RemoteUserMiddleware):
header = getenv('PROXY_HEADER', 'HTTP_REMOTE_USER')
"""
Gist code by vstoykov, you can check his original gist at:
https://gist.github.com/vstoykov/1390853/5d2e8fac3ca2b2ada8c7de2fb70c021e50927375
Changes:
Ignoring static file requests and a certain useless admin request from triggering the logger.
Updated statements to make it Python 3 friendly.
"""
def terminal_width():
"""
Function to compute the terminal width.
"""
width = 0
try:
import struct, fcntl, termios
s = struct.pack('HHHH', 0, 0, 0, 0)
x = fcntl.ioctl(1, termios.TIOCGWINSZ, s)
width = struct.unpack('HHHH', x)[1]
except:
pass
if width <= 0:
try:
width = int(getenv['COLUMNS'])
except:
pass
if width <= 0:
width = 80
return width
def SqlPrintingMiddleware(get_response):
def middleware(request):
response = get_response(request)
if (
not settings.DEBUG
or len(connection.queries) == 0
or request.path_info.startswith(settings.MEDIA_URL)
or '/admin/jsi18n/' in request.path_info
):
return response
indentation = 2
print("\n\n%s\033[1;35m[SQL Queries for]\033[1;34m %s\033[0m\n" % (" " * indentation, request.path_info))
width = terminal_width()
total_time = 0.0
for query in connection.queries:
nice_sql = query['sql'].replace('"', '').replace(',', ', ')
sql = "\033[1;31m[%s]\033[0m %s" % (query['time'], nice_sql)
total_time = total_time + float(query['time'])
while len(sql) > width - indentation:
#print("%s%s" % (" " * indentation, sql[:width - indentation]))
sql = sql[width - indentation:]
#print("%s%s\n" % (" " * indentation, sql))
replace_tuple = (" " * indentation, str(total_time))
print("%s\033[1;32m[TOTAL TIME: %s seconds]\033[0m" % replace_tuple)
print("%s\033[1;32m[TOTAL QUERIES: %s]\033[0m" % (" " * indentation, len(connection.queries)))
return response
return middleware

View File

@ -16,8 +16,8 @@ import re
from django.contrib import messages
from django.utils.translation import gettext_lazy as _
# from dotenv import load_dotenv
# load_dotenv()
from dotenv import load_dotenv
load_dotenv()
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Get vars from .env files
@ -389,3 +389,6 @@ EMAIL_USE_TLS = bool(int(os.getenv('EMAIL_USE_TLS', False)))
EMAIL_USE_SSL = bool(int(os.getenv('EMAIL_USE_SSL', False)))
DEFAULT_FROM_EMAIL = os.getenv('DEFAULT_FROM_EMAIL', 'webmaster@localhost')
ACCOUNT_EMAIL_SUBJECT_PREFIX = os.getenv('ACCOUNT_EMAIL_SUBJECT_PREFIX', '[Tandoor Recipes] ') # allauth sender prefix
if os.getenv('SQL_DEBUG', False):
MIDDLEWARE += ('recipes.middleware.SqlPrintingMiddleware',)

View File

@ -223,12 +223,6 @@
style="flex-grow: 1; flex-shrink: 1; flex-basis: 0"/>
<b-input-group-append>
<b-input-group-text style="width:85px">
<!-- <b-form-checkbox v-model="settings.search_books_or" name="check-button"
@change="refreshData(false)"
class="shadow-none" tyle="width: 100%" switch>
<span class="text-uppercase" v-if="settings.search_books_or">{{ $t('or') }}</span>
<span class="text-uppercase" v-else>{{ $t('and') }}</span>
</b-form-checkbox> -->
</b-input-group-text>
</b-input-group-append>
</b-input-group>
@ -303,8 +297,7 @@ import VueCookies from 'vue-cookies'
Vue.use(VueCookies)
import {ResolveUrlMixin} from "@/utils/utils";
import {ApiMixin} from "@/utils/utils";
import {ApiMixin, ResolveUrlMixin} from "@/utils/utils";
import LoadingSpinner from "@/components/LoadingSpinner"; // is this deprecated?
@ -325,7 +318,7 @@ export default {
return {
// this.Models and this.Actions inherited from ApiMixin
recipes: [],
facets: [],
facets: {},
meal_plans: [],
last_viewed_recipes: [],
@ -387,7 +380,6 @@ export default {
if (this.$cookies.isKey(SETTINGS_COOKIE_NAME)) {
this.settings = Object.assign({}, this.settings, this.$cookies.get(SETTINGS_COOKIE_NAME))
}
let urlParams = new URLSearchParams(window.location.search);
if (urlParams.has('keyword')) {
@ -398,6 +390,18 @@ export default {
this.facets.Keywords.push({'id':x, 'name': 'loading...'})
}
}
this.facets.Foods = []
for (let x of this.settings.search_foods) {
this.facets.Foods.push({'id':x, 'name': 'loading...'})
}
this.facets.Keywords = []
for (let x of this.settings.search_keywords) {
this.facets.Keywords.push({'id':x, 'name': 'loading...'})
}
this.facets.Books = []
for (let x of this.settings.search_books) {
this.facets.Books.push({'id':x, 'name': 'loading...'})
}
this.loadMealPlan()
this.refreshData(false)
})
@ -457,6 +461,9 @@ export default {
this.pagination_count = result.data.count
this.facets = result.data.facets
if(this.facets?.cache_key) {
this.getFacets(this.facets.cache_key)
}
this.recipes = this.removeDuplicates(result.data.results, recipe => recipe.id)
if (!this.searchFiltered){
// if meal plans are being shown - filter out any meal plan recipes from the recipe list
@ -530,6 +537,11 @@ export default {
return [undefined, undefined]
}
},
getFacets: function(hash) {
this.genericGetAPI('api_get_facets', {hash: hash}).then((response) => {
this.facets = {...this.facets, ...response.data.facets}
})
}
}
}

View File

@ -203,6 +203,9 @@ export const ApiMixin = {
});
let apiClient = new ApiApiFactory()
return apiClient[func](...parameters)
},
genericGetAPI: function(url, options) {
return axios.get(this.resolveDjangoUrl(url), {'params':options, 'emulateJSON': true})
}
}
}

View File

@ -3,83 +3,83 @@
"assets": {
"../../templates/sw.js": {
"name": "../../templates/sw.js",
"path": "..\\..\\templates\\sw.js"
"path": "../../templates/sw.js"
},
"js/chunk-2d0da313.js": {
"name": "js/chunk-2d0da313.js",
"path": "js\\chunk-2d0da313.js"
"path": "js/chunk-2d0da313.js"
},
"css/chunk-vendors.css": {
"name": "css/chunk-vendors.css",
"path": "css\\chunk-vendors.css"
"path": "css/chunk-vendors.css"
},
"js/chunk-vendors.js": {
"name": "js/chunk-vendors.js",
"path": "js\\chunk-vendors.js"
"path": "js/chunk-vendors.js"
},
"css/cookbook_view.css": {
"name": "css/cookbook_view.css",
"path": "css\\cookbook_view.css"
"path": "css/cookbook_view.css"
},
"js/cookbook_view.js": {
"name": "js/cookbook_view.js",
"path": "js\\cookbook_view.js"
"path": "js/cookbook_view.js"
},
"css/edit_internal_recipe.css": {
"name": "css/edit_internal_recipe.css",
"path": "css\\edit_internal_recipe.css"
"path": "css/edit_internal_recipe.css"
},
"js/edit_internal_recipe.js": {
"name": "js/edit_internal_recipe.js",
"path": "js\\edit_internal_recipe.js"
"path": "js/edit_internal_recipe.js"
},
"js/import_response_view.js": {
"name": "js/import_response_view.js",
"path": "js\\import_response_view.js"
"path": "js/import_response_view.js"
},
"css/meal_plan_view.css": {
"name": "css/meal_plan_view.css",
"path": "css\\meal_plan_view.css"
"path": "css/meal_plan_view.css"
},
"js/meal_plan_view.js": {
"name": "js/meal_plan_view.js",
"path": "js\\meal_plan_view.js"
"path": "js/meal_plan_view.js"
},
"css/model_list_view.css": {
"name": "css/model_list_view.css",
"path": "css\\model_list_view.css"
"path": "css/model_list_view.css"
},
"js/model_list_view.js": {
"name": "js/model_list_view.js",
"path": "js\\model_list_view.js"
"path": "js/model_list_view.js"
},
"js/offline_view.js": {
"name": "js/offline_view.js",
"path": "js\\offline_view.js"
"path": "js/offline_view.js"
},
"css/recipe_search_view.css": {
"name": "css/recipe_search_view.css",
"path": "css\\recipe_search_view.css"
"path": "css/recipe_search_view.css"
},
"js/recipe_search_view.js": {
"name": "js/recipe_search_view.js",
"path": "js\\recipe_search_view.js"
"path": "js/recipe_search_view.js"
},
"css/recipe_view.css": {
"name": "css/recipe_view.css",
"path": "css\\recipe_view.css"
"path": "css/recipe_view.css"
},
"js/recipe_view.js": {
"name": "js/recipe_view.js",
"path": "js\\recipe_view.js"
"path": "js/recipe_view.js"
},
"js/supermarket_view.js": {
"name": "js/supermarket_view.js",
"path": "js\\supermarket_view.js"
"path": "js/supermarket_view.js"
},
"js/user_file_view.js": {
"name": "js/user_file_view.js",
"path": "js\\user_file_view.js"
"path": "js/user_file_view.js"
},
"recipe_search_view.html": {
"name": "recipe_search_view.html",