search form and help doc

cookbook/forms.py

@@ -482,7 +482,7 @@ class SearchPreferenceForm(forms.ModelForm):
         fields = ('search', 'unaccent', 'icontains', 'istartswith', 'trigram', 'fulltext')
 
         help_texts = {
-            'search': _('Select type method of search.  Click here for full desciption of choices.'),
+            'search': _('Select type method of search.  Click <a href="/docs/search/">here</a> for full desciption of choices.'),
             'unaccent': _('Fields to search ignoring accents.  Selecting this option can improve or degrade search quality depending on language'),
             'icontains': _("Fields to search for partial matches.  (e.g. searching for 'Pie' will return 'pie' and 'piece' and 'soapie')"),
             'istartswith': _("Fields to search for beginning of word matches. (e.g. searching for 'sa' will return 'salad' and 'sandwich')"),
@@ -507,3 +507,5 @@ class SearchPreferenceForm(forms.ModelForm):
             'trigram': MultiSelectWidget,
             'fulltext': MultiSelectWidget,
         }
+
+        

cookbook/helper/recipe_search.py

@@ -12,14 +12,7 @@ from cookbook.models import Food, Keyword, ViewLog
 
 
 def search_recipes(request, queryset, params):
-    fields = {
+    search_prefs = request.user.searchpreference
-        'name': 'name',
-        'description': 'description',
-        'instructions': 'steps__instruction',
-        'foods': 'steps__ingredients__food__name',
-        'keywords': 'keywords__name'
-    }
-
     search_string = params.get('query', '')
     search_keywords = params.getlist('keywords', [])
     search_foods = params.getlist('foods', [])
@@ -46,35 +39,38 @@ def search_recipes(request, queryset, params):
             created_at__gte=(datetime.now() - timedelta(days=7)), then=Value(100)),
             default=Value(0), )).order_by('-new_recipe', 'name')
 
-    search_type = None
+    search_type = search_prefs.search or 'plain'
     search_sort = None
     if len(search_string) > 0:
-        # TODO move all of these to settings somewhere - probably user settings
+        unaccent_include = search_prefs.unaccent.values_list('field', flat=True)
 
-        unaccent_include = ['name', 'description', 'instructions', 'keywords', 'foods']  # can also contain: description, instructions, keywords, foods
+        icontains_include = [x + '__unaccent' if x in unaccent_include else x for x in search_prefs.icontains.values_list('field', flat=True)]
-        # TODO when setting up settings length of arrays below must be >=1
+        istartswith_include = [x + '__unaccent' if x in unaccent_include else x for x in search_prefs.istartswith.values_list('field', flat=True)]
-
+        trigram_include = [x + '__unaccent' if x in unaccent_include else x for x in search_prefs.trigram.values_list('field', flat=True)]
-        icontains_include = []  # can contain: name, description, instructions, keywords, foods
+        fulltext_include = search_prefs.fulltext.values_list('field', flat=True)  # fulltext doesn't use field name directly
-        istartswith_include = ['name']  # can also contain: description, instructions, keywords, foods
-        trigram_include = ['name', 'description', 'instructions']  # only these choices - keywords and foods are really, really, really slow maybe add to subquery?
-        fulltext_include = ['name', 'description', 'instructions', 'foods', 'keywords']
-
-        # END OF SETTINGS SECTION
-        for f in unaccent_include:
-            fields[f] += '__unaccent'
 
+        # if no filters are configured use name__icontains as default
+        if len(icontains_include) + len(istartswith_include) + len(trigram_include) + len(fulltext_include) == 0:
+            filters = [Q(**{"name__icontains": search_string})]
+        else:
             filters = []
+
+        # dynamically build array of filters that will be applied
         for f in icontains_include:
-            filters += [Q(**{"%s__icontains" % fields[f]: search_string})]
+            filters += [Q(**{"%s__icontains" % f: search_string})]
 
         for f in istartswith_include:
-            filters += [Q(**{"%s__istartswith" % fields[f]: search_string})]
+            filters += [Q(**{"%s__istartswith" % f: search_string})]
 
         if settings.DATABASES['default']['ENGINE'] in ['django.db.backends.postgresql_psycopg2', 'django.db.backends.postgresql']:
             language = DICTIONARY.get(translation.get_language(), 'simple')
             # django full text search https://docs.djangoproject.com/en/3.2/ref/contrib/postgres/search/#searchquery
-            search_type = 'websearch'  # other postgress options are phrase or plain or raw (websearch and trigrams are mutually exclusive)
+            # TODO can options install this extension to further enhance search query language https://github.com/caub/pg-tsquery
+            # trigram breaks full text search 'websearch' and 'raw' capabilities and will be ignored if those methods are chosen
+            if search_type in ['websearch', 'raw']:
                 search_trigram = False
+            else:
+                search_trigram = True
             search_query = SearchQuery(
                 search_string,
                 search_type=search_type,
@@ -86,10 +82,11 @@ def search_recipes(request, queryset, params):
                 trigram = None
                 for f in trigram_include:
                     if trigram:
-                        trigram += TrigramSimilarity(fields[f], search_string)
+                        trigram += TrigramSimilarity(f, search_string)
                     else:
-                        trigram = TrigramSimilarity(fields[f], search_string)
+                        trigram = TrigramSimilarity(f, search_string)
                 queryset.annotate(simularity=trigram)
+                # TODO allow user to play with trigram scores
                 filters += [Q(simularity__gt=0.5)]
 
             if 'name' in fulltext_include:

cookbook/migrations/0124_build_full_text_index.py

@@ -95,7 +95,7 @@ class Migration(migrations.Migration):
             name='SearchPreference',
             fields=[
                 ('user', annoying.fields.AutoOneToOneField(on_delete=deletion.CASCADE, primary_key=True, serialize=False, to='auth.user')),
-                ('search', models.CharField(choices=[('PLAIN', 'Plain'), ('PHRASE', 'Phrase'), ('WEBSEARCH', 'Web'), ('RAW', 'Raw')], default='SIMPLE', max_length=32)),
+                ('search', models.CharField(choices=[('plain', 'Plain'), ('phrase', 'Phrase'), ('websearch', 'Web'), ('raw', 'Raw')], default='plain', max_length=32)),
                 ('fulltext', models.ManyToManyField(blank=True, related_name='fulltext_fields', to='cookbook.SearchFields')),
                 ('icontains', models.ManyToManyField(blank=True, default=nameSearchField, related_name='icontains_fields', to='cookbook.SearchFields')),
                 ('istartswith', models.ManyToManyField(blank=True, related_name='istartswith_fields', to='cookbook.SearchFields')),

cookbook/models.py

@@ -746,13 +746,12 @@ def nameSearchField():
 class SearchPreference(models.Model, PermissionModelMixin):
     # Search Style (validation parsleyjs.org)
     # phrase or plain or raw (websearch and trigrams are mutually exclusive)
-    SIMPLE = 'SIMPLE'
+    SIMPLE = 'plain'
-    PLAIN = 'PLAIN'
+    PHRASE = 'phrase'
-    PHRASE = 'PHRASE'
+    WEB = 'websearch'
-    WEB = 'WEBSEARCH'
+    RAW = 'raw'
-    RAW = 'RAW'
     SEARCH_STYLE = (
-        (PLAIN, _('Plain')),
+        (SIMPLE, _('Simple')),
         (PHRASE, _('Phrase')),
         (WEB, _('Web')),
         (RAW, _('Raw'))

cookbook/templates/search_info.html

@@ -0,0 +1,110 @@
+{% extends "base.html" %}
+{% load static %}
+{% load i18n %}
+
+{% block title %}{% trans "Search Settings" %}{% endblock %}
+
+{% block content %}
+
+    <h1>{% trans 'Search Settings' %}</h1>
+    {% blocktrans %}
+        Creating the best search experience is complicated and weighs heavily on your personal configuration.  
+        Changing any of the search settings can have significant impact on the speed and quality of the results.
+        Search Methods, Trigrams and Full Text Search configurations are only available if you are using Postgres for your database.
+    {% endblocktrans %}
+
+    <br/>
+    <br/>
+
+    <h3>{% trans 'Search Methods' %}</h3>
+
+    <div class="card">
+        <div class="card-body">
+            <p> {% blocktrans %} 
+            Full text searches attempt to normalize the words provided to match common variants.  For example: 'forked', 'forking', 'forks' will all normalize to 'fork'.
+            There are several methods available, described below, that will control how the search behavior should react when multiple words are searched.
+            Full technical details on how these operate can be viewed on <a href=https://www.postgresql.org/docs/current/textsearch-controls.html#TEXTSEARCH-PARSING-QUERIES>Postgresql's website.</a>
+            {% endblocktrans %}</p>
+            <h4>{% trans 'Simple' %}</h4>
+            <p> {% blocktrans %} 
+            Simple searches ignore punctuation and common words such as 'the', 'a', 'and'. And will treat seperate words as required.
+            Searching for 'apple or flour' will return any recipe that includes both 'apple' and 'flour' anywhere in the fields that have been selected for a full text search.
+            {% endblocktrans %}</p>
+            <h4>{% trans 'Phrase' %}</h4>
+            <p> {% blocktrans %} 
+            Phrase searches ignore punctuation, but will search for all of the words in the exact order provided.
+            Searching for 'apple or flour' will only return a recipe that includes the exact phrase 'apple or flour' in any of the fields that have been selected for a full text search.
+            {% endblocktrans %}</p>
+            <h4>{% trans 'Web' %}</h4>
+            <p> {% blocktrans %} 
+            Web searches simulate functionality found on many web search sites supporting special syntax.
+            Placing quotes around several words will convert those words into a phrase.
+            'or' is recongized as searching for the word (or phrase) immediately before 'or' OR the word (or phrase) directly after.
+            '-' is recognized as searching for recipes that do not include the word (or phrase) that comes immediately after. 
+            For example searching for 'apple pie' or cherry -butter will return any recipe that includes the phrase 'apple pie' or the word 'cherry' 
+            in any field included in the full text search but exclude any recipe that has the word 'butter' in any field included.
+            {% endblocktrans %}</p>
+            <h4>{% trans 'Raw' %}</h4>
+            <p> {% blocktrans %} 
+            Raw search is similar to Web except will take puncuation operators such as '|', '&' and '()'
+            {% endblocktrans %}</p>
+        </div>
+
+    </div>
+
+    <br/>
+    <h4>fuzzy search</h4>
+    <div class="card">
+        <div class="card-body">
+            {% blocktrans %} 
+            Another approach to searching that also requires Postgresql is fuzzy search or trigram similarity. A trigram is a group of three consecutive characters.
+            For example searching for 'apple' will create x trigrams 'app', 'ppl', 'ple' and will create a score of how closely words match the generated trigrams.
+            One benefit of searching trigams is that a search for 'sandwich' will find mispelled words such as 'sandwhich' that would be missed by other methods.
+            {% endblocktrans %} 
+        </div>
+
+    </div>
+
+    <br/>
+    <h4>{% trans 'Search Fields' %}</h4>
+
+    <div class="card">
+        <div class="card-body">
+            {% blocktrans %} 
+            Unaccent is a special case in that it enables searching a field 'unaccented' for each search style attempting to ignore accented values. 
+            For example when you enable unaccent for 'Name' any search (starts with, contains, trigram) will attempt the search ignoring accented characters.
+            
+            For the other options, you can enable search on any or all fields and they will be combined together with an assumed 'OR'.
+            For example enabling 'Name' for Starts With, 'Name' and 'Description' for Partial Match and 'Ingredients' and 'Keywords' for Full Search
+            and searching for 'apple' will generate a search that will return recipes that have:
+            - A recipe name that starts with 'apple'
+            - OR a recipe name that contains 'apple'
+            - OR a recipe description that contains 'apple'
+            - OR a recipe that will have a full text search match ('apple' or 'apples') in ingredients
+            - OR a recipe that will have a full text search match in Keywords
+
+            Combining too many fields in too many types of search can have a negative impact on performance, create duplicate results or return unexpected results.
+            For example, enabling fuzzy search or partial matches will interfere with web search methods.  
+            Searching for 'apple -pie' with fuzzy search and full text search will return the recipe Apple Pie.  Though it is not included in the full text results, it does match the trigram results.
+            {% endblocktrans %} 
+        </div>
+
+    </div>
+
+    <br/>
+    <h4>{% trans 'Search Index' %}</h4>
+
+    <div class="card">
+        <div class="card-body">
+            {% blocktrans %} 
+            Trigram search and Full Text Search both rely on database indexes to perform effectively.  
+            You can rebuild the indexes on all fields in the Admin page for Recipes and selecting all recipes and running 'rebuild index for selected recipes'
+            You can also rebuild indexes at the command line by executing the management command 'python manage.py rebuildindex'
+            {% endblocktrans %} 
+        </div>
+
+    </div>
+<br>
+<br>
+<br>
+{% endblock %}

cookbook/templates/settings.html

@@ -21,27 +21,35 @@
     <!-- Nav tabs -->
     <ul class="nav nav-tabs" id="myTab" role="tablist" style="margin-bottom: 2vh">
         <li class="nav-item" role="presentation">
-            <a class="nav-link active" id="account-tab" data-toggle="tab" href="#account" role="tab"
+            <a class="nav-link {% if active_tab == 'account' %} active {% endif %}" id="account-tab" data-toggle="tab" href="#account" role="tab"
-               aria-controls="account" aria-selected="true">{% trans 'Account' %}</a>
+               aria-controls="account"  
+               aria-selected="{% if active_tab == 'account' %} 'true' {% else %} 'false' {% endif %}">
+               {% trans 'Account' %}</a>
         </li>
         <li class="nav-item" role="presentation">
-            <a class="nav-link" id="preferences-tab" data-toggle="tab" href="#preferences" role="tab"
+            <a class="nav-link {% if active_tab == 'prefernces' %} active {% endif %}" id="preferences-tab" data-toggle="tab" href="#preferences" role="tab"
-               aria-controls="preferences" aria-selected="false">{% trans 'Preferences' %}</a>
+               aria-controls="preferences" 
+               aria-selected="{% if active_tab == 'prefernces' %} 'true' {% else %} 'false' {% endif %}">
+               {% trans 'Preferences' %}</a>
         </li>
         <li class="nav-item" role="presentation">
-            <a class="nav-link" id="api-tab" data-toggle="tab" href="#api" role="tab" aria-controls="api"
+            <a class="nav-link {% if active_tab == 'api' %} active {% endif %}" id="api-tab" data-toggle="tab" href="#api" role="tab" 
-               aria-selected="false">{% trans 'API-Settings' %}</a>
+                aria-controls="api" 
+                aria-selected="{% if active_tab == 'api' %} 'true' {% else %} 'false' {% endif %}">
+                {% trans 'API-Settings' %}</a>
         </li>
         <li class="nav-item" role="presentation">
-            <a class="nav-link" id="search-tab" data-toggle="tab" href="#search" role="tab" aria-controls="search"
+            <a class="nav-link {% if active_tab == 'search' %} active {% endif %}" id="search-tab" data-toggle="tab" href="#search" role="tab"
-               aria-selected="false">{% trans 'Search-Settings' %}</a>
+                aria-controls="search"  
+                aria-selected="{% if active_tab == 'search' %} 'true' {% else %} 'false' {% endif %}">
+                {% trans 'Search-Settings' %}</a>
         </li>
 
     </ul>
 
     <!-- Tab panes -->
     <div class="tab-content">
-        <div class="tab-pane active" id="account" role="tabpanel" aria-labelledby="account-tab">
+        <div class="tab-pane {% if active_tab == 'account' %} active {% endif %}" id="account" role="tabpanel" aria-labelledby="account-tab">
             <h4>{% trans 'Name Settings' %}</h4>
             <form action="." method="post">
                 {% csrf_token %}
@@ -62,7 +70,7 @@
             <br/>
 
         </div>
-        <div class="tab-pane" id="preferences" role="tabpanel" aria-labelledby="preferences-tab">
+        <div class="tab-pane {% if active_tab == 'preferences' %} active {% endif %}" id="preferences" role="tabpanel" aria-labelledby="preferences-tab">
 
 
             <div class="row">
@@ -113,7 +121,7 @@
 
 
         </div>
-        <div class="tab-pane" id="api" role="tabpanel" aria-labelledby="api-tab">
+        <div class="tab-pane {% if active_tab == 'api' %} active {% endif %}" id="api" role="tabpanel" aria-labelledby="api-tab">
 
             <div class="row">
                 <div class="col col-md-12">
@@ -145,7 +153,7 @@
 
 
         </div>
-        <div class="tab-pane" id="search" role="tabpanel" aria-labelledby="search-tab">
+        <div class="tab-pane {% if active_tab == 'search' %} active {% endif %}" id="search" role="tabpanel" aria-labelledby="search-tab">
             <h4>{% trans 'Search Settings' %}</h4>
             <form action="." method="post">
                 {% csrf_token %}

cookbook/urls.py

@@ -118,6 +118,7 @@ urlpatterns = [
     path('telegram/hook/<slug:token>/', telegram.hook, name='telegram_hook'),
 
     path('docs/markdown/', views.markdown_info, name='docs_markdown'),
+    path('docs/search/', views.search_info, name='docs_search'),
     path('docs/api/', views.api_info, name='docs_api'),
 
     path('openapi/', get_schema_view(title="Django Recipes", version=VERSION_NUMBER, public=True,

cookbook/views/views.py

@@ -34,6 +34,8 @@ from cookbook.models import (Comment, CookLog, InviteLink, MealPlan,
 from cookbook.tables import (CookLogTable, RecipeTable, RecipeTableSmall,
                              ViewLogTable, InviteLinkTable)
 from cookbook.views.data import Object
+from recipes import settings
+from recipes.settings import DEMO
 from recipes.version import BUILD_REF, VERSION_NUMBER
 
 
@@ -305,11 +307,14 @@ def user_settings(request):
 
     up = request.user.userpreference
     sp = request.user.searchpreference
+    search_error = False
+    active_tab = 'account'
 
     user_name_form = UserNameForm(instance=request.user)
 
     if request.method == "POST":
         if 'preference_form' in request.POST:
+            active_tab = 'preferences'
             form = UserPreferenceForm(request.POST, prefix='preference')
             if form.is_valid():
                 if not up:
@@ -347,17 +352,33 @@ def user_settings(request):
                 update_session_auth_hash(request, user)
 
         elif 'search_form' in request.POST:
+            active_tab = 'search'
             search_form = SearchPreferenceForm(request.POST, prefix='search')
-            if form.is_valid():
+            if search_form.is_valid():
                 if not sp:
-                    sp = search_form(user=request.user)
+                    sp = SearchPreferenceForm(user=request.user)
-
+                fields_searched = (
+                    len(search_form.cleaned_data['icontains'])
+                    + len(search_form.cleaned_data['istartswith'])
+                    + len(search_form.cleaned_data['trigram'])
+                    + len(search_form.cleaned_data['fulltext']))
+                # TODO add 'recommended' option
+                if fields_searched == 0:
+                    search_form.add_error(None, _('You must select at least one field to search!'))
+                    search_error = True
+                elif search_form.cleaned_data['search'] in ['websearch', 'raw'] and len(search_form.cleaned_data['fulltext']) == 0:
+                    search_form.add_error('search', _('To use this search method you must select at least one full text search field!'))
+                    search_error = True
+                elif search_form.cleaned_data['search'] in ['websearch', 'raw'] and len(search_form.cleaned_data['trigram']) > 0:
+                    search_form.add_error(None, _('Fuzzy search is not compatible with this search method!'))
+                    search_error = True
+                else:
                     sp.search = search_form.cleaned_data['search']
-                sp.unaccent = search_form.cleaned_data['unaccent']
+                    sp.unaccent.set(search_form.cleaned_data['unaccent'])
-                sp.icontains = search_form.cleaned_data['icontains']
+                    sp.icontains.set(search_form.cleaned_data['icontains'])
-                sp.istartswith = search_form.cleaned_data['istartswith']
+                    sp.istartswith.set(search_form.cleaned_data['istartswith'])
-                sp.trigram = search_form.cleaned_data['trigram']
+                    sp.trigram.set(search_form.cleaned_data['trigram'])
-                sp.fulltext = search_form.cleaned_data['fulltext']
+                    sp.fulltext.set(search_form.cleaned_data['fulltext'])
 
                     sp.save()
     if up:
@@ -365,19 +386,27 @@ def user_settings(request):
     else:
         preference_form = UserPreferenceForm()
 
-    if sp:
+    fields_searched = len(sp.icontains.all()) + len(sp.istartswith.all()) + len(sp.trigram.all()) + len(sp.fulltext.all())
-        preference_form = SearchPreferenceForm(instance=sp)
+    if sp and not search_error and fields_searched > 0:
-    else:
+        search_form = SearchPreferenceForm(instance=sp)
-        preference_form = SearchPreferenceForm()
+    elif not search_error:
+        search_form = SearchPreferenceForm()
 
     if (api_token := Token.objects.filter(user=request.user).first()) is None:
         api_token = Token.objects.create(user=request.user)
 
+    # these fields require postgress - just disable them if postgress isn't available
+    if not settings.DATABASES['default']['ENGINE'] in ['django.db.backends.postgresql_psycopg2', 'django.db.backends.postgresql']:
+        search_form.fields['search'].disabled = True
+        search_form.fields['trigram'].disabled = True
+        search_form.fields['fulltext'].disabled = True
+
     return render(request, 'settings.html', {
         'preference_form': preference_form,
         'user_name_form': user_name_form,
         'api_token': api_token,
-        'search_form': search_form
+        'search_form': search_form,
+        'active_tab': active_tab
     })
 
 
@@ -560,6 +589,10 @@ def markdown_info(request):
     return render(request, 'markdown_info.html', {})
 
 
+def search_info(request):
+    return render(request, 'search_info.html', {})
+
+
 @group_required('guest')
 def api_info(request):
     return render(request, 'api_info.html', {})