From 502a606534fe27000524eaa223f4a3bff940fb0f Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Sun, 28 Jan 2024 22:59:51 +0100 Subject: [PATCH] Update the code based on feedback. set Default to enabled, add to documentation how to disable it. Add extra documentation --- .gitignore | 1 - cookbook/connectors/connector.py | 2 ++ cookbook/connectors/connector_manager.py | 28 +++++++++++++++++------- cookbook/signals.py | 3 +-- cookbook/views/api.py | 13 +++++------ cookbook/views/new.py | 6 ++++- docs/system/configuration.md | 10 +++++++++ recipes/settings.py | 3 ++- 8 files changed, 46 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index 4c8df7a7..1c5e40a4 100644 --- a/.gitignore +++ b/.gitignore @@ -54,7 +54,6 @@ docs/_build/ target/ \.idea/dataSources/ -.idea \.idea/dataSources\.xml \.idea/dataSources\.local\.xml diff --git a/cookbook/connectors/connector.py b/cookbook/connectors/connector.py index 3647dc71..27e9408d 100644 --- a/cookbook/connectors/connector.py +++ b/cookbook/connectors/connector.py @@ -3,6 +3,7 @@ from abc import ABC, abstractmethod from cookbook.models import ShoppingListEntry, Space, ConnectorConfig +# A Connector is 'destroyed' & recreated each time 'any' ConnectorConfig in a space changes. class Connector(ABC): @abstractmethod def __init__(self, config: ConnectorConfig): @@ -12,6 +13,7 @@ class Connector(ABC): async def on_shopping_list_entry_created(self, space: Space, instance: ShoppingListEntry) -> None: pass + # This method might not trigger on 'direct' entry updates: https://stackoverflow.com/a/35238823 @abstractmethod async def on_shopping_list_entry_updated(self, space: Space, instance: ShoppingListEntry) -> None: pass diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 64c55265..fe42c1a7 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -7,18 +7,18 @@ from dataclasses import dataclass from enum import Enum from multiprocessing import JoinableQueue from types import UnionType -from typing import List, Any, Dict, Optional +from typing import List, Any, Dict, Optional, Type from django_scopes import scope +from django.conf import settings from cookbook.connectors.connector import Connector from cookbook.connectors.homeassistant import HomeAssistant -from cookbook.models import ShoppingListEntry, Recipe, MealPlan, Space, ConnectorConfig +from cookbook.models import ShoppingListEntry, Space, ConnectorConfig multiprocessing.set_start_method('fork') # https://code.djangoproject.com/ticket/31169 -QUEUE_MAX_SIZE = 25 -REGISTERED_CLASSES: UnionType = ShoppingListEntry | Recipe | MealPlan +REGISTERED_CLASSES: UnionType | Type = ShoppingListEntry class ActionType(Enum): @@ -33,12 +33,20 @@ class Work: actionType: ActionType +# The way ConnectionManager works is as follows: +# 1. On init, it starts a worker & creates a queue for 'Work' +# 2. Then any time its called, it verifies the type of action (create/update/delete) and if the item is of interest, pushes the Work (non blocking) to the queue. +# 3. The worker consumes said work from the queue. +# 3.1 If the work is of type ConnectorConfig, it flushes its cache of known connectors (per space.id) +# 3.2 If work is of type REGISTERED_CLASSES, it asynchronously fires of all connectors and wait for them to finish (runtime should depend on the 'slowest' connector) +# 4. Work is marked as consumed, and next entry of the queue is consumed. +# Each 'Work' is processed in sequential by the worker, so the throughput is about [workers * the slowest connector] class ConnectorManager: _queue: JoinableQueue _listening_to_classes = REGISTERED_CLASSES | ConnectorConfig def __init__(self): - self._queue = multiprocessing.JoinableQueue(maxsize=QUEUE_MAX_SIZE) + self._queue = multiprocessing.JoinableQueue(maxsize=settings.EXTERNAL_CONNECTORS_QUEUE_SIZE) self._worker = multiprocessing.Process(target=self.worker, args=(self._queue,), daemon=True) self._worker.start() @@ -75,7 +83,7 @@ class ConnectorManager: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - _connectors: Dict[str, List[Connector]] = dict() + _connectors: Dict[int, List[Connector]] = dict() while True: try: @@ -90,7 +98,7 @@ class ConnectorManager: refresh_connector_cache = isinstance(item.instance, ConnectorConfig) space: Space = item.instance.space - connectors: Optional[List[Connector]] = _connectors.get(space.name) + connectors: Optional[List[Connector]] = _connectors.get(space.id) if connectors is None or refresh_connector_cache: if connectors is not None: @@ -111,7 +119,7 @@ class ConnectorManager: connectors.append(connector) - _connectors[space.name] = connectors + _connectors[space.id] = connectors if len(connectors) == 0 or refresh_connector_cache: worker_queue.task_done() @@ -134,6 +142,9 @@ class ConnectorManager: async def close_connectors(connectors: List[Connector]): tasks: List[Task] = [asyncio.create_task(connector.close()) for connector in connectors] + if len(tasks) == 0: + return + try: await asyncio.gather(*tasks, return_exceptions=False) except BaseException: @@ -161,6 +172,7 @@ async def run_connectors(connectors: List[Connector], space: Space, instance: RE return try: + # Wait for all async tasks to finish, if one fails, the others still continue. await asyncio.gather(*tasks, return_exceptions=False) except BaseException: logging.exception("received an exception from one of the connectors") diff --git a/cookbook/signals.py b/cookbook/signals.py index ce957bce..cd183c58 100644 --- a/cookbook/signals.py +++ b/cookbook/signals.py @@ -15,7 +15,6 @@ from cookbook.helper.shopping_helper import RecipeShoppingEditor from cookbook.managers import DICTIONARY from cookbook.models import (Food, MealPlan, PropertyType, Recipe, SearchFields, SearchPreference, Step, Unit, UserPreference) -from recipes.settings import ENABLE_EXTERNAL_CONNECTORS SQLITE = True if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.postgresql': @@ -165,7 +164,7 @@ def clear_property_type_cache(sender, instance=None, created=False, **kwargs): caches['default'].delete(CacheHelper(instance.space).PROPERTY_TYPE_CACHE_KEY) -if ENABLE_EXTERNAL_CONNECTORS: +if not settings.DISABLE_EXTERNAL_CONNECTORS: handler = ConnectorManager() post_save.connect(handler, dispatch_uid="connector_manager") post_delete.connect(handler, dispatch_uid="connector_manager") diff --git a/cookbook/views/api.py b/cookbook/views/api.py index bffc3ee4..446f0b7f 100644 --- a/cookbook/views/api.py +++ b/cookbook/views/api.py @@ -643,13 +643,12 @@ class FoodViewSet(viewsets.ModelViewSet, TreeMixin): if pt.fdc_id: for fn in data['foodNutrients']: if fn['nutrient']['id'] == pt.fdc_id: - food_property_list.append( - Property( - property_type_id=pt.id, - property_amount=round(fn['amount'], 2), - import_food_id=food.id, - space=self.request.space, - )) + food_property_list.append(Property( + property_type_id=pt.id, + property_amount=round(fn['amount'], 2), + import_food_id=food.id, + space=self.request.space, + )) Property.objects.bulk_create(food_property_list, ignore_conflicts=True, unique_fields=('space', 'import_food_id', 'property_type',)) diff --git a/cookbook/views/new.py b/cookbook/views/new.py index 93ad9996..7e973251 100644 --- a/cookbook/views/new.py +++ b/cookbook/views/new.py @@ -78,10 +78,14 @@ class ConnectorConfigCreate(GroupRequiredMixin, CreateView): success_url = reverse_lazy('list_connector_config') def form_valid(self, form): - if self.request.space.demo or settings.HOSTED: + if self.request.space.demo: messages.add_message(self.request, messages.ERROR, _('This feature is not yet available in the hosted version of tandoor!')) return redirect('index') + if settings.DISABLE_EXTERNAL_CONNECTORS: + messages.add_message(self.request, messages.ERROR, _('This feature is not enabled by the server admin!')) + return redirect('index') + obj = form.save(commit=False) obj.token = form.cleaned_data['update_token'] obj.created_by = self.request.user diff --git a/docs/system/configuration.md b/docs/system/configuration.md index f6e64d24..dc64d41d 100644 --- a/docs/system/configuration.md +++ b/docs/system/configuration.md @@ -437,6 +437,16 @@ key [here](https://fdc.nal.usda.gov/api-key-signup.html). FDC_API_KEY=DEMO_KEY ``` +#### External Connectors + +`DISABLE_EXTERNAL_CONNECTORS` is a global switch to disable External Connectors entirely (e.g. HomeAssistant). +`EXTERNAL_CONNECTORS_QUEUE_SIZE` is the amount of changes that are kept in memory if the worker cannot keep up. + +```env +DISABLE_EXTERNAL_CONNECTORS=0 // 0 = connectors enabled, 1 = connectors enabled +EXTERNAL_CONNECTORS_QUEUE_SIZE=25 +``` + ### Debugging/Development settings !!! warning diff --git a/recipes/settings.py b/recipes/settings.py index 319271ea..6439f727 100644 --- a/recipes/settings.py +++ b/recipes/settings.py @@ -555,6 +555,7 @@ DEFAULT_FROM_EMAIL = os.getenv('DEFAULT_FROM_EMAIL', 'webmaster@localhost') ACCOUNT_EMAIL_SUBJECT_PREFIX = os.getenv( 'ACCOUNT_EMAIL_SUBJECT_PREFIX', '[Tandoor Recipes] ') # allauth sender prefix -ENABLE_EXTERNAL_CONNECTORS = bool(int(os.getenv('ENABLE_EXTERNAL_CONNECTORS', False))) +DISABLE_EXTERNAL_CONNECTORS = bool(int(os.getenv('DISABLE_EXTERNAL_CONNECTORS', False))) +EXTERNAL_CONNECTORS_QUEUE_SIZE = int(os.getenv('EXTERNAL_CONNECTORS_QUEUE_SIZE', 25)) mimetypes.add_type("text/javascript", ".js", True)