From e5f0c19cdcceb042ca24b2831483ba38348482c3 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Thu, 11 Jan 2024 22:05:34 +0100 Subject: [PATCH 01/41] Add ConnectorManager component which allows for Connectors to listen to triggers and do actions on them. Also add HomeAssistantConfig which stores the configuration for the HomeAssistantConnector --- cookbook/admin.py | 10 +- cookbook/connectors/__init__.py | 0 cookbook/connectors/connector.py | 41 ++++++++ cookbook/connectors/connector_manager.py | 98 +++++++++++++++++++ cookbook/connectors/homeassistant.py | 62 ++++++++++++ cookbook/forms.py | 41 +++++++- ..._alter_storage_path_homeassistantconfig.py | 30 ++++++ ...ing_list_entry_created_enabled_and_more.py | 33 +++++++ cookbook/models.py | 78 ++++++++++----- cookbook/serializer.py | 63 ++++++++---- cookbook/signals.py | 8 +- cookbook/tables.py | 11 ++- cookbook/urls.py | 6 +- cookbook/views/api.py | 10 ++ cookbook/views/delete.py | 25 ++++- cookbook/views/edit.py | 61 ++++++++++-- cookbook/views/lists.py | 30 ++++-- cookbook/views/new.py | 29 +++++- 18 files changed, 566 insertions(+), 70 deletions(-) create mode 100644 cookbook/connectors/__init__.py create mode 100644 cookbook/connectors/connector.py create mode 100644 cookbook/connectors/connector_manager.py create mode 100644 cookbook/connectors/homeassistant.py create mode 100644 cookbook/migrations/0206_alter_storage_path_homeassistantconfig.py create mode 100644 cookbook/migrations/0207_homeassistantconfig_on_shopping_list_entry_created_enabled_and_more.py diff --git a/cookbook/admin.py b/cookbook/admin.py index 82324628..4a2b2697 100644 --- a/cookbook/admin.py +++ b/cookbook/admin.py @@ -16,7 +16,7 @@ from .models import (BookmarkletImport, Comment, CookLog, Food, ImportLog, Ingre ShoppingList, ShoppingListEntry, ShoppingListRecipe, Space, Step, Storage, Supermarket, SupermarketCategory, SupermarketCategoryRelation, Sync, SyncLog, TelegramBot, Unit, UnitConversion, UserFile, UserPreference, UserSpace, - ViewLog) + ViewLog, HomeAssistantConfig) class CustomUserAdmin(UserAdmin): @@ -95,6 +95,14 @@ class StorageAdmin(admin.ModelAdmin): admin.site.register(Storage, StorageAdmin) +class HomeAssistantConfigAdmin(admin.ModelAdmin): + list_display = ('name',) + search_fields = ('name',) + + +admin.site.register(HomeAssistantConfig, HomeAssistantConfigAdmin) + + class SyncAdmin(admin.ModelAdmin): list_display = ('storage', 'path', 'active', 'last_checked') search_fields = ('storage__name', 'path') diff --git a/cookbook/connectors/__init__.py b/cookbook/connectors/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cookbook/connectors/connector.py b/cookbook/connectors/connector.py new file mode 100644 index 00000000..d2a435d7 --- /dev/null +++ b/cookbook/connectors/connector.py @@ -0,0 +1,41 @@ +from abc import ABC, abstractmethod + +from cookbook.models import ShoppingListEntry, Space + + +class Connector(ABC): + @abstractmethod + async def on_shopping_list_entry_created(self, space: Space, instance: ShoppingListEntry) -> None: + pass + + @abstractmethod + async def on_shopping_list_entry_updated(self, space: Space, instance: ShoppingListEntry) -> None: + pass + + @abstractmethod + async def on_shopping_list_entry_deleted(self, space: Space, instance: ShoppingListEntry) -> None: + pass + + # @abstractmethod + # def on_recipe_created(self, instance: Recipe, **kwargs) -> None: + # pass + # + # @abstractmethod + # def on_recipe_updated(self, instance: Recipe, **kwargs) -> None: + # pass + # + # @abstractmethod + # def on_recipe_deleted(self, instance: Recipe, **kwargs) -> None: + # pass + # + # @abstractmethod + # def on_meal_plan_created(self, instance: MealPlan, **kwargs) -> None: + # pass + # + # @abstractmethod + # def on_meal_plan_updated(self, instance: MealPlan, **kwargs) -> None: + # pass + # + # @abstractmethod + # def on_meal_plan_deleted(self, instance: MealPlan, **kwargs) -> None: + # pass diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py new file mode 100644 index 00000000..0d3df8cf --- /dev/null +++ b/cookbook/connectors/connector_manager.py @@ -0,0 +1,98 @@ +import asyncio +from enum import Enum +from types import UnionType +from typing import List, Any, Dict + +from django_scopes import scope + +from cookbook.connectors.connector import Connector +from cookbook.connectors.homeassistant import HomeAssistant +from cookbook.models import ShoppingListEntry, Recipe, MealPlan, Space + + +class ActionType(Enum): + CREATED = 1 + UPDATED = 2 + DELETED = 3 + + +class ConnectorManager: + _connectors: Dict[str, List[Connector]] + _listening_to_classes: UnionType = ShoppingListEntry | Recipe | MealPlan | Connector + max_concurrent_tasks = 2 + + def __init__(self): + self._connectors = dict() + + def __call__(self, instance: Any, **kwargs) -> None: + if not isinstance(instance, self._listening_to_classes): + return + + # If a Connector was changed/updated, refresh connector from the database for said space + purge_connector_cache = isinstance(instance, Connector) + + space: Space = instance.space + if space.name in self._connectors and not purge_connector_cache: + connectors: List[Connector] = self._connectors[space.name] + else: + with scope(space=space): + connectors: List[Connector] = [HomeAssistant(config) for config in space.homeassistantconfig_set.all()] + self._connectors[space.name] = connectors + + if len(connectors) == 0 or purge_connector_cache: + return + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop.run_until_complete(self.run_connectors(connectors, space, instance, **kwargs)) + loop.close() + + @staticmethod + async def run_connectors(connectors: List[Connector], space: Space, instance: Any, **kwargs): + action_type: ActionType + if "created" in kwargs and kwargs["created"]: + action_type = ActionType.CREATED + elif "created" in kwargs and not kwargs["created"]: + action_type = ActionType.UPDATED + elif "origin" in kwargs: + action_type = ActionType.DELETED + else: + return + + tasks: List[asyncio.Task] = list() + + if isinstance(instance, ShoppingListEntry): + shopping_list_entry: ShoppingListEntry = instance + + match action_type: + case ActionType.CREATED: + for connector in connectors: + tasks.append(asyncio.create_task(connector.on_shopping_list_entry_created(space, shopping_list_entry))) + case ActionType.UPDATED: + for connector in connectors: + tasks.append(asyncio.create_task(connector.on_shopping_list_entry_updated(space, shopping_list_entry))) + case ActionType.DELETED: + for connector in connectors: + tasks.append(asyncio.create_task(connector.on_shopping_list_entry_deleted(space, shopping_list_entry))) + + try: + await asyncio.gather(*tasks, return_exceptions=False) + except BaseException as e: + print("received an exception from one of the tasks: ", e) + # if isinstance(instance, Recipe): + # if "created" in kwargs and kwargs["created"]: + # for connector in self._connectors: + # connector.on_recipe_created(instance, **kwargs) + # return + # for connector in self._connectors: + # connector.on_recipe_updated(instance, **kwargs) + # return + # + # if isinstance(instance, MealPlan): + # if "created" in kwargs and kwargs["created"]: + # for connector in self._connectors: + # connector.on_meal_plan_created(instance, **kwargs) + # return + # for connector in self._connectors: + # connector.on_meal_plan_updated(instance, **kwargs) + # return diff --git a/cookbook/connectors/homeassistant.py b/cookbook/connectors/homeassistant.py new file mode 100644 index 00000000..6d504d6c --- /dev/null +++ b/cookbook/connectors/homeassistant.py @@ -0,0 +1,62 @@ +import logging + +from homeassistant_api import Client, HomeassistantAPIError + +from cookbook.connectors.connector import Connector +from cookbook.models import ShoppingListEntry, HomeAssistantConfig, Space + + +class HomeAssistant(Connector): + _config: HomeAssistantConfig + + def __init__(self, config: HomeAssistantConfig): + self._config = config + self._logger = logging.getLogger("connector.HomeAssistant") + + async def on_shopping_list_entry_created(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None: + if not self._config.on_shopping_list_entry_created_enabled: + return + + item, description = _format_shopping_list_entry(shopping_list_entry) + async with Client(self._config.url, self._config.token, use_async=True) as client: + try: + todo_domain = await client.async_get_domain('todo') + await todo_domain.add_item(entity_id=self._config.todo_entity, item=item) + except HomeassistantAPIError as err: + self._logger.warning(f"[HomeAssistant {self._config.name}] Received an exception from the api: {err=}, {type(err)=}") + + async def on_shopping_list_entry_updated(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None: + if not self._config.on_shopping_list_entry_updated_enabled: + return + pass + + async def on_shopping_list_entry_deleted(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None: + if not self._config.on_shopping_list_entry_deleted_enabled: + return + + item, description = _format_shopping_list_entry(shopping_list_entry) + async with Client(self._config.url, self._config.token, use_async=True) as client: + try: + todo_domain = await client.async_get_domain('todo') + await todo_domain.remove_item(entity_id=self._config.todo_entity, item=item) + except HomeassistantAPIError as err: + self._logger.warning(f"[HomeAssistant {self._config.name}] Received an exception from the api: {err=}, {type(err)=}") + + +def _format_shopping_list_entry(shopping_list_entry: ShoppingListEntry): + item = shopping_list_entry.food.name + if shopping_list_entry.amount > 0: + if shopping_list_entry.unit and shopping_list_entry.unit.base_unit and len(shopping_list_entry.unit.base_unit) > 0: + item += f" ({shopping_list_entry.amount} {shopping_list_entry.unit.base_unit})" + elif shopping_list_entry.unit and shopping_list_entry.unit.name and len(shopping_list_entry.unit.name) > 0: + item += f" ({shopping_list_entry.amount} {shopping_list_entry.unit.name})" + else: + item += f" ({shopping_list_entry.amount})" + + description = "Imported by TandoorRecipes" + if shopping_list_entry.created_by.first_name and len(shopping_list_entry.created_by.first_name) > 0: + description += f", created by {shopping_list_entry.created_by.first_name}" + else: + description += f", created by {shopping_list_entry.created_by.username}" + + return item, description diff --git a/cookbook/forms.py b/cookbook/forms.py index dd62aa0b..76d4a17f 100644 --- a/cookbook/forms.py +++ b/cookbook/forms.py @@ -10,7 +10,7 @@ from django_scopes.forms import SafeModelChoiceField, SafeModelMultipleChoiceFie from hcaptcha.fields import hCaptchaField from .models import (Comment, Food, InviteLink, Keyword, Recipe, RecipeBook, RecipeBookEntry, - SearchPreference, Space, Storage, Sync, User, UserPreference) + SearchPreference, Space, Storage, Sync, User, UserPreference, HomeAssistantConfig) class SelectWidget(widgets.Select): @@ -188,6 +188,45 @@ class StorageForm(forms.ModelForm): } +class HomeAssistantConfigForm(forms.ModelForm): + token = forms.CharField( + widget=forms.TextInput( + attrs={'autocomplete': 'new-password', 'type': 'password'} + ), + required=True, + help_text=_('Long Lived Access Token for your HomeAssistant instance') + ) + + url = forms.URLField( + required=True, + help_text=_('Something like http://homeassistant.local:8123/api'), + ) + + on_shopping_list_entry_created_enabled = forms.BooleanField( + help_text="Enable syncing ShoppingListEntry to Homeassistant Todo List -- Warning: Might have negative performance impact", + required=False, + ) + + on_shopping_list_entry_updated_enabled = forms.BooleanField( + help_text="PLACEHOLDER", + required=False, + ) + + on_shopping_list_entry_deleted_enabled = forms.BooleanField( + help_text="Enable syncing ShoppingListEntry deletion to Homeassistant Todo List -- Warning: Might have negative performance impact", + required=False, + ) + + class Meta: + model = HomeAssistantConfig + fields = ( + 'name', 'url', 'token', 'todo_entity', 'on_shopping_list_entry_created_enabled', 'on_shopping_list_entry_updated_enabled', 'on_shopping_list_entry_deleted_enabled') + + help_texts = { + 'url': _('http://homeassistant.local:8123/api for example'), + } + + # TODO: Deprecate class RecipeBookEntryForm(forms.ModelForm): prefix = 'bookmark' diff --git a/cookbook/migrations/0206_alter_storage_path_homeassistantconfig.py b/cookbook/migrations/0206_alter_storage_path_homeassistantconfig.py new file mode 100644 index 00000000..acbec237 --- /dev/null +++ b/cookbook/migrations/0206_alter_storage_path_homeassistantconfig.py @@ -0,0 +1,30 @@ +# Generated by Django 4.2.7 on 2024-01-10 21:28 + +import cookbook.models +from django.conf import settings +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('cookbook', '0205_alter_food_fdc_id_alter_propertytype_fdc_id'), + ] + + operations = [ + migrations.CreateModel( + name='HomeAssistantConfig', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=128, validators=[django.core.validators.MinLengthValidator(1)])), + ('url', models.URLField(blank=True, help_text='Something like http://homeassistant:8123/api')), + ('token', models.CharField(blank=True, max_length=512)), + ('todo_entity', models.CharField(default='todo.shopping_list', max_length=128)), + ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)), + ('space', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='cookbook.space')), + ], + bases=(models.Model, cookbook.models.PermissionModelMixin), + ), + ] diff --git a/cookbook/migrations/0207_homeassistantconfig_on_shopping_list_entry_created_enabled_and_more.py b/cookbook/migrations/0207_homeassistantconfig_on_shopping_list_entry_created_enabled_and_more.py new file mode 100644 index 00000000..67f93b18 --- /dev/null +++ b/cookbook/migrations/0207_homeassistantconfig_on_shopping_list_entry_created_enabled_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 4.2.7 on 2024-01-11 19:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('cookbook', '0206_alter_storage_path_homeassistantconfig'), + ] + + operations = [ + migrations.AddField( + model_name='homeassistantconfig', + name='on_shopping_list_entry_created_enabled', + field=models.BooleanField(default=False, help_text='Enable syncing ShoppingListEntry to Homeassistant Todo List'), + ), + migrations.AddField( + model_name='homeassistantconfig', + name='on_shopping_list_entry_deleted_enabled', + field=models.BooleanField(default=False, help_text='Enable syncing ShoppingListEntry deletion to Homeassistant Todo List'), + ), + migrations.AddField( + model_name='homeassistantconfig', + name='on_shopping_list_entry_updated_enabled', + field=models.BooleanField(default=False, help_text='PLACEHOLDER'), + ), + migrations.AlterField( + model_name='homeassistantconfig', + name='url', + field=models.URLField(blank=True), + ), + ] diff --git a/cookbook/models.py b/cookbook/models.py index 0614c974..bb9eb716 100644 --- a/cookbook/models.py +++ b/cookbook/models.py @@ -49,14 +49,16 @@ def get_active_space(self): def get_shopping_share(self): # get list of users that shared shopping list with user. Django ORM forbids this type of query, so raw is required - return User.objects.raw(' '.join([ - 'SELECT auth_user.id FROM auth_user', - 'INNER JOIN cookbook_userpreference', - 'ON (auth_user.id = cookbook_userpreference.user_id)', - 'INNER JOIN cookbook_userpreference_shopping_share', - 'ON (cookbook_userpreference.user_id = cookbook_userpreference_shopping_share.userpreference_id)', - 'WHERE cookbook_userpreference_shopping_share.user_id ={}'.format(self.id) - ])) + return User.objects.raw( + ' '.join( + [ + 'SELECT auth_user.id FROM auth_user', + 'INNER JOIN cookbook_userpreference', + 'ON (auth_user.id = cookbook_userpreference.user_id)', + 'INNER JOIN cookbook_userpreference_shopping_share', + 'ON (cookbook_userpreference.user_id = cookbook_userpreference_shopping_share.userpreference_id)', + 'WHERE cookbook_userpreference_shopping_share.user_id ={}'.format(self.id) + ])) auth.models.User.add_to_class('get_user_display_name', get_user_display_name) @@ -339,6 +341,7 @@ class Space(ExportModelOperationsMixin('space'), models.Model): SyncLog.objects.filter(sync__space=self).delete() Sync.objects.filter(space=self).delete() Storage.objects.filter(space=self).delete() + HomeAssistantConfig.objects.filter(space=self).delete() ShoppingListEntry.objects.filter(shoppinglist__space=self).delete() ShoppingListRecipe.objects.filter(shoppinglist__space=self).delete() @@ -363,6 +366,24 @@ class Space(ExportModelOperationsMixin('space'), models.Model): return self.name +class HomeAssistantConfig(models.Model, PermissionModelMixin): + name = models.CharField(max_length=128, validators=[MinLengthValidator(1)]) + + url = models.URLField(blank=True) + token = models.CharField(max_length=512, blank=True) + + todo_entity = models.CharField(max_length=128, default='todo.shopping_list') + + on_shopping_list_entry_created_enabled = models.BooleanField(default=False, help_text="Enable syncing ShoppingListEntry to Homeassistant Todo List") + on_shopping_list_entry_updated_enabled = models.BooleanField(default=False, help_text="PLACEHOLDER") + on_shopping_list_entry_deleted_enabled = models.BooleanField(default=False, help_text="Enable syncing ShoppingListEntry deletion to Homeassistant Todo List") + + created_by = models.ForeignKey(User, on_delete=models.PROTECT) + + space = models.ForeignKey(Space, on_delete=models.CASCADE) + objects = ScopedManager(space='space') + + class UserPreference(models.Model, PermissionModelMixin): # Themes BOOTSTRAP = 'BOOTSTRAP' @@ -674,10 +695,11 @@ class Food(ExportModelOperationsMixin('food'), TreeModel, PermissionModelMixin): if len(inherit) > 0: # ManyToMany cannot be updated through an UPDATE operation for i in inherit: - trough.objects.bulk_create([ - trough(food_id=x, foodinheritfield_id=i['id']) - for x in Food.objects.filter(tree_filter).values_list('id', flat=True) - ]) + trough.objects.bulk_create( + [ + trough(food_id=x, foodinheritfield_id=i['id']) + for x in Food.objects.filter(tree_filter).values_list('id', flat=True) + ]) inherit = [x['field'] for x in inherit] for field in ['ignore_shopping', 'substitute_children', 'substitute_siblings']: @@ -804,8 +826,9 @@ class PropertyType(models.Model, PermissionModelMixin): unit = models.CharField(max_length=64, blank=True, null=True) order = models.IntegerField(default=0) description = models.CharField(max_length=512, blank=True, null=True) - category = models.CharField(max_length=64, choices=((NUTRITION, _('Nutrition')), (ALLERGEN, _('Allergen')), - (PRICE, _('Price')), (GOAL, _('Goal')), (OTHER, _('Other'))), null=True, blank=True) + category = models.CharField( + max_length=64, choices=((NUTRITION, _('Nutrition')), (ALLERGEN, _('Allergen')), + (PRICE, _('Price')), (GOAL, _('Goal')), (OTHER, _('Other'))), null=True, blank=True) open_data_slug = models.CharField(max_length=128, null=True, blank=True, default=None) fdc_id = models.IntegerField(null=True, default=None, blank=True) @@ -1368,19 +1391,20 @@ class Automation(ExportModelOperationsMixin('automations'), models.Model, Permis UNIT_REPLACE = 'UNIT_REPLACE' NAME_REPLACE = 'NAME_REPLACE' - type = models.CharField(max_length=128, - choices=( - (FOOD_ALIAS, _('Food Alias')), - (UNIT_ALIAS, _('Unit Alias')), - (KEYWORD_ALIAS, _('Keyword Alias')), - (DESCRIPTION_REPLACE, _('Description Replace')), - (INSTRUCTION_REPLACE, _('Instruction Replace')), - (NEVER_UNIT, _('Never Unit')), - (TRANSPOSE_WORDS, _('Transpose Words')), - (FOOD_REPLACE, _('Food Replace')), - (UNIT_REPLACE, _('Unit Replace')), - (NAME_REPLACE, _('Name Replace')), - )) + type = models.CharField( + max_length=128, + choices=( + (FOOD_ALIAS, _('Food Alias')), + (UNIT_ALIAS, _('Unit Alias')), + (KEYWORD_ALIAS, _('Keyword Alias')), + (DESCRIPTION_REPLACE, _('Description Replace')), + (INSTRUCTION_REPLACE, _('Instruction Replace')), + (NEVER_UNIT, _('Never Unit')), + (TRANSPOSE_WORDS, _('Transpose Words')), + (FOOD_REPLACE, _('Food Replace')), + (UNIT_REPLACE, _('Unit Replace')), + (NAME_REPLACE, _('Name Replace')), + )) name = models.CharField(max_length=128, default='') description = models.TextField(blank=True, null=True) diff --git a/cookbook/serializer.py b/cookbook/serializer.py index d4f7bd47..8e338c17 100644 --- a/cookbook/serializer.py +++ b/cookbook/serializer.py @@ -34,7 +34,7 @@ from cookbook.models import (Automation, BookmarkletImport, Comment, CookLog, Cu ShareLink, ShoppingList, ShoppingListEntry, ShoppingListRecipe, Space, Step, Storage, Supermarket, SupermarketCategory, SupermarketCategoryRelation, Sync, SyncLog, Unit, UnitConversion, - UserFile, UserPreference, UserSpace, ViewLog) + UserFile, UserPreference, UserSpace, ViewLog, HomeAssistantConfig) from cookbook.templatetags.custom_tags import markdown from recipes.settings import AWS_ENABLED, MEDIA_URL @@ -413,6 +413,27 @@ class StorageSerializer(SpacedModelSerializer): } +class HomeAssistantConfigSerializer(SpacedModelSerializer): + + def create(self, validated_data): + validated_data['created_by'] = self.context['request'].user + return super().create(validated_data) + + class Meta: + model = HomeAssistantConfig + fields = ( + 'id', 'name', 'url', 'token', 'todo_entity', + 'on_shopping_list_entry_created_enabled', 'on_shopping_list_entry_updated_enabled', + 'on_shopping_list_entry_deleted_enabled', 'created_by' + ) + + read_only_fields = ('created_by',) + + extra_kwargs = { + 'token': {'write_only': True}, + } + + class SyncSerializer(SpacedModelSerializer): class Meta: model = Sync @@ -665,8 +686,9 @@ class FoodSerializer(UniqueFieldsMixin, WritableNestedModelSerializer, ExtendedR properties = validated_data.pop('properties', None) - obj, created = Food.objects.get_or_create(name=name, plural_name=plural_name, space=space, properties_food_unit=properties_food_unit, - defaults=validated_data) + obj, created = Food.objects.get_or_create( + name=name, plural_name=plural_name, space=space, properties_food_unit=properties_food_unit, + defaults=validated_data) if properties and len(properties) > 0: for p in properties: @@ -1254,8 +1276,9 @@ class InviteLinkSerializer(WritableNestedModelSerializer): if obj.email: try: - if InviteLink.objects.filter(space=self.context['request'].space, - created_at__gte=datetime.now() - timedelta(hours=4)).count() < 20: + if InviteLink.objects.filter( + space=self.context['request'].space, + created_at__gte=datetime.now() - timedelta(hours=4)).count() < 20: message = _('Hello') + '!\n\n' + _('You have been invited by ') + escape( self.context['request'].user.get_user_display_name()) message += _(' to join their Tandoor Recipes space ') + escape( @@ -1410,12 +1433,15 @@ class RecipeExportSerializer(WritableNestedModelSerializer): class RecipeShoppingUpdateSerializer(serializers.ModelSerializer): - list_recipe = serializers.IntegerField(write_only=True, allow_null=True, required=False, - help_text=_("Existing shopping list to update")) - ingredients = serializers.IntegerField(write_only=True, allow_null=True, required=False, help_text=_( - "List of ingredient IDs from the recipe to add, if not provided all ingredients will be added.")) - servings = serializers.IntegerField(default=1, write_only=True, allow_null=True, required=False, help_text=_( - "Providing a list_recipe ID and servings of 0 will delete that shopping list.")) + list_recipe = serializers.IntegerField( + write_only=True, allow_null=True, required=False, + help_text=_("Existing shopping list to update")) + ingredients = serializers.IntegerField( + write_only=True, allow_null=True, required=False, help_text=_( + "List of ingredient IDs from the recipe to add, if not provided all ingredients will be added.")) + servings = serializers.IntegerField( + default=1, write_only=True, allow_null=True, required=False, help_text=_( + "Providing a list_recipe ID and servings of 0 will delete that shopping list.")) class Meta: model = Recipe @@ -1423,12 +1449,15 @@ class RecipeShoppingUpdateSerializer(serializers.ModelSerializer): class FoodShoppingUpdateSerializer(serializers.ModelSerializer): - amount = serializers.IntegerField(write_only=True, allow_null=True, required=False, - help_text=_("Amount of food to add to the shopping list")) - unit = serializers.IntegerField(write_only=True, allow_null=True, required=False, - help_text=_("ID of unit to use for the shopping list")) - delete = serializers.ChoiceField(choices=['true'], write_only=True, allow_null=True, allow_blank=True, - help_text=_("When set to true will delete all food from active shopping lists.")) + amount = serializers.IntegerField( + write_only=True, allow_null=True, required=False, + help_text=_("Amount of food to add to the shopping list")) + unit = serializers.IntegerField( + write_only=True, allow_null=True, required=False, + help_text=_("ID of unit to use for the shopping list")) + delete = serializers.ChoiceField( + choices=['true'], write_only=True, allow_null=True, allow_blank=True, + help_text=_("When set to true will delete all food from active shopping lists.")) class Meta: model = Recipe diff --git a/cookbook/signals.py b/cookbook/signals.py index a93ffba1..2d94b1d0 100644 --- a/cookbook/signals.py +++ b/cookbook/signals.py @@ -4,11 +4,12 @@ from django.conf import settings from django.contrib.auth.models import User from django.contrib.postgres.search import SearchVector from django.core.cache import caches -from django.db.models.signals import post_save +from django.db.models.signals import post_save, post_delete from django.dispatch import receiver from django.utils import translation from django_scopes import scope, scopes_disabled +from cookbook.connectors.connector_manager import ConnectorManager from cookbook.helper.cache_helper import CacheHelper from cookbook.helper.shopping_helper import RecipeShoppingEditor from cookbook.managers import DICTIONARY @@ -161,3 +162,8 @@ def clear_unit_cache(sender, instance=None, created=False, **kwargs): def clear_property_type_cache(sender, instance=None, created=False, **kwargs): if instance: caches['default'].delete(CacheHelper(instance.space).PROPERTY_TYPE_CACHE_KEY) + + +handler = ConnectorManager() +post_save.connect(handler, dispatch_uid="connector_manager") +post_delete.connect(handler, dispatch_uid="connector_manager") diff --git a/cookbook/tables.py b/cookbook/tables.py index 6392f791..8ffe2a53 100644 --- a/cookbook/tables.py +++ b/cookbook/tables.py @@ -3,7 +3,7 @@ from django.utils.html import format_html from django.utils.translation import gettext as _ from django_tables2.utils import A -from .models import CookLog, InviteLink, RecipeImport, Storage, Sync, SyncLog, ViewLog +from .models import CookLog, InviteLink, RecipeImport, Storage, Sync, SyncLog, ViewLog, HomeAssistantConfig class StorageTable(tables.Table): @@ -15,6 +15,15 @@ class StorageTable(tables.Table): fields = ('id', 'name', 'method') +class HomeAssistantConfigTable(tables.Table): + id = tables.LinkColumn('edit_home_assistant_config', args=[A('id')]) + + class Meta: + model = HomeAssistantConfig + template_name = 'generic/table_template.html' + fields = ('id', 'name', 'url') + + class ImportLogTable(tables.Table): sync_id = tables.LinkColumn('edit_sync', args=[A('sync_id')]) diff --git a/cookbook/urls.py b/cookbook/urls.py index 9566541a..96735037 100644 --- a/cookbook/urls.py +++ b/cookbook/urls.py @@ -12,7 +12,7 @@ from recipes.settings import DEBUG, PLUGINS from .models import (Automation, Comment, CustomFilter, Food, InviteLink, Keyword, PropertyType, Recipe, RecipeBook, RecipeBookEntry, RecipeImport, ShoppingList, Space, Step, Storage, Supermarket, SupermarketCategory, Sync, SyncLog, Unit, UnitConversion, - UserFile, UserSpace, get_model_name) + UserFile, UserSpace, get_model_name, HomeAssistantConfig) from .views import api, data, delete, edit, import_export, lists, new, telegram, views from .views.api import CustomAuthToken, ImportOpenData @@ -51,6 +51,7 @@ router.register(r'shopping-list-recipe', api.ShoppingListRecipeViewSet) router.register(r'space', api.SpaceViewSet) router.register(r'step', api.StepViewSet) router.register(r'storage', api.StorageViewSet) +router.register(r'home-assistant-config', api.HomeAssistantConfigViewSet) router.register(r'supermarket', api.SupermarketViewSet) router.register(r'supermarket-category', api.SupermarketCategoryViewSet) router.register(r'supermarket-category-relation', api.SupermarketCategoryRelationViewSet) @@ -114,6 +115,7 @@ urlpatterns = [ path('edit/recipe/convert//', edit.convert_recipe, name='edit_convert_recipe'), path('edit/storage//', edit.edit_storage, name='edit_storage'), + path('edit/home-assistant-config//', edit.edit_home_assistant_config, name='edit_home_assistant_config'), path('delete/recipe-source//', delete.delete_recipe_source, name='delete_recipe_source'), @@ -166,7 +168,7 @@ urlpatterns = [ ] generic_models = ( - Recipe, RecipeImport, Storage, RecipeBook, SyncLog, Sync, + Recipe, RecipeImport, Storage, HomeAssistantConfig, RecipeBook, SyncLog, Sync, Comment, RecipeBookEntry, ShoppingList, InviteLink, UserSpace, Space ) diff --git a/cookbook/views/api.py b/cookbook/views/api.py index 98b4d9bc..b5f16568 100644 --- a/cookbook/views/api.py +++ b/cookbook/views/api.py @@ -460,6 +460,16 @@ class StorageViewSet(viewsets.ModelViewSet): return self.queryset.filter(space=self.request.space) +class HomeAssistantConfigViewSet(viewsets.ModelViewSet): + # TODO handle delete protect error and adjust test + queryset = HomeAssistantConfig.objects + serializer_class = HomeAssistantConfigSerializer + permission_classes = [CustomIsAdmin & CustomTokenHasReadWriteScope] + + def get_queryset(self): + return self.queryset.filter(space=self.request.space) + + class SyncViewSet(viewsets.ModelViewSet): queryset = Sync.objects serializer_class = SyncSerializer diff --git a/cookbook/views/delete.py b/cookbook/views/delete.py index 411eb323..b6cf857b 100644 --- a/cookbook/views/delete.py +++ b/cookbook/views/delete.py @@ -9,7 +9,7 @@ from django.views.generic import DeleteView from cookbook.helper.permission_helper import GroupRequiredMixin, OwnerRequiredMixin, group_required from cookbook.models import (Comment, InviteLink, MealPlan, Recipe, RecipeBook, RecipeBookEntry, - RecipeImport, Space, Storage, Sync, UserSpace) + RecipeImport, Space, Storage, Sync, UserSpace, HomeAssistantConfig) from cookbook.provider.dropbox import Dropbox from cookbook.provider.local import Local from cookbook.provider.nextcloud import Nextcloud @@ -122,6 +122,29 @@ class StorageDelete(GroupRequiredMixin, DeleteView): return HttpResponseRedirect(reverse('list_storage')) +class HomeAssistantConfigDelete(GroupRequiredMixin, DeleteView): + groups_required = ['admin'] + template_name = "generic/delete_template.html" + model = HomeAssistantConfig + success_url = reverse_lazy('list_storage') + + def get_context_data(self, **kwargs): + context = super(HomeAssistantConfigDelete, self).get_context_data(**kwargs) + context['title'] = _("HomeAssistant Config Backend") + return context + + def post(self, request, *args, **kwargs): + try: + return self.delete(request, *args, **kwargs) + except ProtectedError: + messages.add_message( + request, + messages.WARNING, + _('Could not delete this storage backend as it is used in at least one monitor.') # noqa: E501 + ) + return HttpResponseRedirect(reverse('list_storage')) + + class CommentDelete(OwnerRequiredMixin, DeleteView): template_name = "generic/delete_template.html" model = Comment diff --git a/cookbook/views/edit.py b/cookbook/views/edit.py index 1726b2b5..ce4aac27 100644 --- a/cookbook/views/edit.py +++ b/cookbook/views/edit.py @@ -1,3 +1,4 @@ +import copy import os from django.contrib import messages @@ -8,15 +9,17 @@ from django.utils.translation import gettext as _ from django.views.generic import UpdateView from django.views.generic.edit import FormMixin -from cookbook.forms import CommentForm, ExternalRecipeForm, StorageForm, SyncForm +from cookbook.forms import CommentForm, ExternalRecipeForm, StorageForm, SyncForm, HomeAssistantConfigForm from cookbook.helper.permission_helper import (GroupRequiredMixin, OwnerRequiredMixin, above_space_limit, group_required) -from cookbook.models import Comment, Recipe, RecipeImport, Storage, Sync +from cookbook.models import Comment, Recipe, RecipeImport, Storage, Sync, HomeAssistantConfig from cookbook.provider.dropbox import Dropbox from cookbook.provider.local import Local from cookbook.provider.nextcloud import Nextcloud from recipes import settings +VALUE_NOT_CHANGED = '__NO__CHANGE__' + @group_required('guest') def switch_recipe(request, pk): @@ -76,7 +79,7 @@ class SyncUpdate(GroupRequiredMixin, UpdateView, SpaceFormMixing): @group_required('admin') def edit_storage(request, pk): - instance = get_object_or_404(Storage, pk=pk, space=request.space) + instance: Storage = get_object_or_404(Storage, pk=pk, space=request.space) if not (instance.created_by == request.user or request.user.is_superuser): messages.add_message(request, messages.ERROR, _('You cannot edit this storage!')) @@ -87,17 +90,18 @@ def edit_storage(request, pk): return redirect('index') if request.method == "POST": - form = StorageForm(request.POST, instance=instance) + form = StorageForm(request.POST, instance=copy.deepcopy(instance)) if form.is_valid(): instance.name = form.cleaned_data['name'] instance.method = form.cleaned_data['method'] instance.username = form.cleaned_data['username'] instance.url = form.cleaned_data['url'] + instance.path = form.cleaned_data['path'] - if form.cleaned_data['password'] != '__NO__CHANGE__': + if form.cleaned_data['password'] != VALUE_NOT_CHANGED: instance.password = form.cleaned_data['password'] - if form.cleaned_data['token'] != '__NO__CHANGE__': + if form.cleaned_data['token'] != VALUE_NOT_CHANGED: instance.token = form.cleaned_data['token'] instance.save() @@ -113,8 +117,8 @@ def edit_storage(request, pk): ) else: pseudo_instance = instance - pseudo_instance.password = '__NO__CHANGE__' - pseudo_instance.token = '__NO__CHANGE__' + pseudo_instance.password = VALUE_NOT_CHANGED + pseudo_instance.token = VALUE_NOT_CHANGED form = StorageForm(instance=pseudo_instance) return render( @@ -124,6 +128,47 @@ def edit_storage(request, pk): ) +@group_required('admin') +def edit_home_assistant_config(request, pk): + instance: HomeAssistantConfig = get_object_or_404(HomeAssistantConfig, pk=pk, space=request.space) + + if not (instance.created_by == request.user or request.user.is_superuser): + messages.add_message(request, messages.ERROR, _('You cannot edit this homeassistant config!')) + return HttpResponseRedirect(reverse('edit_home_assistant_config')) + + if request.space.demo or settings.HOSTED: + messages.add_message(request, messages.ERROR, _('This feature is not yet available in the hosted version of tandoor!')) + return redirect('index') + + if request.method == "POST": + form = HomeAssistantConfigForm(request.POST, instance=copy.deepcopy(instance)) + if form.is_valid(): + instance.name = form.cleaned_data['name'] + instance.url = form.cleaned_data['url'] + instance.todo_entity = form.cleaned_data['todo_entity'] + instance.on_shopping_list_entry_created_enabled = form.cleaned_data['on_shopping_list_entry_created_enabled'] + instance.on_shopping_list_entry_updated_enabled = form.cleaned_data['on_shopping_list_entry_updated_enabled'] + instance.on_shopping_list_entry_deleted_enabled = form.cleaned_data['on_shopping_list_entry_deleted_enabled'] + + if form.cleaned_data['token'] != VALUE_NOT_CHANGED: + instance.token = form.cleaned_data['token'] + + instance.save() + + messages.add_message(request, messages.SUCCESS, _('HomeAssistant config saved!')) + else: + messages.add_message(request, messages.ERROR, _('There was an error updating this config!')) + else: + instance.token = VALUE_NOT_CHANGED + form = HomeAssistantConfigForm(instance=instance) + + return render( + request, + 'generic/edit_template.html', + {'form': form, 'title': _('HomeAssistantConfig')} + ) + + class CommentUpdate(OwnerRequiredMixin, UpdateView): template_name = "generic/edit_template.html" model = Comment diff --git a/cookbook/views/lists.py b/cookbook/views/lists.py index b9ef9cc7..20dbb73c 100644 --- a/cookbook/views/lists.py +++ b/cookbook/views/lists.py @@ -6,8 +6,8 @@ from django.utils.translation import gettext as _ from django_tables2 import RequestConfig from cookbook.helper.permission_helper import group_required -from cookbook.models import InviteLink, RecipeImport, Storage, SyncLog, UserFile -from cookbook.tables import ImportLogTable, InviteLinkTable, RecipeImportTable, StorageTable +from cookbook.models import InviteLink, RecipeImport, Storage, SyncLog, UserFile, HomeAssistantConfig +from cookbook.tables import ImportLogTable, InviteLinkTable, RecipeImportTable, StorageTable, HomeAssistantConfigTable @group_required('admin') @@ -65,17 +65,31 @@ def storage(request): ) +@group_required('admin') +def home_assistant_config(request): + table = HomeAssistantConfigTable(HomeAssistantConfig.objects.filter(space=request.space).all()) + RequestConfig(request, paginate={'per_page': 25}).configure(table) + + return render( + request, 'generic/list_template.html', { + 'title': _("HomeAssistant Config Backend"), + 'table': table, + 'create_url': 'new_home_assistant_config' + }) + + @group_required('admin') def invite_link(request): table = InviteLinkTable( InviteLink.objects.filter(valid_until__gte=datetime.today(), used_by=None, space=request.space).all()) RequestConfig(request, paginate={'per_page': 25}).configure(table) - return render(request, 'generic/list_template.html', { - 'title': _("Invite Links"), - 'table': table, - 'create_url': 'new_invite_link' - }) + return render( + request, 'generic/list_template.html', { + 'title': _("Invite Links"), + 'table': table, + 'create_url': 'new_invite_link' + }) @group_required('user') @@ -195,7 +209,7 @@ def custom_filter(request): def user_file(request): try: current_file_size_mb = UserFile.objects.filter(space=request.space).aggregate(Sum('file_size_kb'))[ - 'file_size_kb__sum'] / 1000 + 'file_size_kb__sum'] / 1000 except TypeError: current_file_size_mb = 0 diff --git a/cookbook/views/new.py b/cookbook/views/new.py index d4368f67..263dca09 100644 --- a/cookbook/views/new.py +++ b/cookbook/views/new.py @@ -1,4 +1,3 @@ - from django.contrib import messages from django.http import HttpResponseRedirect from django.shortcuts import get_object_or_404, redirect, render @@ -6,9 +5,9 @@ from django.urls import reverse, reverse_lazy from django.utils.translation import gettext as _ from django.views.generic import CreateView -from cookbook.forms import ImportRecipeForm, Storage, StorageForm +from cookbook.forms import ImportRecipeForm, Storage, StorageForm, HomeAssistantConfigForm from cookbook.helper.permission_helper import GroupRequiredMixin, above_space_limit, group_required -from cookbook.models import Recipe, RecipeImport, ShareLink, Step +from cookbook.models import Recipe, RecipeImport, ShareLink, Step, HomeAssistantConfig from recipes import settings @@ -71,6 +70,30 @@ class StorageCreate(GroupRequiredMixin, CreateView): return context +class HomeAssistantConfigCreate(GroupRequiredMixin, CreateView): + groups_required = ['admin'] + template_name = "generic/new_template.html" + model = HomeAssistantConfig + form_class = HomeAssistantConfigForm + success_url = reverse_lazy('list_home_assistant_config') + + def form_valid(self, form): + if self.request.space.demo or settings.HOSTED: + messages.add_message(self.request, messages.ERROR, _('This feature is not yet available in the hosted version of tandoor!')) + return redirect('index') + + obj = form.save(commit=False) + obj.created_by = self.request.user + obj.space = self.request.space + obj.save() + return HttpResponseRedirect(reverse('edit_home_assistant_config', kwargs={'pk': obj.pk})) + + def get_context_data(self, **kwargs): + context = super(HomeAssistantConfigCreate, self).get_context_data(**kwargs) + context['title'] = _("HomeAssistant Config Backend") + return context + + @group_required('user') def create_new_external_recipe(request, import_id): if request.method == "POST": From bf0462cd74ed1166cdbe3e3dca48743528e24eb8 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Thu, 11 Jan 2024 22:14:22 +0100 Subject: [PATCH 02/41] add missing from rebase --- cookbook/views/api.py | 216 ++++++++++++++++++++++++------------------ 1 file changed, 125 insertions(+), 91 deletions(-) diff --git a/cookbook/views/api.py b/cookbook/views/api.py index b5f16568..d2cf65f8 100644 --- a/cookbook/views/api.py +++ b/cookbook/views/api.py @@ -76,7 +76,7 @@ from cookbook.models import (Automation, BookmarkletImport, CookLog, CustomFilte ShoppingListEntry, ShoppingListRecipe, Space, Step, Storage, Supermarket, SupermarketCategory, SupermarketCategoryRelation, Sync, SyncLog, Unit, UnitConversion, UserFile, UserPreference, UserSpace, - ViewLog) + ViewLog, HomeAssistantConfig) from cookbook.provider.dropbox import Dropbox from cookbook.provider.local import Local from cookbook.provider.nextcloud import Nextcloud @@ -102,7 +102,7 @@ from cookbook.serializer import (AccessTokenSerializer, AutomationSerializer, SupermarketCategorySerializer, SupermarketSerializer, SyncLogSerializer, SyncSerializer, UnitConversionSerializer, UnitSerializer, UserFileSerializer, UserPreferenceSerializer, - UserSerializer, UserSpaceSerializer, ViewLogSerializer) + UserSerializer, UserSpaceSerializer, ViewLogSerializer, HomeAssistantConfigSerializer) from cookbook.views.import_export import get_integration from recipes import settings from recipes.settings import FDC_API_KEY, DRF_THROTTLE_RECIPE_URL_IMPORT @@ -181,9 +181,10 @@ class FuzzyFilterMixin(ViewSetMixin, ExtendedRecipeMixin): self.queryset = self.queryset.filter(space=self.request.space).order_by(Lower('name').asc()) query = self.request.query_params.get('query', None) if self.request.user.is_authenticated: - fuzzy = self.request.user.searchpreference.lookup or any([self.model.__name__.lower() in x for x in - self.request.user.searchpreference.trigram.values_list( - 'field', flat=True)]) + fuzzy = self.request.user.searchpreference.lookup or any( + [self.model.__name__.lower() in x for x in + self.request.user.searchpreference.trigram.values_list( + 'field', flat=True)]) else: fuzzy = True @@ -203,8 +204,10 @@ class FuzzyFilterMixin(ViewSetMixin, ExtendedRecipeMixin): filter |= Q(name__unaccent__icontains=query) self.queryset = ( - self.queryset.annotate(starts=Case(When(name__istartswith=query, then=(Value(100))), - default=Value(0))) # put exact matches at the top of the result set + self.queryset.annotate( + starts=Case( + When(name__istartswith=query, then=(Value(100))), + default=Value(0))) # put exact matches at the top of the result set .filter(filter).order_by('-starts', Lower('name').asc()) ) @@ -326,8 +329,9 @@ class TreeMixin(MergeMixin, FuzzyFilterMixin, ExtendedRecipeMixin): return self.annotate_recipe(queryset=super().get_queryset(), request=self.request, serializer=self.serializer_class, tree=True) self.queryset = self.queryset.filter(space=self.request.space).order_by(Lower('name').asc()) - return self.annotate_recipe(queryset=self.queryset, request=self.request, serializer=self.serializer_class, - tree=True) + return self.annotate_recipe( + queryset=self.queryset, request=self.request, serializer=self.serializer_class, + tree=True) @decorators.action(detail=True, url_path='move/(?P[^/.]+)', methods=['PUT'], ) @decorators.renderer_classes((TemplateHTMLRenderer, JSONRenderer)) @@ -572,8 +576,9 @@ class FoodViewSet(viewsets.ModelViewSet, TreeMixin): pass self.queryset = super().get_queryset() - shopping_status = ShoppingListEntry.objects.filter(space=self.request.space, food=OuterRef('id'), - checked=False).values('id') + shopping_status = ShoppingListEntry.objects.filter( + space=self.request.space, food=OuterRef('id'), + checked=False).values('id') # onhand_status = self.queryset.annotate(onhand_status=Exists(onhand_users_set__in=[shared_users])) return self.queryset \ .annotate(shopping_status=Exists(shopping_status)) \ @@ -594,8 +599,9 @@ class FoodViewSet(viewsets.ModelViewSet, TreeMixin): shared_users = list(self.request.user.get_shopping_share()) shared_users.append(request.user) if request.data.get('_delete', False) == 'true': - ShoppingListEntry.objects.filter(food=obj, checked=False, space=request.space, - created_by__in=shared_users).delete() + ShoppingListEntry.objects.filter( + food=obj, checked=False, space=request.space, + created_by__in=shared_users).delete() content = {'msg': _(f'{obj.name} was removed from the shopping list.')} return Response(content, status=status.HTTP_204_NO_CONTENT) @@ -603,8 +609,9 @@ class FoodViewSet(viewsets.ModelViewSet, TreeMixin): unit = request.data.get('unit', None) content = {'msg': _(f'{obj.name} was added to the shopping list.')} - ShoppingListEntry.objects.create(food=obj, amount=amount, unit=unit, space=request.space, - created_by=request.user) + ShoppingListEntry.objects.create( + food=obj, amount=amount, unit=unit, space=request.space, + created_by=request.user) return Response(content, status=status.HTTP_204_NO_CONTENT) @decorators.action(detail=True, methods=['POST'], ) @@ -617,8 +624,11 @@ class FoodViewSet(viewsets.ModelViewSet, TreeMixin): response = requests.get(f'https://api.nal.usda.gov/fdc/v1/food/{food.fdc_id}?api_key={FDC_API_KEY}') if response.status_code == 429: - return JsonResponse({'msg', 'API Key Rate Limit reached/exceeded, see https://api.data.gov/docs/rate-limits/ for more information. Configure your key in Tandoor using environment FDC_API_KEY variable.'}, status=429, - json_dumps_params={'indent': 4}) + return JsonResponse( + {'msg', + 'API Key Rate Limit reached/exceeded, see https://api.data.gov/docs/rate-limits/ for more information. Configure your key in Tandoor using environment FDC_API_KEY variable.'}, + status=429, + json_dumps_params={'indent': 4}) try: data = json.loads(response.content) @@ -634,12 +644,13 @@ class FoodViewSet(viewsets.ModelViewSet, TreeMixin): if pt.fdc_id: for fn in data['foodNutrients']: if fn['nutrient']['id'] == pt.fdc_id: - food_property_list.append(Property( - property_type_id=pt.id, - property_amount=round(fn['amount'], 2), - import_food_id=food.id, - space=self.request.space, - )) + food_property_list.append( + Property( + property_type_id=pt.id, + property_amount=round(fn['amount'], 2), + import_food_id=food.id, + space=self.request.space, + )) Property.objects.bulk_create(food_property_list, ignore_conflicts=True, unique_fields=('space', 'import_food_id', 'property_type',)) @@ -874,12 +885,14 @@ class RecipePagination(PageNumberPagination): return super().paginate_queryset(queryset, request, view) def get_paginated_response(self, data): - return Response(OrderedDict([ - ('count', self.page.paginator.count), - ('next', self.get_next_link()), - ('previous', self.get_previous_link()), - ('results', data), - ])) + return Response( + OrderedDict( + [ + ('count', self.page.paginator.count), + ('next', self.get_next_link()), + ('previous', self.get_previous_link()), + ('results', data), + ])) class RecipeViewSet(viewsets.ModelViewSet): @@ -965,9 +978,10 @@ class RecipeViewSet(viewsets.ModelViewSet): def list(self, request, *args, **kwargs): if self.request.GET.get('debug', False): - return JsonResponse({ - 'new': str(self.get_queryset().query), - }) + return JsonResponse( + { + 'new': str(self.get_queryset().query), + }) return super().list(request, *args, **kwargs) def get_serializer_class(self): @@ -1137,8 +1151,10 @@ class ShoppingListEntryViewSet(viewsets.ModelViewSet): permission_classes = [(CustomIsOwner | CustomIsShared) & CustomTokenHasReadWriteScope] query_params = [ QueryParam(name='id', description=_('Returns the shopping list entry with a primary key of id. Multiple values allowed.'), qtype='int'), - QueryParam(name='checked', description=_('Filter shopping list entries on checked. [''true'', ''false'', ''both'', ''recent'']
- ''recent'' includes unchecked items and recently completed items.') - ), + QueryParam( + name='checked', description=_( + 'Filter shopping list entries on checked. [''true'', ''false'', ''both'', ''recent'']
- ''recent'' includes unchecked items and recently completed items.') + ), QueryParam(name='supermarket', description=_('Returns the shopping list entries sorted by supermarket category order.'), qtype='int'), ] schema = QueryParamAutoSchema() @@ -1329,25 +1345,28 @@ class CustomAuthToken(ObtainAuthToken): throttle_classes = [AuthTokenThrottle] def post(self, request, *args, **kwargs): - serializer = self.serializer_class(data=request.data, - context={'request': request}) + serializer = self.serializer_class( + data=request.data, + context={'request': request}) serializer.is_valid(raise_exception=True) user = serializer.validated_data['user'] if token := AccessToken.objects.filter(user=user, expires__gt=timezone.now(), scope__contains='read').filter( scope__contains='write').first(): access_token = token else: - access_token = AccessToken.objects.create(user=user, token=f'tda_{str(uuid.uuid4()).replace("-", "_")}', - expires=(timezone.now() + timezone.timedelta(days=365 * 5)), - scope='read write app') - return Response({ - 'id': access_token.id, - 'token': access_token.token, - 'scope': access_token.scope, - 'expires': access_token.expires, - 'user_id': access_token.user.pk, - 'test': user.pk - }) + access_token = AccessToken.objects.create( + user=user, token=f'tda_{str(uuid.uuid4()).replace("-", "_")}', + expires=(timezone.now() + timezone.timedelta(days=365 * 5)), + scope='read write app') + return Response( + { + 'id': access_token.id, + 'token': access_token.token, + 'scope': access_token.scope, + 'expires': access_token.expires, + 'user_id': access_token.user.pk, + 'test': user.pk + }) class RecipeUrlImportView(APIView): @@ -1376,61 +1395,71 @@ class RecipeUrlImportView(APIView): url = serializer.validated_data.get('url', None) data = unquote(serializer.validated_data.get('data', None)) if not url and not data: - return Response({ - 'error': True, - 'msg': _('Nothing to do.') - }, status=status.HTTP_400_BAD_REQUEST) + return Response( + { + 'error': True, + 'msg': _('Nothing to do.') + }, status=status.HTTP_400_BAD_REQUEST) elif url and not data: if re.match('^(https?://)?(www\\.youtube\\.com|youtu\\.be)/.+$', url): if validators.url(url, public=True): - return Response({ - 'recipe_json': get_from_youtube_scraper(url, request), - 'recipe_images': [], - }, status=status.HTTP_200_OK) + return Response( + { + 'recipe_json': get_from_youtube_scraper(url, request), + 'recipe_images': [], + }, status=status.HTTP_200_OK) if re.match( '^(.)*/view/recipe/[0-9]+/[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$', url): recipe_json = requests.get( - url.replace('/view/recipe/', '/api/recipe/').replace(re.split('/view/recipe/[0-9]+', url)[1], - '') + '?share=' + + url.replace('/view/recipe/', '/api/recipe/').replace( + re.split('/view/recipe/[0-9]+', url)[1], + '') + '?share=' + re.split('/view/recipe/[0-9]+', url)[1].replace('/', '')).json() recipe_json = clean_dict(recipe_json, 'id') serialized_recipe = RecipeExportSerializer(data=recipe_json, context={'request': request}) if serialized_recipe.is_valid(): recipe = serialized_recipe.save() if validators.url(recipe_json['image'], public=True): - recipe.image = File(handle_image(request, - File(io.BytesIO(requests.get(recipe_json['image']).content), - name='image'), - filetype=pathlib.Path(recipe_json['image']).suffix), - name=f'{uuid.uuid4()}_{recipe.pk}{pathlib.Path(recipe_json["image"]).suffix}') + recipe.image = File( + handle_image( + request, + File( + io.BytesIO(requests.get(recipe_json['image']).content), + name='image'), + filetype=pathlib.Path(recipe_json['image']).suffix), + name=f'{uuid.uuid4()}_{recipe.pk}{pathlib.Path(recipe_json["image"]).suffix}') recipe.save() - return Response({ - 'link': request.build_absolute_uri(reverse('view_recipe', args={recipe.pk})) - }, status=status.HTTP_201_CREATED) + return Response( + { + 'link': request.build_absolute_uri(reverse('view_recipe', args={recipe.pk})) + }, status=status.HTTP_201_CREATED) else: try: if validators.url(url, public=True): scrape = scrape_me(url_path=url, wild_mode=True) else: - return Response({ - 'error': True, - 'msg': _('Invalid Url') - }, status=status.HTTP_400_BAD_REQUEST) + return Response( + { + 'error': True, + 'msg': _('Invalid Url') + }, status=status.HTTP_400_BAD_REQUEST) except NoSchemaFoundInWildMode: pass except requests.exceptions.ConnectionError: - return Response({ - 'error': True, - 'msg': _('Connection Refused.') - }, status=status.HTTP_400_BAD_REQUEST) + return Response( + { + 'error': True, + 'msg': _('Connection Refused.') + }, status=status.HTTP_400_BAD_REQUEST) except requests.exceptions.MissingSchema: - return Response({ - 'error': True, - 'msg': _('Bad URL Schema.') - }, status=status.HTTP_400_BAD_REQUEST) + return Response( + { + 'error': True, + 'msg': _('Bad URL Schema.') + }, status=status.HTTP_400_BAD_REQUEST) else: try: data_json = json.loads(data) @@ -1446,16 +1475,18 @@ class RecipeUrlImportView(APIView): scrape = text_scraper(text=data, url=found_url) if scrape: - return Response({ - 'recipe_json': helper.get_from_scraper(scrape, request), - 'recipe_images': list(dict.fromkeys(get_images_from_soup(scrape.soup, url))), - }, status=status.HTTP_200_OK) + return Response( + { + 'recipe_json': helper.get_from_scraper(scrape, request), + 'recipe_images': list(dict.fromkeys(get_images_from_soup(scrape.soup, url))), + }, status=status.HTTP_200_OK) else: - return Response({ - 'error': True, - 'msg': _('No usable data could be found.') - }, status=status.HTTP_400_BAD_REQUEST) + return Response( + { + 'error': True, + 'msg': _('No usable data could be found.') + }, status=status.HTTP_400_BAD_REQUEST) else: return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) @@ -1547,8 +1578,9 @@ def import_files(request): return Response({'import_id': il.pk}, status=status.HTTP_200_OK) except NotImplementedError: - return Response({'error': True, 'msg': _('Importing is not implemented for this provider')}, - status=status.HTTP_400_BAD_REQUEST) + return Response( + {'error': True, 'msg': _('Importing is not implemented for this provider')}, + status=status.HTTP_400_BAD_REQUEST) else: return Response({'error': True, 'msg': form.errors}, status=status.HTTP_400_BAD_REQUEST) @@ -1624,8 +1656,9 @@ def get_recipe_file(request, recipe_id): @group_required('user') def sync_all(request): if request.space.demo or settings.HOSTED: - messages.add_message(request, messages.ERROR, - _('This feature is not yet available in the hosted version of tandoor!')) + messages.add_message( + request, messages.ERROR, + _('This feature is not yet available in the hosted version of tandoor!')) return redirect('index') monitors = Sync.objects.filter(active=True).filter(space=request.user.userspace_set.filter(active=1).first().space) @@ -1664,8 +1697,9 @@ def share_link(request, pk): if request.space.allow_sharing and has_group_permission(request.user, ('user',)): recipe = get_object_or_404(Recipe, pk=pk, space=request.space) link = ShareLink.objects.create(recipe=recipe, created_by=request.user, space=request.space) - return JsonResponse({'pk': pk, 'share': link.uuid, - 'link': request.build_absolute_uri(reverse('view_recipe', args=[pk, link.uuid]))}) + return JsonResponse( + {'pk': pk, 'share': link.uuid, + 'link': request.build_absolute_uri(reverse('view_recipe', args=[pk, link.uuid]))}) else: return JsonResponse({'error': 'sharing_disabled'}, status=403) From 6a393acd265cb397b803d75b4552f9d74c1b7326 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Thu, 11 Jan 2024 22:35:58 +0100 Subject: [PATCH 03/41] redo migration. cleanup commented out code --- .gitignore | 2 +- cookbook/connectors/connector.py | 24 +------------- cookbook/connectors/connector_manager.py | 18 ---------- ...ing_list_entry_created_enabled_and_more.py | 33 ------------------- ...tconfig.py => 0208_homeassistantconfig.py} | 10 ++++-- 5 files changed, 9 insertions(+), 78 deletions(-) delete mode 100644 cookbook/migrations/0207_homeassistantconfig_on_shopping_list_entry_created_enabled_and_more.py rename cookbook/migrations/{0206_alter_storage_path_homeassistantconfig.py => 0208_homeassistantconfig.py} (63%) diff --git a/.gitignore b/.gitignore index 553403a5..4c8df7a7 100644 --- a/.gitignore +++ b/.gitignore @@ -54,7 +54,7 @@ docs/_build/ target/ \.idea/dataSources/ - +.idea \.idea/dataSources\.xml \.idea/dataSources\.local\.xml diff --git a/cookbook/connectors/connector.py b/cookbook/connectors/connector.py index d2a435d7..f97905b9 100644 --- a/cookbook/connectors/connector.py +++ b/cookbook/connectors/connector.py @@ -16,26 +16,4 @@ class Connector(ABC): async def on_shopping_list_entry_deleted(self, space: Space, instance: ShoppingListEntry) -> None: pass - # @abstractmethod - # def on_recipe_created(self, instance: Recipe, **kwargs) -> None: - # pass - # - # @abstractmethod - # def on_recipe_updated(self, instance: Recipe, **kwargs) -> None: - # pass - # - # @abstractmethod - # def on_recipe_deleted(self, instance: Recipe, **kwargs) -> None: - # pass - # - # @abstractmethod - # def on_meal_plan_created(self, instance: MealPlan, **kwargs) -> None: - # pass - # - # @abstractmethod - # def on_meal_plan_updated(self, instance: MealPlan, **kwargs) -> None: - # pass - # - # @abstractmethod - # def on_meal_plan_deleted(self, instance: MealPlan, **kwargs) -> None: - # pass + # TODO: Add Recipes & possibly Meal Place listeners/hooks (And maybe more?) diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 0d3df8cf..93fa637d 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -19,7 +19,6 @@ class ActionType(Enum): class ConnectorManager: _connectors: Dict[str, List[Connector]] _listening_to_classes: UnionType = ShoppingListEntry | Recipe | MealPlan | Connector - max_concurrent_tasks = 2 def __init__(self): self._connectors = dict() @@ -79,20 +78,3 @@ class ConnectorManager: await asyncio.gather(*tasks, return_exceptions=False) except BaseException as e: print("received an exception from one of the tasks: ", e) - # if isinstance(instance, Recipe): - # if "created" in kwargs and kwargs["created"]: - # for connector in self._connectors: - # connector.on_recipe_created(instance, **kwargs) - # return - # for connector in self._connectors: - # connector.on_recipe_updated(instance, **kwargs) - # return - # - # if isinstance(instance, MealPlan): - # if "created" in kwargs and kwargs["created"]: - # for connector in self._connectors: - # connector.on_meal_plan_created(instance, **kwargs) - # return - # for connector in self._connectors: - # connector.on_meal_plan_updated(instance, **kwargs) - # return diff --git a/cookbook/migrations/0207_homeassistantconfig_on_shopping_list_entry_created_enabled_and_more.py b/cookbook/migrations/0207_homeassistantconfig_on_shopping_list_entry_created_enabled_and_more.py deleted file mode 100644 index 67f93b18..00000000 --- a/cookbook/migrations/0207_homeassistantconfig_on_shopping_list_entry_created_enabled_and_more.py +++ /dev/null @@ -1,33 +0,0 @@ -# Generated by Django 4.2.7 on 2024-01-11 19:53 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('cookbook', '0206_alter_storage_path_homeassistantconfig'), - ] - - operations = [ - migrations.AddField( - model_name='homeassistantconfig', - name='on_shopping_list_entry_created_enabled', - field=models.BooleanField(default=False, help_text='Enable syncing ShoppingListEntry to Homeassistant Todo List'), - ), - migrations.AddField( - model_name='homeassistantconfig', - name='on_shopping_list_entry_deleted_enabled', - field=models.BooleanField(default=False, help_text='Enable syncing ShoppingListEntry deletion to Homeassistant Todo List'), - ), - migrations.AddField( - model_name='homeassistantconfig', - name='on_shopping_list_entry_updated_enabled', - field=models.BooleanField(default=False, help_text='PLACEHOLDER'), - ), - migrations.AlterField( - model_name='homeassistantconfig', - name='url', - field=models.URLField(blank=True), - ), - ] diff --git a/cookbook/migrations/0206_alter_storage_path_homeassistantconfig.py b/cookbook/migrations/0208_homeassistantconfig.py similarity index 63% rename from cookbook/migrations/0206_alter_storage_path_homeassistantconfig.py rename to cookbook/migrations/0208_homeassistantconfig.py index acbec237..6eca066f 100644 --- a/cookbook/migrations/0206_alter_storage_path_homeassistantconfig.py +++ b/cookbook/migrations/0208_homeassistantconfig.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.7 on 2024-01-10 21:28 +# Generated by Django 4.2.7 on 2024-01-11 21:34 import cookbook.models from django.conf import settings @@ -8,9 +8,10 @@ import django.db.models.deletion class Migration(migrations.Migration): + dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('cookbook', '0205_alter_food_fdc_id_alter_propertytype_fdc_id'), + ('cookbook', '0207_space_logo_color_128_space_logo_color_144_and_more'), ] operations = [ @@ -19,9 +20,12 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('name', models.CharField(max_length=128, validators=[django.core.validators.MinLengthValidator(1)])), - ('url', models.URLField(blank=True, help_text='Something like http://homeassistant:8123/api')), + ('url', models.URLField(blank=True)), ('token', models.CharField(blank=True, max_length=512)), ('todo_entity', models.CharField(default='todo.shopping_list', max_length=128)), + ('on_shopping_list_entry_created_enabled', models.BooleanField(default=False, help_text='Enable syncing ShoppingListEntry to Homeassistant Todo List')), + ('on_shopping_list_entry_updated_enabled', models.BooleanField(default=False, help_text='PLACEHOLDER')), + ('on_shopping_list_entry_deleted_enabled', models.BooleanField(default=False, help_text='Enable syncing ShoppingListEntry deletion to Homeassistant Todo List')), ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)), ('space', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='cookbook.space')), ], From f1b41461db0a373320dc4f82ef08a32e3f78b30c Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Thu, 11 Jan 2024 22:46:29 +0100 Subject: [PATCH 04/41] bugfix for not working space loading --- cookbook/templatetags/theming_tags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookbook/templatetags/theming_tags.py b/cookbook/templatetags/theming_tags.py index f35096fd..68695082 100644 --- a/cookbook/templatetags/theming_tags.py +++ b/cookbook/templatetags/theming_tags.py @@ -11,7 +11,7 @@ register = template.Library() @register.simple_tag def theme_values(request): space = None - if request.space: + if space in request and request.space: space = request.space if not request.user.is_authenticated and UNAUTHENTICATED_THEME_FROM_SPACE > 0: with scopes_disabled(): From a61f79507b031be06c90d740ae74ae851d4e40c8 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Thu, 11 Jan 2024 23:09:42 +0100 Subject: [PATCH 05/41] add enabled field --- cookbook/admin.py | 4 ++-- cookbook/connectors/connector.py | 1 + cookbook/connectors/connector_manager.py | 2 +- cookbook/forms.py | 8 +++++++- cookbook/migrations/0208_homeassistantconfig.py | 3 ++- cookbook/models.py | 1 + cookbook/serializer.py | 2 +- cookbook/tables.py | 2 +- cookbook/views/edit.py | 1 + requirements.txt | 1 + 10 files changed, 18 insertions(+), 7 deletions(-) diff --git a/cookbook/admin.py b/cookbook/admin.py index 4a2b2697..3429e63d 100644 --- a/cookbook/admin.py +++ b/cookbook/admin.py @@ -96,8 +96,8 @@ admin.site.register(Storage, StorageAdmin) class HomeAssistantConfigAdmin(admin.ModelAdmin): - list_display = ('name',) - search_fields = ('name',) + list_display = ('id', 'name', 'enabled', 'url') + search_fields = ('name', 'url') admin.site.register(HomeAssistantConfig, HomeAssistantConfigAdmin) diff --git a/cookbook/connectors/connector.py b/cookbook/connectors/connector.py index f97905b9..76699e09 100644 --- a/cookbook/connectors/connector.py +++ b/cookbook/connectors/connector.py @@ -16,4 +16,5 @@ class Connector(ABC): async def on_shopping_list_entry_deleted(self, space: Space, instance: ShoppingListEntry) -> None: pass + # TODO: Maybe add an 'IsEnabled(self) -> Bool' to here # TODO: Add Recipes & possibly Meal Place listeners/hooks (And maybe more?) diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 93fa637d..8a50f90b 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -35,7 +35,7 @@ class ConnectorManager: connectors: List[Connector] = self._connectors[space.name] else: with scope(space=space): - connectors: List[Connector] = [HomeAssistant(config) for config in space.homeassistantconfig_set.all()] + connectors: List[Connector] = [HomeAssistant(config) for config in space.homeassistantconfig_set.all() if config.enabled] self._connectors[space.name] = connectors if len(connectors) == 0 or purge_connector_cache: diff --git a/cookbook/forms.py b/cookbook/forms.py index 76d4a17f..dce4da57 100644 --- a/cookbook/forms.py +++ b/cookbook/forms.py @@ -202,6 +202,11 @@ class HomeAssistantConfigForm(forms.ModelForm): help_text=_('Something like http://homeassistant.local:8123/api'), ) + enabled = forms.BooleanField( + help_text="Is the HomeAssistantConnector enabled", + required=False, + ) + on_shopping_list_entry_created_enabled = forms.BooleanField( help_text="Enable syncing ShoppingListEntry to Homeassistant Todo List -- Warning: Might have negative performance impact", required=False, @@ -220,7 +225,8 @@ class HomeAssistantConfigForm(forms.ModelForm): class Meta: model = HomeAssistantConfig fields = ( - 'name', 'url', 'token', 'todo_entity', 'on_shopping_list_entry_created_enabled', 'on_shopping_list_entry_updated_enabled', 'on_shopping_list_entry_deleted_enabled') + 'name', 'url', 'token', 'todo_entity', 'enabled', 'on_shopping_list_entry_created_enabled', 'on_shopping_list_entry_updated_enabled', + 'on_shopping_list_entry_deleted_enabled') help_texts = { 'url': _('http://homeassistant.local:8123/api for example'), diff --git a/cookbook/migrations/0208_homeassistantconfig.py b/cookbook/migrations/0208_homeassistantconfig.py index 6eca066f..a7140b34 100644 --- a/cookbook/migrations/0208_homeassistantconfig.py +++ b/cookbook/migrations/0208_homeassistantconfig.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.7 on 2024-01-11 21:34 +# Generated by Django 4.2.7 on 2024-01-11 22:06 import cookbook.models from django.conf import settings @@ -23,6 +23,7 @@ class Migration(migrations.Migration): ('url', models.URLField(blank=True)), ('token', models.CharField(blank=True, max_length=512)), ('todo_entity', models.CharField(default='todo.shopping_list', max_length=128)), + ('enabled', models.BooleanField(default=True, help_text='Is HomeAssistant Connector Enabled')), ('on_shopping_list_entry_created_enabled', models.BooleanField(default=False, help_text='Enable syncing ShoppingListEntry to Homeassistant Todo List')), ('on_shopping_list_entry_updated_enabled', models.BooleanField(default=False, help_text='PLACEHOLDER')), ('on_shopping_list_entry_deleted_enabled', models.BooleanField(default=False, help_text='Enable syncing ShoppingListEntry deletion to Homeassistant Todo List')), diff --git a/cookbook/models.py b/cookbook/models.py index bb9eb716..d6cf5078 100644 --- a/cookbook/models.py +++ b/cookbook/models.py @@ -374,6 +374,7 @@ class HomeAssistantConfig(models.Model, PermissionModelMixin): todo_entity = models.CharField(max_length=128, default='todo.shopping_list') + enabled = models.BooleanField(default=True, help_text="Is HomeAssistant Connector Enabled") on_shopping_list_entry_created_enabled = models.BooleanField(default=False, help_text="Enable syncing ShoppingListEntry to Homeassistant Todo List") on_shopping_list_entry_updated_enabled = models.BooleanField(default=False, help_text="PLACEHOLDER") on_shopping_list_entry_deleted_enabled = models.BooleanField(default=False, help_text="Enable syncing ShoppingListEntry deletion to Homeassistant Todo List") diff --git a/cookbook/serializer.py b/cookbook/serializer.py index 8e338c17..53fc8eeb 100644 --- a/cookbook/serializer.py +++ b/cookbook/serializer.py @@ -422,7 +422,7 @@ class HomeAssistantConfigSerializer(SpacedModelSerializer): class Meta: model = HomeAssistantConfig fields = ( - 'id', 'name', 'url', 'token', 'todo_entity', + 'id', 'name', 'url', 'token', 'todo_entity', 'enabled', 'on_shopping_list_entry_created_enabled', 'on_shopping_list_entry_updated_enabled', 'on_shopping_list_entry_deleted_enabled', 'created_by' ) diff --git a/cookbook/tables.py b/cookbook/tables.py index 8ffe2a53..4f49690c 100644 --- a/cookbook/tables.py +++ b/cookbook/tables.py @@ -21,7 +21,7 @@ class HomeAssistantConfigTable(tables.Table): class Meta: model = HomeAssistantConfig template_name = 'generic/table_template.html' - fields = ('id', 'name', 'url') + fields = ('id', 'name', 'enabled', 'url') class ImportLogTable(tables.Table): diff --git a/cookbook/views/edit.py b/cookbook/views/edit.py index ce4aac27..deb9654b 100644 --- a/cookbook/views/edit.py +++ b/cookbook/views/edit.py @@ -146,6 +146,7 @@ def edit_home_assistant_config(request, pk): instance.name = form.cleaned_data['name'] instance.url = form.cleaned_data['url'] instance.todo_entity = form.cleaned_data['todo_entity'] + instance.enabled = form.cleaned_data['enabled'] instance.on_shopping_list_entry_created_enabled = form.cleaned_data['on_shopping_list_entry_created_enabled'] instance.on_shopping_list_entry_updated_enabled = form.cleaned_data['on_shopping_list_entry_updated_enabled'] instance.on_shopping_list_entry_deleted_enabled = form.cleaned_data['on_shopping_list_entry_deleted_enabled'] diff --git a/requirements.txt b/requirements.txt index 9cf4efdf..0611ea42 100644 --- a/requirements.txt +++ b/requirements.txt @@ -46,3 +46,4 @@ pytest-factoryboy==2.5.1 pyppeteer==1.0.2 validators==0.20.0 pytube==15.0.0 +homeassistant-api==4.1.1.post2 From d576394c9957d48dbe08e4ac6f48d2275bf2ddd9 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Fri, 12 Jan 2024 20:50:23 +0100 Subject: [PATCH 06/41] run everything in a seperate process --- cookbook/connectors/connector_manager.py | 124 +++++++++++++++-------- cookbook/connectors/homeassistant.py | 42 +++++--- 2 files changed, 110 insertions(+), 56 deletions(-) diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 8a50f90b..2be64732 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -1,7 +1,12 @@ import asyncio +import logging +import multiprocessing +from asyncio import Task +from dataclasses import dataclass from enum import Enum +from multiprocessing import Queue from types import UnionType -from typing import List, Any, Dict +from typing import List, Any, Dict, Optional from django_scopes import scope @@ -9,6 +14,11 @@ from cookbook.connectors.connector import Connector from cookbook.connectors.homeassistant import HomeAssistant from cookbook.models import ShoppingListEntry, Recipe, MealPlan, Space +multiprocessing.set_start_method('fork') # https://code.djangoproject.com/ticket/31169 + +QUEUE_MAX_SIZE = 10 +REGISTERED_CLASSES: UnionType = ShoppingListEntry | Recipe | MealPlan | Connector + class ActionType(Enum): CREATED = 1 @@ -16,38 +26,25 @@ class ActionType(Enum): DELETED = 3 +@dataclass +class Payload: + instance: REGISTERED_CLASSES + actionType: ActionType + + class ConnectorManager: - _connectors: Dict[str, List[Connector]] - _listening_to_classes: UnionType = ShoppingListEntry | Recipe | MealPlan | Connector + _queue: Queue + _listening_to_classes = REGISTERED_CLASSES def __init__(self): - self._connectors = dict() + self._queue = multiprocessing.Queue(maxsize=QUEUE_MAX_SIZE) + self._worker = multiprocessing.Process(target=self.worker, args=(self._queue,), daemon=True) + self._worker.start() def __call__(self, instance: Any, **kwargs) -> None: - if not isinstance(instance, self._listening_to_classes): + if not isinstance(instance, self._listening_to_classes) or not hasattr(instance, "space"): return - # If a Connector was changed/updated, refresh connector from the database for said space - purge_connector_cache = isinstance(instance, Connector) - - space: Space = instance.space - if space.name in self._connectors and not purge_connector_cache: - connectors: List[Connector] = self._connectors[space.name] - else: - with scope(space=space): - connectors: List[Connector] = [HomeAssistant(config) for config in space.homeassistantconfig_set.all() if config.enabled] - self._connectors[space.name] = connectors - - if len(connectors) == 0 or purge_connector_cache: - return - - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - loop.run_until_complete(self.run_connectors(connectors, space, instance, **kwargs)) - loop.close() - - @staticmethod - async def run_connectors(connectors: List[Connector], space: Space, instance: Any, **kwargs): action_type: ActionType if "created" in kwargs and kwargs["created"]: action_type = ActionType.CREATED @@ -58,23 +55,64 @@ class ConnectorManager: else: return - tasks: List[asyncio.Task] = list() + self._queue.put_nowait(Payload(instance, action_type)) - if isinstance(instance, ShoppingListEntry): - shopping_list_entry: ShoppingListEntry = instance + def stop(self): + self._queue.close() + self._worker.join() - match action_type: - case ActionType.CREATED: - for connector in connectors: - tasks.append(asyncio.create_task(connector.on_shopping_list_entry_created(space, shopping_list_entry))) - case ActionType.UPDATED: - for connector in connectors: - tasks.append(asyncio.create_task(connector.on_shopping_list_entry_updated(space, shopping_list_entry))) - case ActionType.DELETED: - for connector in connectors: - tasks.append(asyncio.create_task(connector.on_shopping_list_entry_deleted(space, shopping_list_entry))) + @staticmethod + def worker(queue: Queue): + from django.db import connections + connections.close_all() - try: - await asyncio.gather(*tasks, return_exceptions=False) - except BaseException as e: - print("received an exception from one of the tasks: ", e) + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + _connectors: Dict[str, List[Connector]] = dict() + + while True: + item: Optional[Payload] = queue.get() + if item is None: + break + + # If a Connector was changed/updated, refresh connector from the database for said space + refresh_connector_cache = isinstance(item.instance, Connector) + + space: Space = item.instance.space + connectors: Optional[List[Connector]] = _connectors.get(space.name, None) + + if connectors is None or refresh_connector_cache: + with scope(space=space): + connectors: List[Connector] = [HomeAssistant(config) for config in space.homeassistantconfig_set.all() if config.enabled] + _connectors[space.name] = connectors + + if len(connectors) == 0 or refresh_connector_cache: + return + + loop.run_until_complete(run_connectors(connectors, space, item.instance, item.actionType)) + + loop.close() + + +async def run_connectors(connectors: List[Connector], space: Space, instance: REGISTERED_CLASSES, action_type: ActionType): + tasks: List[Task] = list() + + if isinstance(instance, ShoppingListEntry): + shopping_list_entry: ShoppingListEntry = instance + + match action_type: + case ActionType.CREATED: + for connector in connectors: + tasks.append(asyncio.create_task(connector.on_shopping_list_entry_created(space, shopping_list_entry))) + case ActionType.UPDATED: + for connector in connectors: + tasks.append(asyncio.create_task(connector.on_shopping_list_entry_updated(space, shopping_list_entry))) + case ActionType.DELETED: + for connector in connectors: + tasks.append(asyncio.create_task(connector.on_shopping_list_entry_deleted(space, shopping_list_entry))) + + try: + await asyncio.gather(*tasks, return_exceptions=False) + except BaseException: + logging.exception("received an exception from one of the tasks") diff --git a/cookbook/connectors/homeassistant.py b/cookbook/connectors/homeassistant.py index 6d504d6c..f5ae454f 100644 --- a/cookbook/connectors/homeassistant.py +++ b/cookbook/connectors/homeassistant.py @@ -1,29 +1,41 @@ import logging +from collections import defaultdict +from logging import Logger +from typing import Dict, Any, Optional -from homeassistant_api import Client, HomeassistantAPIError +from homeassistant_api import Client, HomeassistantAPIError, Domain from cookbook.connectors.connector import Connector from cookbook.models import ShoppingListEntry, HomeAssistantConfig, Space class HomeAssistant(Connector): + _domains_cache: dict[str, Domain] _config: HomeAssistantConfig + _logger: Logger + _client: Client def __init__(self, config: HomeAssistantConfig): + self._domains_cache = dict() self._config = config self._logger = logging.getLogger("connector.HomeAssistant") + self._client = Client(self._config.url, self._config.token, async_cache_session=False, use_async=True) async def on_shopping_list_entry_created(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None: if not self._config.on_shopping_list_entry_created_enabled: return item, description = _format_shopping_list_entry(shopping_list_entry) - async with Client(self._config.url, self._config.token, use_async=True) as client: - try: - todo_domain = await client.async_get_domain('todo') - await todo_domain.add_item(entity_id=self._config.todo_entity, item=item) - except HomeassistantAPIError as err: - self._logger.warning(f"[HomeAssistant {self._config.name}] Received an exception from the api: {err=}, {type(err)=}") + + todo_domain = self._domains_cache.get('todo') + try: + if todo_domain is None: + todo_domain = await self._client.async_get_domain('todo') + self._domains_cache['todo'] = todo_domain + + await todo_domain.add_item(entity_id=self._config.todo_entity, item=item) + except HomeassistantAPIError as err: + self._logger.warning(f"[HomeAssistant {self._config.name}] Received an exception from the api: {err=}, {type(err)=}") async def on_shopping_list_entry_updated(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None: if not self._config.on_shopping_list_entry_updated_enabled: @@ -35,12 +47,16 @@ class HomeAssistant(Connector): return item, description = _format_shopping_list_entry(shopping_list_entry) - async with Client(self._config.url, self._config.token, use_async=True) as client: - try: - todo_domain = await client.async_get_domain('todo') - await todo_domain.remove_item(entity_id=self._config.todo_entity, item=item) - except HomeassistantAPIError as err: - self._logger.warning(f"[HomeAssistant {self._config.name}] Received an exception from the api: {err=}, {type(err)=}") + + todo_domain = self._domains_cache.get('todo') + try: + if todo_domain is None: + todo_domain = await self._client.async_get_domain('todo') + self._domains_cache['todo'] = todo_domain + + await todo_domain.remove_item(entity_id=self._config.todo_entity, item=item) + except HomeassistantAPIError as err: + self._logger.warning(f"[HomeAssistant {self._config.name}] Received an exception from the api: {err=}, {type(err)=}") def _format_shopping_list_entry(shopping_list_entry: ShoppingListEntry): From 445e64c71edafef27d46d19c1facec25deab0170 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Fri, 12 Jan 2024 22:20:55 +0100 Subject: [PATCH 07/41] add an config toggle for external connectors --- cookbook/connectors/connector_manager.py | 6 ++++- .../migrations/0208_homeassistantconfig.py | 13 ++++++---- cookbook/models.py | 24 +++++++++++-------- cookbook/signals.py | 8 ++++--- cookbook/templates/base.html | 4 ++++ recipes/settings.py | 2 ++ 6 files changed, 38 insertions(+), 19 deletions(-) diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 2be64732..53252307 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -1,6 +1,7 @@ import asyncio import logging import multiprocessing +import queue from asyncio import Task from dataclasses import dataclass from enum import Enum @@ -55,7 +56,10 @@ class ConnectorManager: else: return - self._queue.put_nowait(Payload(instance, action_type)) + try: + self._queue.put_nowait(Payload(instance, action_type)) + except queue.Full: + return def stop(self): self._queue.close() diff --git a/cookbook/migrations/0208_homeassistantconfig.py b/cookbook/migrations/0208_homeassistantconfig.py index a7140b34..f21ff06a 100644 --- a/cookbook/migrations/0208_homeassistantconfig.py +++ b/cookbook/migrations/0208_homeassistantconfig.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.7 on 2024-01-11 22:06 +# Generated by Django 4.2.7 on 2024-01-12 21:01 import cookbook.models from django.conf import settings @@ -20,16 +20,19 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('name', models.CharField(max_length=128, validators=[django.core.validators.MinLengthValidator(1)])), + ('enabled', models.BooleanField(default=True, help_text='Is Connector Enabled')), + ('on_shopping_list_entry_created_enabled', models.BooleanField(default=False)), + ('on_shopping_list_entry_updated_enabled', models.BooleanField(default=False)), + ('on_shopping_list_entry_deleted_enabled', models.BooleanField(default=False)), ('url', models.URLField(blank=True)), ('token', models.CharField(blank=True, max_length=512)), ('todo_entity', models.CharField(default='todo.shopping_list', max_length=128)), - ('enabled', models.BooleanField(default=True, help_text='Is HomeAssistant Connector Enabled')), - ('on_shopping_list_entry_created_enabled', models.BooleanField(default=False, help_text='Enable syncing ShoppingListEntry to Homeassistant Todo List')), - ('on_shopping_list_entry_updated_enabled', models.BooleanField(default=False, help_text='PLACEHOLDER')), - ('on_shopping_list_entry_deleted_enabled', models.BooleanField(default=False, help_text='Enable syncing ShoppingListEntry deletion to Homeassistant Todo List')), ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)), ('space', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='cookbook.space')), ], + options={ + 'abstract': False, + }, bases=(models.Model, cookbook.models.PermissionModelMixin), ), ] diff --git a/cookbook/models.py b/cookbook/models.py index d6cf5078..a28cc5a5 100644 --- a/cookbook/models.py +++ b/cookbook/models.py @@ -366,24 +366,28 @@ class Space(ExportModelOperationsMixin('space'), models.Model): return self.name -class HomeAssistantConfig(models.Model, PermissionModelMixin): +class ConnectorConfig(models.Model, PermissionModelMixin): name = models.CharField(max_length=128, validators=[MinLengthValidator(1)]) - url = models.URLField(blank=True) - token = models.CharField(max_length=512, blank=True) - - todo_entity = models.CharField(max_length=128, default='todo.shopping_list') - - enabled = models.BooleanField(default=True, help_text="Is HomeAssistant Connector Enabled") - on_shopping_list_entry_created_enabled = models.BooleanField(default=False, help_text="Enable syncing ShoppingListEntry to Homeassistant Todo List") - on_shopping_list_entry_updated_enabled = models.BooleanField(default=False, help_text="PLACEHOLDER") - on_shopping_list_entry_deleted_enabled = models.BooleanField(default=False, help_text="Enable syncing ShoppingListEntry deletion to Homeassistant Todo List") + enabled = models.BooleanField(default=True, help_text="Is Connector Enabled") + on_shopping_list_entry_created_enabled = models.BooleanField(default=False) + on_shopping_list_entry_updated_enabled = models.BooleanField(default=False) + on_shopping_list_entry_deleted_enabled = models.BooleanField(default=False) created_by = models.ForeignKey(User, on_delete=models.PROTECT) space = models.ForeignKey(Space, on_delete=models.CASCADE) objects = ScopedManager(space='space') + class Meta: + abstract = True + + +class HomeAssistantConfig(ConnectorConfig): + url = models.URLField(blank=True) + token = models.CharField(max_length=512, blank=True) + todo_entity = models.CharField(max_length=128, default='todo.shopping_list') + class UserPreference(models.Model, PermissionModelMixin): # Themes diff --git a/cookbook/signals.py b/cookbook/signals.py index 2d94b1d0..ce957bce 100644 --- a/cookbook/signals.py +++ b/cookbook/signals.py @@ -15,6 +15,7 @@ from cookbook.helper.shopping_helper import RecipeShoppingEditor from cookbook.managers import DICTIONARY from cookbook.models import (Food, MealPlan, PropertyType, Recipe, SearchFields, SearchPreference, Step, Unit, UserPreference) +from recipes.settings import ENABLE_EXTERNAL_CONNECTORS SQLITE = True if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.postgresql': @@ -164,6 +165,7 @@ def clear_property_type_cache(sender, instance=None, created=False, **kwargs): caches['default'].delete(CacheHelper(instance.space).PROPERTY_TYPE_CACHE_KEY) -handler = ConnectorManager() -post_save.connect(handler, dispatch_uid="connector_manager") -post_delete.connect(handler, dispatch_uid="connector_manager") +if ENABLE_EXTERNAL_CONNECTORS: + handler = ConnectorManager() + post_save.connect(handler, dispatch_uid="connector_manager") + post_delete.connect(handler, dispatch_uid="connector_manager") diff --git a/cookbook/templates/base.html b/cookbook/templates/base.html index d84ccbd4..d9e2eda1 100644 --- a/cookbook/templates/base.html +++ b/cookbook/templates/base.html @@ -335,6 +335,10 @@ {% trans 'Space Settings' %} {% endif %} + {% if request.user == request.space.created_by or user.is_superuser %} + {% trans 'External Connectors' %} + {% endif %} {% if user.is_superuser %}
Date: Fri, 12 Jan 2024 23:13:53 +0100 Subject: [PATCH 08/41] write some simple tests --- cookbook/connectors/connector_manager.py | 10 +- .../api/test_api_home_assistant_config.py | 126 ++++++++++++++++++ .../edits/test_edits_home_assistant_config.py | 58 ++++++++ .../tests/other/test_connector_manager.py | 28 ++++ requirements.txt | 1 + 5 files changed, 218 insertions(+), 5 deletions(-) create mode 100644 cookbook/tests/api/test_api_home_assistant_config.py create mode 100644 cookbook/tests/edits/test_edits_home_assistant_config.py create mode 100644 cookbook/tests/other/test_connector_manager.py diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 53252307..d660b6d8 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -28,7 +28,7 @@ class ActionType(Enum): @dataclass -class Payload: +class Work: instance: REGISTERED_CLASSES actionType: ActionType @@ -57,7 +57,7 @@ class ConnectorManager: return try: - self._queue.put_nowait(Payload(instance, action_type)) + self._queue.put_nowait(Work(instance, action_type)) except queue.Full: return @@ -66,7 +66,7 @@ class ConnectorManager: self._worker.join() @staticmethod - def worker(queue: Queue): + def worker(worker_queue: Queue): from django.db import connections connections.close_all() @@ -76,7 +76,7 @@ class ConnectorManager: _connectors: Dict[str, List[Connector]] = dict() while True: - item: Optional[Payload] = queue.get() + item: Optional[Work] = worker_queue.get() if item is None: break @@ -119,4 +119,4 @@ async def run_connectors(connectors: List[Connector], space: Space, instance: RE try: await asyncio.gather(*tasks, return_exceptions=False) except BaseException: - logging.exception("received an exception from one of the tasks") + logging.exception("received an exception from one of the connectors") diff --git a/cookbook/tests/api/test_api_home_assistant_config.py b/cookbook/tests/api/test_api_home_assistant_config.py new file mode 100644 index 00000000..a3de0f45 --- /dev/null +++ b/cookbook/tests/api/test_api_home_assistant_config.py @@ -0,0 +1,126 @@ +import json + +import pytest +from django.contrib import auth +from django.urls import reverse +from django_scopes import scopes_disabled + +from cookbook.models import HomeAssistantConfig + +LIST_URL = 'api:homeassistantconfig-list' +DETAIL_URL = 'api:homeassistantconfig-detail' + + +@pytest.fixture() +def obj_1(space_1, u1_s1): + return HomeAssistantConfig.objects.create( + name='HomeAssistant 1', token='token', url='url', todo_entity='todo.shopping_list', enabled=True, created_by=auth.get_user(u1_s1), space=space_1, ) + + +@pytest.fixture +def obj_2(space_1, u1_s1): + return HomeAssistantConfig.objects.create( + name='HomeAssistant 2', token='token', url='url', todo_entity='todo.shopping_list', enabled=True, created_by=auth.get_user(u1_s1), space=space_1, ) + + +@pytest.mark.parametrize( + "arg", [ + ['a_u', 403], + ['g1_s1', 403], + ['u1_s1', 403], + ['a1_s1', 200], + ]) +def test_list_permission(arg, request): + c = request.getfixturevalue(arg[0]) + r = c.get(reverse(LIST_URL)) + assert r.status_code == arg[1] + if r.status_code == 200: + response = json.loads(r.content) + assert 'token' not in response + + +def test_list_space(obj_1, obj_2, a1_s1, a1_s2, space_2): + assert len(json.loads(a1_s1.get(reverse(LIST_URL)).content)) == 2 + assert len(json.loads(a1_s2.get(reverse(LIST_URL)).content)) == 0 + + obj_1.space = space_2 + obj_1.save() + + assert len(json.loads(a1_s1.get(reverse(LIST_URL)).content)) == 1 + assert len(json.loads(a1_s2.get(reverse(LIST_URL)).content)) == 1 + + +@pytest.mark.parametrize( + "arg", [ + ['a_u', 403], + ['g1_s1', 403], + ['u1_s1', 403], + ['a1_s1', 200], + ['g1_s2', 403], + ['u1_s2', 403], + ['a1_s2', 404], + ]) +def test_update(arg, request, obj_1): + test_token = '1234' + + c = request.getfixturevalue(arg[0]) + r = c.patch( + reverse( + DETAIL_URL, + args={obj_1.id} + ), + {'name': 'new', 'token': test_token}, + content_type='application/json' + ) + response = json.loads(r.content) + assert r.status_code == arg[1] + if r.status_code == 200: + assert response['name'] == 'new' + obj_1.refresh_from_db() + assert obj_1.token == test_token + + +@pytest.mark.parametrize( + "arg", [ + ['a_u', 403], + ['g1_s1', 403], + ['u1_s1', 403], + ['a1_s1', 201], + ]) +def test_add(arg, request, a1_s2, obj_1): + c = request.getfixturevalue(arg[0]) + r = c.post( + reverse(LIST_URL), + {'name': 'test', 'url': 'http://localhost:8123/api', 'token': '1234', 'enabled': 'true'}, + content_type='application/json' + ) + response = json.loads(r.content) + print(r.content) + assert r.status_code == arg[1] + if r.status_code == 201: + assert response['name'] == 'test' + r = c.get(reverse(DETAIL_URL, args={response['id']})) + assert r.status_code == 200 + r = a1_s2.get(reverse(DETAIL_URL, args={response['id']})) + assert r.status_code == 404 + + +def test_delete(a1_s1, a1_s2, obj_1): + r = a1_s2.delete( + reverse( + DETAIL_URL, + args={obj_1.id} + ) + ) + assert r.status_code == 404 + + r = a1_s1.delete( + reverse( + DETAIL_URL, + args={obj_1.id} + ) + ) + + assert r.status_code == 204 + with scopes_disabled(): + assert HomeAssistantConfig.objects.count() == 0 diff --git a/cookbook/tests/edits/test_edits_home_assistant_config.py b/cookbook/tests/edits/test_edits_home_assistant_config.py new file mode 100644 index 00000000..053972b6 --- /dev/null +++ b/cookbook/tests/edits/test_edits_home_assistant_config.py @@ -0,0 +1,58 @@ +from cookbook.models import Storage, HomeAssistantConfig +from django.contrib import auth +from django.urls import reverse +import pytest + +EDIT_VIEW_NAME = 'edit_home_assistant_config' + + +@pytest.fixture +def home_assistant_config_obj(a1_s1, space_1): + return HomeAssistantConfig.objects.create( + name='HomeAssistant 1', + token='token', + url='url', + todo_entity='todo.shopping_list', + enabled=True, + created_by=auth.get_user(a1_s1), + space=space_1, + ) + + +def test_edit_home_assistant_config(home_assistant_config_obj, a1_s1, a1_s2): + new_token = '1234_token' + + r = a1_s1.post( + reverse(EDIT_VIEW_NAME, args={home_assistant_config_obj.pk}), + { + 'name': 'HomeAssistant 1', + 'token': new_token, + } + ) + home_assistant_config_obj.refresh_from_db() + assert r.status_code == 200 + assert home_assistant_config_obj.token == new_token + + r = a1_s2.post( + reverse(EDIT_VIEW_NAME, args={home_assistant_config_obj.pk}), + { + 'name': 'HomeAssistant 1', + 'token': new_token, + } + ) + assert r.status_code == 404 + + +@pytest.mark.parametrize( + "arg", [ + ['a_u', 302], + ['g1_s1', 302], + ['u1_s1', 302], + ['a1_s1', 200], + ['g1_s2', 302], + ['u1_s2', 302], + ['a1_s2', 404], + ]) +def test_view_permission(arg, request, home_assistant_config_obj): + c = request.getfixturevalue(arg[0]) + assert c.get(reverse(EDIT_VIEW_NAME, args={home_assistant_config_obj.pk})).status_code == arg[1] diff --git a/cookbook/tests/other/test_connector_manager.py b/cookbook/tests/other/test_connector_manager.py new file mode 100644 index 00000000..c6df778f --- /dev/null +++ b/cookbook/tests/other/test_connector_manager.py @@ -0,0 +1,28 @@ +from dataclasses import dataclass + +import pytest +from django.contrib import auth +from mock.mock import Mock + +from cookbook.connectors.connector import Connector +from cookbook.connectors.connector_manager import run_connectors, ActionType +from cookbook.models import ShoppingListEntry, ShoppingList, Food + + +@pytest.fixture() +def obj_1(space_1, u1_s1): + e = ShoppingListEntry.objects.create(created_by=auth.get_user(u1_s1), food=Food.objects.get_or_create(name='test 1', space=space_1)[0], space=space_1) + s = ShoppingList.objects.create(created_by=auth.get_user(u1_s1), space=space_1, ) + s.entries.add(e) + return e + + +@pytest.mark.asyncio +async def test_run_connectors(space_1, u1_s1, obj_1) -> None: + connector_mock = Mock(spec=Connector) + + await run_connectors([connector_mock], space_1, obj_1, ActionType.DELETED) + + assert not connector_mock.on_shopping_list_entry_updated.called + assert not connector_mock.on_shopping_list_entry_created.called + connector_mock.on_shopping_list_entry_deleted.assert_called_once_with(space_1, obj_1) diff --git a/requirements.txt b/requirements.txt index 0611ea42..23789d2c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,6 +33,7 @@ django-allauth==0.58.1 recipe-scrapers==14.52.0 django-scopes==2.0.0 pytest==7.4.3 +pytest-asyncio==0.23.3 pytest-django==4.6.0 django-treebeard==4.7 django-cors-headers==4.2.0 From 9c804863a815bb83eee5119fff4ea63b5be48e73 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Fri, 12 Jan 2024 23:15:28 +0100 Subject: [PATCH 09/41] undo accidental changes --- cookbook/models.py | 59 +++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 32 deletions(-) diff --git a/cookbook/models.py b/cookbook/models.py index a28cc5a5..9d6b9c5a 100644 --- a/cookbook/models.py +++ b/cookbook/models.py @@ -49,16 +49,14 @@ def get_active_space(self): def get_shopping_share(self): # get list of users that shared shopping list with user. Django ORM forbids this type of query, so raw is required - return User.objects.raw( - ' '.join( - [ - 'SELECT auth_user.id FROM auth_user', - 'INNER JOIN cookbook_userpreference', - 'ON (auth_user.id = cookbook_userpreference.user_id)', - 'INNER JOIN cookbook_userpreference_shopping_share', - 'ON (cookbook_userpreference.user_id = cookbook_userpreference_shopping_share.userpreference_id)', - 'WHERE cookbook_userpreference_shopping_share.user_id ={}'.format(self.id) - ])) + return User.objects.raw(' '.join([ + 'SELECT auth_user.id FROM auth_user', + 'INNER JOIN cookbook_userpreference', + 'ON (auth_user.id = cookbook_userpreference.user_id)', + 'INNER JOIN cookbook_userpreference_shopping_share', + 'ON (cookbook_userpreference.user_id = cookbook_userpreference_shopping_share.userpreference_id)', + 'WHERE cookbook_userpreference_shopping_share.user_id ={}'.format(self.id) + ])) auth.models.User.add_to_class('get_user_display_name', get_user_display_name) @@ -700,11 +698,10 @@ class Food(ExportModelOperationsMixin('food'), TreeModel, PermissionModelMixin): if len(inherit) > 0: # ManyToMany cannot be updated through an UPDATE operation for i in inherit: - trough.objects.bulk_create( - [ - trough(food_id=x, foodinheritfield_id=i['id']) - for x in Food.objects.filter(tree_filter).values_list('id', flat=True) - ]) + trough.objects.bulk_create([ + trough(food_id=x, foodinheritfield_id=i['id']) + for x in Food.objects.filter(tree_filter).values_list('id', flat=True) + ]) inherit = [x['field'] for x in inherit] for field in ['ignore_shopping', 'substitute_children', 'substitute_siblings']: @@ -831,9 +828,8 @@ class PropertyType(models.Model, PermissionModelMixin): unit = models.CharField(max_length=64, blank=True, null=True) order = models.IntegerField(default=0) description = models.CharField(max_length=512, blank=True, null=True) - category = models.CharField( - max_length=64, choices=((NUTRITION, _('Nutrition')), (ALLERGEN, _('Allergen')), - (PRICE, _('Price')), (GOAL, _('Goal')), (OTHER, _('Other'))), null=True, blank=True) + category = models.CharField(max_length=64, choices=((NUTRITION, _('Nutrition')), (ALLERGEN, _('Allergen')), + (PRICE, _('Price')), (GOAL, _('Goal')), (OTHER, _('Other'))), null=True, blank=True) open_data_slug = models.CharField(max_length=128, null=True, blank=True, default=None) fdc_id = models.IntegerField(null=True, default=None, blank=True) @@ -1396,20 +1392,19 @@ class Automation(ExportModelOperationsMixin('automations'), models.Model, Permis UNIT_REPLACE = 'UNIT_REPLACE' NAME_REPLACE = 'NAME_REPLACE' - type = models.CharField( - max_length=128, - choices=( - (FOOD_ALIAS, _('Food Alias')), - (UNIT_ALIAS, _('Unit Alias')), - (KEYWORD_ALIAS, _('Keyword Alias')), - (DESCRIPTION_REPLACE, _('Description Replace')), - (INSTRUCTION_REPLACE, _('Instruction Replace')), - (NEVER_UNIT, _('Never Unit')), - (TRANSPOSE_WORDS, _('Transpose Words')), - (FOOD_REPLACE, _('Food Replace')), - (UNIT_REPLACE, _('Unit Replace')), - (NAME_REPLACE, _('Name Replace')), - )) + type = models.CharField(max_length=128, + choices=( + (FOOD_ALIAS, _('Food Alias')), + (UNIT_ALIAS, _('Unit Alias')), + (KEYWORD_ALIAS, _('Keyword Alias')), + (DESCRIPTION_REPLACE, _('Description Replace')), + (INSTRUCTION_REPLACE, _('Instruction Replace')), + (NEVER_UNIT, _('Never Unit')), + (TRANSPOSE_WORDS, _('Transpose Words')), + (FOOD_REPLACE, _('Food Replace')), + (UNIT_REPLACE, _('Unit Replace')), + (NAME_REPLACE, _('Name Replace')), + )) name = models.CharField(max_length=128, default='') description = models.TextField(blank=True, null=True) From 022439e01711ca3c7b5c7e721dfe8017191e4922 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Fri, 12 Jan 2024 23:40:16 +0100 Subject: [PATCH 10/41] increase queue size to account for recipe adding burst --- cookbook/connectors/connector_manager.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index d660b6d8..332f2f0d 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -17,7 +17,7 @@ from cookbook.models import ShoppingListEntry, Recipe, MealPlan, Space multiprocessing.set_start_method('fork') # https://code.djangoproject.com/ticket/31169 -QUEUE_MAX_SIZE = 10 +QUEUE_MAX_SIZE = 25 REGISTERED_CLASSES: UnionType = ShoppingListEntry | Recipe | MealPlan | Connector @@ -116,6 +116,9 @@ async def run_connectors(connectors: List[Connector], space: Space, instance: RE for connector in connectors: tasks.append(asyncio.create_task(connector.on_shopping_list_entry_deleted(space, shopping_list_entry))) + if len(tasks) == 0: + return + try: await asyncio.gather(*tasks, return_exceptions=False) except BaseException: From 1a37961ceba786c192d71ea1e7bc864744454514 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Fri, 12 Jan 2024 23:44:15 +0100 Subject: [PATCH 11/41] add mock to requirements --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 23789d2c..16fca731 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,6 +26,7 @@ pyyaml==6.0.1 uritemplate==4.1.1 beautifulsoup4==4.12.2 microdata==0.8.0 +mock==5.1.0 Jinja2==3.1.2 django-webpack-loader==1.8.1 git+https://github.com/BITSOLVER/django-js-reverse@071e304fd600107bc64bbde6f2491f1fe049ec82 From 50eb232fff7eba47f370da31b0aa818e8c02b333 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Sat, 13 Jan 2024 00:24:58 +0100 Subject: [PATCH 12/41] update tests and fix small bug in connector_manager --- cookbook/connectors/connector_manager.py | 11 ++++++----- .../edits/test_edits_home_assistant_config.py | 19 +++++++++++++------ 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 332f2f0d..833e771a 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -13,12 +13,13 @@ from django_scopes import scope from cookbook.connectors.connector import Connector from cookbook.connectors.homeassistant import HomeAssistant -from cookbook.models import ShoppingListEntry, Recipe, MealPlan, Space +from cookbook.models import ShoppingListEntry, Recipe, MealPlan, Space, HomeAssistantConfig, ConnectorConfig multiprocessing.set_start_method('fork') # https://code.djangoproject.com/ticket/31169 QUEUE_MAX_SIZE = 25 -REGISTERED_CLASSES: UnionType = ShoppingListEntry | Recipe | MealPlan | Connector +REGISTERED_CLASSES: UnionType = ShoppingListEntry | Recipe | MealPlan +CONNECTOR_UPDATE_CLASSES: UnionType = HomeAssistantConfig | ConnectorConfig class ActionType(Enum): @@ -35,7 +36,7 @@ class Work: class ConnectorManager: _queue: Queue - _listening_to_classes = REGISTERED_CLASSES + _listening_to_classes = REGISTERED_CLASSES | CONNECTOR_UPDATE_CLASSES def __init__(self): self._queue = multiprocessing.Queue(maxsize=QUEUE_MAX_SIZE) @@ -81,10 +82,10 @@ class ConnectorManager: break # If a Connector was changed/updated, refresh connector from the database for said space - refresh_connector_cache = isinstance(item.instance, Connector) + refresh_connector_cache = isinstance(item.instance, CONNECTOR_UPDATE_CLASSES) space: Space = item.instance.space - connectors: Optional[List[Connector]] = _connectors.get(space.name, None) + connectors: Optional[List[Connector]] = _connectors.get(space.name) if connectors is None or refresh_connector_cache: with scope(space=space): diff --git a/cookbook/tests/edits/test_edits_home_assistant_config.py b/cookbook/tests/edits/test_edits_home_assistant_config.py index 053972b6..7f2e19a9 100644 --- a/cookbook/tests/edits/test_edits_home_assistant_config.py +++ b/cookbook/tests/edits/test_edits_home_assistant_config.py @@ -1,7 +1,8 @@ -from cookbook.models import Storage, HomeAssistantConfig +import pytest from django.contrib import auth from django.urls import reverse -import pytest + +from cookbook.models import HomeAssistantConfig EDIT_VIEW_NAME = 'edit_home_assistant_config' @@ -19,25 +20,31 @@ def home_assistant_config_obj(a1_s1, space_1): ) -def test_edit_home_assistant_config(home_assistant_config_obj, a1_s1, a1_s2): +def test_edit_home_assistant_config(home_assistant_config_obj: HomeAssistantConfig, a1_s1, a1_s2): new_token = '1234_token' r = a1_s1.post( reverse(EDIT_VIEW_NAME, args={home_assistant_config_obj.pk}), { - 'name': 'HomeAssistant 1', + 'name': home_assistant_config_obj.name, + 'url': home_assistant_config_obj.url, + 'todo_entity': home_assistant_config_obj.todo_entity, 'token': new_token, + 'enabled': home_assistant_config_obj.enabled, } ) - home_assistant_config_obj.refresh_from_db() assert r.status_code == 200 + home_assistant_config_obj.refresh_from_db() assert home_assistant_config_obj.token == new_token r = a1_s2.post( reverse(EDIT_VIEW_NAME, args={home_assistant_config_obj.pk}), { - 'name': 'HomeAssistant 1', + 'name': home_assistant_config_obj.name, + 'url': home_assistant_config_obj.url, + 'todo_entity': home_assistant_config_obj.todo_entity, 'token': new_token, + 'enabled': home_assistant_config_obj.enabled, } ) assert r.status_code == 404 From 48ac70de95cffdda0c4197ce54437a17632dbc3c Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Sat, 13 Jan 2024 11:56:51 +0100 Subject: [PATCH 13/41] make the tests check for any error message --- .../tests/edits/test_edits_home_assistant_config.py | 9 +++++++-- cookbook/tests/edits/test_edits_storage.py | 12 +++++++++--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/cookbook/tests/edits/test_edits_home_assistant_config.py b/cookbook/tests/edits/test_edits_home_assistant_config.py index 7f2e19a9..0df507de 100644 --- a/cookbook/tests/edits/test_edits_home_assistant_config.py +++ b/cookbook/tests/edits/test_edits_home_assistant_config.py @@ -1,5 +1,7 @@ import pytest from django.contrib import auth +from django.contrib import messages +from django.contrib.messages import get_messages from django.urls import reverse from cookbook.models import HomeAssistantConfig @@ -12,7 +14,7 @@ def home_assistant_config_obj(a1_s1, space_1): return HomeAssistantConfig.objects.create( name='HomeAssistant 1', token='token', - url='url', + url='http://localhost:8123/api', todo_entity='todo.shopping_list', enabled=True, created_by=auth.get_user(a1_s1), @@ -28,12 +30,15 @@ def test_edit_home_assistant_config(home_assistant_config_obj: HomeAssistantConf { 'name': home_assistant_config_obj.name, 'url': home_assistant_config_obj.url, - 'todo_entity': home_assistant_config_obj.todo_entity, 'token': new_token, + 'todo_entity': home_assistant_config_obj.todo_entity, 'enabled': home_assistant_config_obj.enabled, } ) assert r.status_code == 200 + r_messages = [m for m in get_messages(r.wsgi_request)] + assert not any(m.level > messages.SUCCESS for m in r_messages) + home_assistant_config_obj.refresh_from_db() assert home_assistant_config_obj.token == new_token diff --git a/cookbook/tests/edits/test_edits_storage.py b/cookbook/tests/edits/test_edits_storage.py index 125445e1..9c4e08e1 100644 --- a/cookbook/tests/edits/test_edits_storage.py +++ b/cookbook/tests/edits/test_edits_storage.py @@ -1,7 +1,10 @@ -from cookbook.models import Storage -from django.contrib import auth -from django.urls import reverse import pytest +from django.contrib import auth +from django.contrib import messages +from django.contrib.messages import get_messages +from django.urls import reverse + +from cookbook.models import Storage @pytest.fixture @@ -29,6 +32,9 @@ def test_edit_storage(storage_obj, a1_s1, a1_s2): ) storage_obj.refresh_from_db() assert r.status_code == 200 + r_messages = [m for m in get_messages(r.wsgi_request)] + assert not any(m.level > messages.SUCCESS for m in r_messages) + assert storage_obj.password == '1234_pw' assert storage_obj.token == '1234_token' From c7dd61e2391bae009b8e76dd525e114cc793d1ab Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Sat, 13 Jan 2024 11:57:08 +0100 Subject: [PATCH 14/41] add caching to the ci-cd workflow --- .github/workflows/ci.yml | 56 ++++++++++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d699a564..c4a47812 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,31 +10,65 @@ jobs: max-parallel: 4 matrix: python-version: ['3.10'] + node-version: ['18'] steps: - uses: actions/checkout@v3 - - name: Set up Python 3.10 + - uses: awalsh128/cache-apt-pkgs-action@v1.3.1 + with: + packages: libsasl2-dev python3-dev libldap2-dev libssl-dev + version: 1.0 + + # Setup python & dependencies + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: ${{ matrix.python-version }} + cache: 'pip' + + - name: Install Python Dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + # Build Vue frontend - - uses: actions/setup-node@v3 + - name: Set up Node ${{ matrix.node-version }} + uses: actions/setup-node@v3 with: - node-version: '18' + node-version: ${{ matrix.node-version }} + cache: 'yarn' + cache-dependency-path: ./vue/yarn.lock + - name: Install Vue dependencies working-directory: ./vue run: yarn install + - name: Build Vue dependencies working-directory: ./vue run: yarn build - - name: Install Django dependencies + + # Build backend + - name: Cache Django collectstatic + uses: actions/cache@v2 + with: + path: ./staticfiles + key: | + ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}-collectstatic-${{ hashFiles('**/*.css', '**/*.js') }} + restore-keys: | + ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}-collectstatic + + - name: Compile Django StatisFiles run: | - sudo apt-get -y update - sudo apt-get install -y libsasl2-dev python3-dev libldap2-dev libssl-dev - python -m pip install --upgrade pip - pip install -r requirements.txt python3 manage.py collectstatic --noinput python3 manage.py collectstatic_js_reverse + - name: Django Testing project - run: | - pytest + run: pytest --junitxml=junit/test-results-${{ matrix.python-version }}.xml + + - name: Publish Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + comment_mode: off + files: | + junit/test-results-${{ matrix.python-version }}.xml From 87ede4b9cc0e9d26772ad9ddc46af960dcf9610f Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Sat, 13 Jan 2024 13:43:08 +0100 Subject: [PATCH 15/41] change formatting a bit, and add async close method --- cookbook/connectors/connector.py | 4 +++ cookbook/connectors/connector_manager.py | 31 +++++++++++++++++++----- cookbook/connectors/homeassistant.py | 12 +++++---- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/cookbook/connectors/connector.py b/cookbook/connectors/connector.py index 76699e09..4ed61350 100644 --- a/cookbook/connectors/connector.py +++ b/cookbook/connectors/connector.py @@ -16,5 +16,9 @@ class Connector(ABC): async def on_shopping_list_entry_deleted(self, space: Space, instance: ShoppingListEntry) -> None: pass + @abstractmethod + async def close(self) -> None: + pass + # TODO: Maybe add an 'IsEnabled(self) -> Bool' to here # TODO: Add Recipes & possibly Meal Place listeners/hooks (And maybe more?) diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 833e771a..39919597 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -5,7 +5,7 @@ import queue from asyncio import Task from dataclasses import dataclass from enum import Enum -from multiprocessing import Queue +from multiprocessing import JoinableQueue from types import UnionType from typing import List, Any, Dict, Optional @@ -35,11 +35,11 @@ class Work: class ConnectorManager: - _queue: Queue + _queue: JoinableQueue _listening_to_classes = REGISTERED_CLASSES | CONNECTOR_UPDATE_CLASSES def __init__(self): - self._queue = multiprocessing.Queue(maxsize=QUEUE_MAX_SIZE) + self._queue = multiprocessing.JoinableQueue(maxsize=QUEUE_MAX_SIZE) self._worker = multiprocessing.Process(target=self.worker, args=(self._queue,), daemon=True) self._worker.start() @@ -60,14 +60,16 @@ class ConnectorManager: try: self._queue.put_nowait(Work(instance, action_type)) except queue.Full: + logging.info("queue was full, so skipping %s", instance) return def stop(self): + self._queue.join() self._queue.close() self._worker.join() @staticmethod - def worker(worker_queue: Queue): + def worker(worker_queue: JoinableQueue): from django.db import connections connections.close_all() @@ -77,7 +79,10 @@ class ConnectorManager: _connectors: Dict[str, List[Connector]] = dict() while True: - item: Optional[Work] = worker_queue.get() + try: + item: Optional[Work] = worker_queue.get() + except KeyboardInterrupt: + break if item is None: break @@ -88,18 +93,32 @@ class ConnectorManager: connectors: Optional[List[Connector]] = _connectors.get(space.name) if connectors is None or refresh_connector_cache: + if connectors is not None: + loop.run_until_complete(close_connectors(connectors)) + with scope(space=space): connectors: List[Connector] = [HomeAssistant(config) for config in space.homeassistantconfig_set.all() if config.enabled] _connectors[space.name] = connectors if len(connectors) == 0 or refresh_connector_cache: - return + worker_queue.task_done() + continue loop.run_until_complete(run_connectors(connectors, space, item.instance, item.actionType)) + worker_queue.task_done() loop.close() +async def close_connectors(connectors: List[Connector]): + tasks: List[Task] = [asyncio.create_task(connector.close()) for connector in connectors] + + try: + await asyncio.gather(*tasks, return_exceptions=False) + except BaseException: + logging.exception("received an exception while closing one of the connectors") + + async def run_connectors(connectors: List[Connector], space: Space, instance: REGISTERED_CLASSES, action_type: ActionType): tasks: List[Task] = list() diff --git a/cookbook/connectors/homeassistant.py b/cookbook/connectors/homeassistant.py index f5ae454f..4499c7e7 100644 --- a/cookbook/connectors/homeassistant.py +++ b/cookbook/connectors/homeassistant.py @@ -1,7 +1,5 @@ import logging -from collections import defaultdict from logging import Logger -from typing import Dict, Any, Optional from homeassistant_api import Client, HomeassistantAPIError, Domain @@ -58,16 +56,20 @@ class HomeAssistant(Connector): except HomeassistantAPIError as err: self._logger.warning(f"[HomeAssistant {self._config.name}] Received an exception from the api: {err=}, {type(err)=}") + async def close(self) -> None: + await self._client.async_cache_session.close() + def _format_shopping_list_entry(shopping_list_entry: ShoppingListEntry): item = shopping_list_entry.food.name if shopping_list_entry.amount > 0: + item += f" ({shopping_list_entry.amount:.2f}".rstrip('0').rstrip('.') if shopping_list_entry.unit and shopping_list_entry.unit.base_unit and len(shopping_list_entry.unit.base_unit) > 0: - item += f" ({shopping_list_entry.amount} {shopping_list_entry.unit.base_unit})" + item += f" {shopping_list_entry.unit.base_unit})" elif shopping_list_entry.unit and shopping_list_entry.unit.name and len(shopping_list_entry.unit.name) > 0: - item += f" ({shopping_list_entry.amount} {shopping_list_entry.unit.name})" + item += f" {shopping_list_entry.unit.name})" else: - item += f" ({shopping_list_entry.amount})" + item += ")" description = "Imported by TandoorRecipes" if shopping_list_entry.created_by.first_name and len(shopping_list_entry.created_by.first_name) > 0: From 362c0340fce379a87d83ec05684f1ba5504e14a1 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Sat, 13 Jan 2024 16:16:05 +0100 Subject: [PATCH 16/41] skip whole yarn and static files if there was a cache hit --- .github/workflows/ci.yml | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c4a47812..8d70b90f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,8 +31,20 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt + - name: Cache StaticFiles + uses: actions/cache@v3 + id: django_cache + with: + path: | + ./cookbook/static + ./vue/webpack-stats.json + ./staticfiles + key: | + ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}-collectstatic-${{ hashFiles('**/*.css', '**/*.js', 'vue/src/*') }} + # Build Vue frontend - name: Set up Node ${{ matrix.node-version }} + if: steps.django_cache.outputs.cache-hit != 'true' uses: actions/setup-node@v3 with: node-version: ${{ matrix.node-version }} @@ -40,24 +52,18 @@ jobs: cache-dependency-path: ./vue/yarn.lock - name: Install Vue dependencies + if: steps.django_cache.outputs.cache-hit != 'true' working-directory: ./vue run: yarn install - name: Build Vue dependencies + if: steps.django_cache.outputs.cache-hit != 'true' working-directory: ./vue run: yarn build # Build backend - - name: Cache Django collectstatic - uses: actions/cache@v2 - with: - path: ./staticfiles - key: | - ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}-collectstatic-${{ hashFiles('**/*.css', '**/*.js') }} - restore-keys: | - ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}-collectstatic - - name: Compile Django StatisFiles + if: steps.django_cache.outputs.cache-hit != 'true' run: | python3 manage.py collectstatic --noinput python3 manage.py collectstatic_js_reverse From 17163b0dbadedb0d0411c86984955ceaa3fb08d5 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Sat, 13 Jan 2024 16:44:18 +0100 Subject: [PATCH 17/41] save cache on failed tests --- .github/workflows/ci.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8d70b90f..cb72bc0c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,7 @@ jobs: key: | ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}-collectstatic-${{ hashFiles('**/*.css', '**/*.js', 'vue/src/*') }} - # Build Vue frontend + # Build Vue frontend & Dependencies - name: Set up Node ${{ matrix.node-version }} if: steps.django_cache.outputs.cache-hit != 'true' uses: actions/setup-node@v3 @@ -61,13 +61,22 @@ jobs: working-directory: ./vue run: yarn build - # Build backend - name: Compile Django StatisFiles if: steps.django_cache.outputs.cache-hit != 'true' run: | python3 manage.py collectstatic --noinput python3 manage.py collectstatic_js_reverse + - uses: actions/cache/save@v3 + if: steps.django_cache.outputs.cache-hit != 'true' + with: + path: | + ./cookbook/static + ./vue/webpack-stats.json + ./staticfiles + key: | + ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}-collectstatic-${{ hashFiles('**/*.css', '**/*.js', 'vue/src/*') }} + - name: Django Testing project run: pytest --junitxml=junit/test-results-${{ matrix.python-version }}.xml From fb65100b140adc42356da44d78964b13dfb8553d Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Sat, 13 Jan 2024 20:30:54 +0100 Subject: [PATCH 18/41] add debug logging --- cookbook/connectors/homeassistant.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cookbook/connectors/homeassistant.py b/cookbook/connectors/homeassistant.py index 4499c7e7..e394e974 100644 --- a/cookbook/connectors/homeassistant.py +++ b/cookbook/connectors/homeassistant.py @@ -31,6 +31,7 @@ class HomeAssistant(Connector): todo_domain = await self._client.async_get_domain('todo') self._domains_cache['todo'] = todo_domain + logging.debug(f"pushing {item} to {self._config.name}") await todo_domain.add_item(entity_id=self._config.todo_entity, item=item) except HomeassistantAPIError as err: self._logger.warning(f"[HomeAssistant {self._config.name}] Received an exception from the api: {err=}, {type(err)=}") @@ -52,6 +53,7 @@ class HomeAssistant(Connector): todo_domain = await self._client.async_get_domain('todo') self._domains_cache['todo'] = todo_domain + logging.debug(f"deleting {item} from {self._config.name}") await todo_domain.remove_item(entity_id=self._config.todo_entity, item=item) except HomeassistantAPIError as err: self._logger.warning(f"[HomeAssistant {self._config.name}] Received an exception from the api: {err=}, {type(err)=}") From 245787b89e5bc743503dbeaad63045a050c85bb5 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Sun, 14 Jan 2024 16:59:54 +0100 Subject: [PATCH 19/41] make the connectors form be able to display all types for connectors --- cookbook/admin.py | 10 ++- cookbook/connectors/connector_manager.py | 10 ++- cookbook/connectors/example.py | 27 +++++++ cookbook/forms.py | 57 +++++++++++---- ...0208_homeassistantconfig_exampleconfig.py} | 20 ++++- cookbook/models.py | 5 ++ cookbook/tables.py | 41 ++++++++++- cookbook/templates/base.html | 2 +- cookbook/templates/list_connectors.html | 55 ++++++++++++++ cookbook/urls.py | 7 +- cookbook/views/delete.py | 27 +++---- cookbook/views/edit.py | 73 ++++++++++--------- cookbook/views/lists.py | 17 +---- cookbook/views/new.py | 44 +++++++++-- 14 files changed, 302 insertions(+), 93 deletions(-) create mode 100644 cookbook/connectors/example.py rename cookbook/migrations/{0208_homeassistantconfig.py => 0208_homeassistantconfig_exampleconfig.py} (58%) create mode 100644 cookbook/templates/list_connectors.html diff --git a/cookbook/admin.py b/cookbook/admin.py index 3429e63d..64a03173 100644 --- a/cookbook/admin.py +++ b/cookbook/admin.py @@ -16,7 +16,7 @@ from .models import (BookmarkletImport, Comment, CookLog, Food, ImportLog, Ingre ShoppingList, ShoppingListEntry, ShoppingListRecipe, Space, Step, Storage, Supermarket, SupermarketCategory, SupermarketCategoryRelation, Sync, SyncLog, TelegramBot, Unit, UnitConversion, UserFile, UserPreference, UserSpace, - ViewLog, HomeAssistantConfig) + ViewLog, HomeAssistantConfig, ExampleConfig) class CustomUserAdmin(UserAdmin): @@ -103,6 +103,14 @@ class HomeAssistantConfigAdmin(admin.ModelAdmin): admin.site.register(HomeAssistantConfig, HomeAssistantConfigAdmin) +class ExampleConfigAdmin(admin.ModelAdmin): + list_display = ('id', 'name', 'enabled', 'feed_url') + search_fields = ('name',) + + +admin.site.register(ExampleConfig, ExampleConfigAdmin) + + class SyncAdmin(admin.ModelAdmin): list_display = ('storage', 'path', 'active', 'last_checked') search_fields = ('storage__name', 'path') diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 39919597..178c8efb 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -12,14 +12,15 @@ from typing import List, Any, Dict, Optional from django_scopes import scope from cookbook.connectors.connector import Connector +from cookbook.connectors.example import Example from cookbook.connectors.homeassistant import HomeAssistant -from cookbook.models import ShoppingListEntry, Recipe, MealPlan, Space, HomeAssistantConfig, ConnectorConfig +from cookbook.models import ShoppingListEntry, Recipe, MealPlan, Space, HomeAssistantConfig, ExampleConfig multiprocessing.set_start_method('fork') # https://code.djangoproject.com/ticket/31169 QUEUE_MAX_SIZE = 25 REGISTERED_CLASSES: UnionType = ShoppingListEntry | Recipe | MealPlan -CONNECTOR_UPDATE_CLASSES: UnionType = HomeAssistantConfig | ConnectorConfig +CONNECTOR_UPDATE_CLASSES: UnionType = HomeAssistantConfig | ExampleConfig class ActionType(Enum): @@ -97,7 +98,10 @@ class ConnectorManager: loop.run_until_complete(close_connectors(connectors)) with scope(space=space): - connectors: List[Connector] = [HomeAssistant(config) for config in space.homeassistantconfig_set.all() if config.enabled] + connectors: List[Connector] = [ + *(HomeAssistant(config) for config in space.homeassistantconfig_set.all() if config.enabled), + *(Example(config) for config in space.exampleconfig_set.all() if config.enabled) + ] _connectors[space.name] = connectors if len(connectors) == 0 or refresh_connector_cache: diff --git a/cookbook/connectors/example.py b/cookbook/connectors/example.py new file mode 100644 index 00000000..ce55f476 --- /dev/null +++ b/cookbook/connectors/example.py @@ -0,0 +1,27 @@ +from cookbook.connectors.connector import Connector +from cookbook.models import ExampleConfig, Space, ShoppingListEntry + + +class Example(Connector): + _config: ExampleConfig + + def __init__(self, config: ExampleConfig): + self._config = config + + async def on_shopping_list_entry_created(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None: + if not self._config.on_shopping_list_entry_created_enabled: + return + pass + + async def on_shopping_list_entry_updated(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None: + if not self._config.on_shopping_list_entry_updated_enabled: + return + pass + + async def on_shopping_list_entry_deleted(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None: + if not self._config.on_shopping_list_entry_deleted_enabled: + return + pass + + async def close(self) -> None: + pass diff --git a/cookbook/forms.py b/cookbook/forms.py index dce4da57..18867958 100644 --- a/cookbook/forms.py +++ b/cookbook/forms.py @@ -10,7 +10,7 @@ from django_scopes.forms import SafeModelChoiceField, SafeModelMultipleChoiceFie from hcaptcha.fields import hCaptchaField from .models import (Comment, Food, InviteLink, Keyword, Recipe, RecipeBook, RecipeBookEntry, - SearchPreference, Space, Storage, Sync, User, UserPreference, HomeAssistantConfig) + SearchPreference, Space, Storage, Sync, User, UserPreference, HomeAssistantConfig, ExampleConfig) class SelectWidget(widgets.Select): @@ -188,12 +188,35 @@ class StorageForm(forms.ModelForm): } -class HomeAssistantConfigForm(forms.ModelForm): - token = forms.CharField( - widget=forms.TextInput( - attrs={'autocomplete': 'new-password', 'type': 'password'} - ), - required=True, +class ConnectorConfigForm(forms.ModelForm): + enabled = forms.BooleanField( + help_text="Is the connector enabled", + required=False, + ) + + on_shopping_list_entry_created_enabled = forms.BooleanField( + help_text="Enable action for ShoppingListEntry created events", + required=False, + ) + + on_shopping_list_entry_updated_enabled = forms.BooleanField( + help_text="Enable action for ShoppingListEntry updated events", + required=False, + ) + + on_shopping_list_entry_deleted_enabled = forms.BooleanField( + help_text="Enable action for ShoppingListEntry deleted events", + required=False, + ) + + class Meta: + fields = ('name', 'enabled', 'on_shopping_list_entry_created_enabled', 'on_shopping_list_entry_updated_enabled', 'on_shopping_list_entry_deleted_enabled') + + +class HomeAssistantConfigForm(ConnectorConfigForm): + update_token = forms.CharField( + widget=forms.TextInput(attrs={'autocomplete': 'update-token', 'type': 'password'}), + required=False, help_text=_('Long Lived Access Token for your HomeAssistant instance') ) @@ -202,11 +225,6 @@ class HomeAssistantConfigForm(forms.ModelForm): help_text=_('Something like http://homeassistant.local:8123/api'), ) - enabled = forms.BooleanField( - help_text="Is the HomeAssistantConnector enabled", - required=False, - ) - on_shopping_list_entry_created_enabled = forms.BooleanField( help_text="Enable syncing ShoppingListEntry to Homeassistant Todo List -- Warning: Might have negative performance impact", required=False, @@ -224,15 +242,24 @@ class HomeAssistantConfigForm(forms.ModelForm): class Meta: model = HomeAssistantConfig - fields = ( - 'name', 'url', 'token', 'todo_entity', 'enabled', 'on_shopping_list_entry_created_enabled', 'on_shopping_list_entry_updated_enabled', - 'on_shopping_list_entry_deleted_enabled') + + fields = ConnectorConfigForm.Meta.fields + ('url', 'todo_entity') help_texts = { 'url': _('http://homeassistant.local:8123/api for example'), } +class ExampleConfigForm(ConnectorConfigForm): + feed_url = forms.URLField( + required=False, + ) + + class Meta: + model = ExampleConfig + fields = ConnectorConfigForm.Meta.fields + ('feed_url',) + + # TODO: Deprecate class RecipeBookEntryForm(forms.ModelForm): prefix = 'bookmark' diff --git a/cookbook/migrations/0208_homeassistantconfig.py b/cookbook/migrations/0208_homeassistantconfig_exampleconfig.py similarity index 58% rename from cookbook/migrations/0208_homeassistantconfig.py rename to cookbook/migrations/0208_homeassistantconfig_exampleconfig.py index f21ff06a..789094c1 100644 --- a/cookbook/migrations/0208_homeassistantconfig.py +++ b/cookbook/migrations/0208_homeassistantconfig_exampleconfig.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.7 on 2024-01-12 21:01 +# Generated by Django 4.2.7 on 2024-01-14 16:00 import cookbook.models from django.conf import settings @@ -35,4 +35,22 @@ class Migration(migrations.Migration): }, bases=(models.Model, cookbook.models.PermissionModelMixin), ), + migrations.CreateModel( + name='ExampleConfig', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=128, validators=[django.core.validators.MinLengthValidator(1)])), + ('enabled', models.BooleanField(default=True, help_text='Is Connector Enabled')), + ('on_shopping_list_entry_created_enabled', models.BooleanField(default=False)), + ('on_shopping_list_entry_updated_enabled', models.BooleanField(default=False)), + ('on_shopping_list_entry_deleted_enabled', models.BooleanField(default=False)), + ('feed_url', models.URLField(blank=True)), + ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)), + ('space', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='cookbook.space')), + ], + options={ + 'abstract': False, + }, + bases=(models.Model, cookbook.models.PermissionModelMixin), + ), ] diff --git a/cookbook/models.py b/cookbook/models.py index 9d6b9c5a..a41dc37f 100644 --- a/cookbook/models.py +++ b/cookbook/models.py @@ -340,6 +340,7 @@ class Space(ExportModelOperationsMixin('space'), models.Model): Sync.objects.filter(space=self).delete() Storage.objects.filter(space=self).delete() HomeAssistantConfig.objects.filter(space=self).delete() + ExampleConfig.objects.filter(space=self).delete() ShoppingListEntry.objects.filter(shoppinglist__space=self).delete() ShoppingListRecipe.objects.filter(shoppinglist__space=self).delete() @@ -387,6 +388,10 @@ class HomeAssistantConfig(ConnectorConfig): todo_entity = models.CharField(max_length=128, default='todo.shopping_list') +class ExampleConfig(ConnectorConfig): + feed_url = models.URLField(blank=True) + + class UserPreference(models.Model, PermissionModelMixin): # Themes BOOTSTRAP = 'BOOTSTRAP' diff --git a/cookbook/tables.py b/cookbook/tables.py index 4f49690c..ba14fbbe 100644 --- a/cookbook/tables.py +++ b/cookbook/tables.py @@ -1,9 +1,14 @@ +from typing import Any, Dict + import django_tables2 as tables from django.utils.html import format_html from django.utils.translation import gettext as _ +from django.views.generic import TemplateView +from django_tables2 import MultiTableMixin from django_tables2.utils import A -from .models import CookLog, InviteLink, RecipeImport, Storage, Sync, SyncLog, ViewLog, HomeAssistantConfig +from .helper.permission_helper import GroupRequiredMixin +from .models import CookLog, InviteLink, RecipeImport, Storage, Sync, SyncLog, ViewLog, HomeAssistantConfig, ExampleConfig class StorageTable(tables.Table): @@ -15,6 +20,16 @@ class StorageTable(tables.Table): fields = ('id', 'name', 'method') +class ExampleConfigTable(tables.Table): + id = tables.LinkColumn('edit_example_config', args=[A('id')]) + + class Meta: + model = ExampleConfig + template_name = 'generic/table_template.html' + fields = ('id', 'name', 'enabled', 'feed_url') + attrs = {'table_name': "Example Configs", 'create_url': 'new_example_config'} + + class HomeAssistantConfigTable(tables.Table): id = tables.LinkColumn('edit_home_assistant_config', args=[A('id')]) @@ -22,6 +37,30 @@ class HomeAssistantConfigTable(tables.Table): model = HomeAssistantConfig template_name = 'generic/table_template.html' fields = ('id', 'name', 'enabled', 'url') + attrs = {'table_name': "HomeAssistant Configs", 'create_url': 'new_home_assistant_config'} + + +class ConnectorConfigTable(GroupRequiredMixin, MultiTableMixin, TemplateView): + groups_required = ['admin'] + template_name = "list_connectors.html" + + table_pagination = { + "per_page": 25 + } + + def get_context_data(self, **kwargs: Any) -> Dict[str, Any]: + kwargs = super().get_context_data(**kwargs) + kwargs['title'] = _("Connectors") + return kwargs + + def get_tables(self): + example_configs = ExampleConfig.objects.filter(space=self.request.space).all() + home_assistant_configs = HomeAssistantConfig.objects.filter(space=self.request.space).all() + + return [ + ExampleConfigTable(example_configs), + HomeAssistantConfigTable(home_assistant_configs) + ] class ImportLogTable(tables.Table): diff --git a/cookbook/templates/base.html b/cookbook/templates/base.html index d9e2eda1..7fbb14db 100644 --- a/cookbook/templates/base.html +++ b/cookbook/templates/base.html @@ -336,7 +336,7 @@ class="fas fa-server fa-fw"> {% trans 'Space Settings' %} {% endif %} {% if request.user == request.space.created_by or user.is_superuser %} - {% trans 'External Connectors' %} {% endif %} {% if user.is_superuser %} diff --git a/cookbook/templates/list_connectors.html b/cookbook/templates/list_connectors.html new file mode 100644 index 00000000..283f7c31 --- /dev/null +++ b/cookbook/templates/list_connectors.html @@ -0,0 +1,55 @@ +{% extends "base.html" %} +{% load crispy_forms_tags %} +{% load i18n %} +{% load django_tables2 %} + +{% block title %}{% trans 'List' %}{% endblock %} + + +{% block content %} + +{% if request.resolver_match.url_name in 'list_storage,list_recipe_import,list_sync_log' %} + +{% endif %} + +
+ +

{{ title }} {% trans 'List' %}

+ + + {% if filter %} +
+
+
+ {% csrf_token %} + {{ filter.form|crispy }} + +
+ {% endif %} + + {% if import_btn %} + {% trans 'Import all' %} +
+
+ {% endif %} + + {% for table in tables %} + +

{{ table.attrs.table_name }} {% trans 'List' %} + {% if table.attrs.create_url %} + + {% endif %} +

+ + + {% render_table table %} + {% endfor %} + +
+ +{% endblock content %} \ No newline at end of file diff --git a/cookbook/urls.py b/cookbook/urls.py index 96735037..e85cfbaf 100644 --- a/cookbook/urls.py +++ b/cookbook/urls.py @@ -12,7 +12,8 @@ from recipes.settings import DEBUG, PLUGINS from .models import (Automation, Comment, CustomFilter, Food, InviteLink, Keyword, PropertyType, Recipe, RecipeBook, RecipeBookEntry, RecipeImport, ShoppingList, Space, Step, Storage, Supermarket, SupermarketCategory, Sync, SyncLog, Unit, UnitConversion, - UserFile, UserSpace, get_model_name, HomeAssistantConfig) + UserFile, UserSpace, get_model_name, HomeAssistantConfig, ExampleConfig) +from .tables import ConnectorConfigTable from .views import api, data, delete, edit, import_export, lists, new, telegram, views from .views.api import CustomAuthToken, ImportOpenData @@ -115,7 +116,7 @@ urlpatterns = [ path('edit/recipe/convert//', edit.convert_recipe, name='edit_convert_recipe'), path('edit/storage//', edit.edit_storage, name='edit_storage'), - path('edit/home-assistant-config//', edit.edit_home_assistant_config, name='edit_home_assistant_config'), + path('list/connectors', ConnectorConfigTable.as_view(), name='list_connectors'), path('delete/recipe-source//', delete.delete_recipe_source, name='delete_recipe_source'), @@ -168,7 +169,7 @@ urlpatterns = [ ] generic_models = ( - Recipe, RecipeImport, Storage, HomeAssistantConfig, RecipeBook, SyncLog, Sync, + Recipe, RecipeImport, Storage, HomeAssistantConfig, ExampleConfig, RecipeBook, SyncLog, Sync, Comment, RecipeBookEntry, ShoppingList, InviteLink, UserSpace, Space ) diff --git a/cookbook/views/delete.py b/cookbook/views/delete.py index b6cf857b..bbdc98ee 100644 --- a/cookbook/views/delete.py +++ b/cookbook/views/delete.py @@ -9,7 +9,7 @@ from django.views.generic import DeleteView from cookbook.helper.permission_helper import GroupRequiredMixin, OwnerRequiredMixin, group_required from cookbook.models import (Comment, InviteLink, MealPlan, Recipe, RecipeBook, RecipeBookEntry, - RecipeImport, Space, Storage, Sync, UserSpace, HomeAssistantConfig) + RecipeImport, Space, Storage, Sync, UserSpace, HomeAssistantConfig, ExampleConfig) from cookbook.provider.dropbox import Dropbox from cookbook.provider.local import Local from cookbook.provider.nextcloud import Nextcloud @@ -126,23 +126,24 @@ class HomeAssistantConfigDelete(GroupRequiredMixin, DeleteView): groups_required = ['admin'] template_name = "generic/delete_template.html" model = HomeAssistantConfig - success_url = reverse_lazy('list_storage') + success_url = reverse_lazy('list_connectors') def get_context_data(self, **kwargs): - context = super(HomeAssistantConfigDelete, self).get_context_data(**kwargs) + context = super().get_context_data(**kwargs) context['title'] = _("HomeAssistant Config Backend") return context - def post(self, request, *args, **kwargs): - try: - return self.delete(request, *args, **kwargs) - except ProtectedError: - messages.add_message( - request, - messages.WARNING, - _('Could not delete this storage backend as it is used in at least one monitor.') # noqa: E501 - ) - return HttpResponseRedirect(reverse('list_storage')) + +class ExampleConfigDelete(GroupRequiredMixin, DeleteView): + groups_required = ['admin'] + template_name = "generic/delete_template.html" + model = ExampleConfig + success_url = reverse_lazy('list_connectors') + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context['title'] = _("Example Config Backend") + return context class CommentDelete(OwnerRequiredMixin, DeleteView): diff --git a/cookbook/views/edit.py b/cookbook/views/edit.py index deb9654b..2f99216b 100644 --- a/cookbook/views/edit.py +++ b/cookbook/views/edit.py @@ -9,10 +9,10 @@ from django.utils.translation import gettext as _ from django.views.generic import UpdateView from django.views.generic.edit import FormMixin -from cookbook.forms import CommentForm, ExternalRecipeForm, StorageForm, SyncForm, HomeAssistantConfigForm +from cookbook.forms import CommentForm, ExternalRecipeForm, StorageForm, SyncForm, HomeAssistantConfigForm, ExampleConfigForm from cookbook.helper.permission_helper import (GroupRequiredMixin, OwnerRequiredMixin, above_space_limit, group_required) -from cookbook.models import Comment, Recipe, RecipeImport, Storage, Sync, HomeAssistantConfig +from cookbook.models import Comment, Recipe, RecipeImport, Storage, Sync, HomeAssistantConfig, ExampleConfig from cookbook.provider.dropbox import Dropbox from cookbook.provider.local import Local from cookbook.provider.nextcloud import Nextcloud @@ -128,46 +128,49 @@ def edit_storage(request, pk): ) -@group_required('admin') -def edit_home_assistant_config(request, pk): - instance: HomeAssistantConfig = get_object_or_404(HomeAssistantConfig, pk=pk, space=request.space) +class HomeAssistantConfigUpdate(GroupRequiredMixin, UpdateView): + groups_required = ['admin'] + template_name = "generic/edit_template.html" + model = HomeAssistantConfig + form_class = HomeAssistantConfigForm - if not (instance.created_by == request.user or request.user.is_superuser): - messages.add_message(request, messages.ERROR, _('You cannot edit this homeassistant config!')) - return HttpResponseRedirect(reverse('edit_home_assistant_config')) + def get_form_kwargs(self): + kwargs = super().get_form_kwargs() + kwargs['initial']['update_token'] = VALUE_NOT_CHANGED + return kwargs - if request.space.demo or settings.HOSTED: - messages.add_message(request, messages.ERROR, _('This feature is not yet available in the hosted version of tandoor!')) - return redirect('index') + def form_valid(self, form): + if form.cleaned_data['update_token'] != VALUE_NOT_CHANGED and form.cleaned_data['update_token'] != "": + form.instance.token = form.cleaned_data['update_token'] + messages.add_message(self.request, messages.SUCCESS, _('Config saved!')) + return super(HomeAssistantConfigUpdate, self).form_valid(form) - if request.method == "POST": - form = HomeAssistantConfigForm(request.POST, instance=copy.deepcopy(instance)) - if form.is_valid(): - instance.name = form.cleaned_data['name'] - instance.url = form.cleaned_data['url'] - instance.todo_entity = form.cleaned_data['todo_entity'] - instance.enabled = form.cleaned_data['enabled'] - instance.on_shopping_list_entry_created_enabled = form.cleaned_data['on_shopping_list_entry_created_enabled'] - instance.on_shopping_list_entry_updated_enabled = form.cleaned_data['on_shopping_list_entry_updated_enabled'] - instance.on_shopping_list_entry_deleted_enabled = form.cleaned_data['on_shopping_list_entry_deleted_enabled'] + def get_success_url(self): + return reverse('edit_home_assistant_config', kwargs={'pk': self.object.pk}) - if form.cleaned_data['token'] != VALUE_NOT_CHANGED: - instance.token = form.cleaned_data['token'] + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context['title'] = _("HomeAssistantConfig") + return context - instance.save() - messages.add_message(request, messages.SUCCESS, _('HomeAssistant config saved!')) - else: - messages.add_message(request, messages.ERROR, _('There was an error updating this config!')) - else: - instance.token = VALUE_NOT_CHANGED - form = HomeAssistantConfigForm(instance=instance) +class ExampleConfigUpdate(GroupRequiredMixin, UpdateView): + groups_required = ['admin'] + template_name = "generic/edit_template.html" + model = ExampleConfig + form_class = ExampleConfigForm - return render( - request, - 'generic/edit_template.html', - {'form': form, 'title': _('HomeAssistantConfig')} - ) + def form_valid(self, form): + messages.add_message(self.request, messages.SUCCESS, _('Config saved!')) + return super().form_valid(form) + + def get_success_url(self): + return reverse('edit_example_config', kwargs={'pk': self.object.pk}) + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context['title'] = _("ExampleConfig") + return context class CommentUpdate(OwnerRequiredMixin, UpdateView): diff --git a/cookbook/views/lists.py b/cookbook/views/lists.py index 20dbb73c..ed43c143 100644 --- a/cookbook/views/lists.py +++ b/cookbook/views/lists.py @@ -6,8 +6,8 @@ from django.utils.translation import gettext as _ from django_tables2 import RequestConfig from cookbook.helper.permission_helper import group_required -from cookbook.models import InviteLink, RecipeImport, Storage, SyncLog, UserFile, HomeAssistantConfig -from cookbook.tables import ImportLogTable, InviteLinkTable, RecipeImportTable, StorageTable, HomeAssistantConfigTable +from cookbook.models import InviteLink, RecipeImport, Storage, SyncLog, UserFile +from cookbook.tables import ImportLogTable, InviteLinkTable, RecipeImportTable, StorageTable @group_required('admin') @@ -65,19 +65,6 @@ def storage(request): ) -@group_required('admin') -def home_assistant_config(request): - table = HomeAssistantConfigTable(HomeAssistantConfig.objects.filter(space=request.space).all()) - RequestConfig(request, paginate={'per_page': 25}).configure(table) - - return render( - request, 'generic/list_template.html', { - 'title': _("HomeAssistant Config Backend"), - 'table': table, - 'create_url': 'new_home_assistant_config' - }) - - @group_required('admin') def invite_link(request): table = InviteLinkTable( diff --git a/cookbook/views/new.py b/cookbook/views/new.py index 263dca09..22bfbc84 100644 --- a/cookbook/views/new.py +++ b/cookbook/views/new.py @@ -5,9 +5,9 @@ from django.urls import reverse, reverse_lazy from django.utils.translation import gettext as _ from django.views.generic import CreateView -from cookbook.forms import ImportRecipeForm, Storage, StorageForm, HomeAssistantConfigForm +from cookbook.forms import ImportRecipeForm, Storage, StorageForm, HomeAssistantConfigForm, ExampleConfigForm from cookbook.helper.permission_helper import GroupRequiredMixin, above_space_limit, group_required -from cookbook.models import Recipe, RecipeImport, ShareLink, Step, HomeAssistantConfig +from cookbook.models import Recipe, RecipeImport, ShareLink, Step, HomeAssistantConfig, ExampleConfig from recipes import settings @@ -77,6 +77,40 @@ class HomeAssistantConfigCreate(GroupRequiredMixin, CreateView): form_class = HomeAssistantConfigForm success_url = reverse_lazy('list_home_assistant_config') + def get_form_class(self): + form_class = super().get_form_class() + + if self.request.method == 'GET': + update_token_field = form_class.base_fields['update_token'] + update_token_field.required = True + + return form_class + + def form_valid(self, form): + if self.request.space.demo or settings.HOSTED: + messages.add_message(self.request, messages.ERROR, _('This feature is not yet available in the hosted version of tandoor!')) + return redirect('index') + + obj = form.save(commit=False) + obj.token = form.cleaned_data['update_token'] + obj.created_by = self.request.user + obj.space = self.request.space + obj.save() + return HttpResponseRedirect(reverse('edit_home_assistant_config', kwargs={'pk': obj.pk})) + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context['title'] = _("HomeAssistant Config Backend") + return context + + +class ExampleConfigCreate(GroupRequiredMixin, CreateView): + groups_required = ['admin'] + template_name = "generic/new_template.html" + model = ExampleConfig + form_class = ExampleConfigForm + success_url = reverse_lazy('list_connectors') + def form_valid(self, form): if self.request.space.demo or settings.HOSTED: messages.add_message(self.request, messages.ERROR, _('This feature is not yet available in the hosted version of tandoor!')) @@ -86,11 +120,11 @@ class HomeAssistantConfigCreate(GroupRequiredMixin, CreateView): obj.created_by = self.request.user obj.space = self.request.space obj.save() - return HttpResponseRedirect(reverse('edit_home_assistant_config', kwargs={'pk': obj.pk})) + return HttpResponseRedirect(reverse('edit_example_config', kwargs={'pk': obj.pk})) def get_context_data(self, **kwargs): - context = super(HomeAssistantConfigCreate, self).get_context_data(**kwargs) - context['title'] = _("HomeAssistant Config Backend") + context = super().get_context_data(**kwargs) + context['title'] = _("Example Config Backend") return context From 409c0295ec7e2b2c51812773f7e1c0f5a3ca9bbf Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Wed, 17 Jan 2024 22:25:02 +0100 Subject: [PATCH 20/41] convert example & homeassistant specific configs to a generic with all optional fields --- cookbook/admin.py | 16 ++---- cookbook/connectors/connector.py | 7 ++- cookbook/connectors/connector_manager.py | 25 ++++++--- cookbook/connectors/example.py | 27 --------- cookbook/connectors/homeassistant.py | 9 ++- cookbook/forms.py | 41 +++----------- cookbook/migrations/0208_connectorconfig.py | 36 ++++++++++++ .../0208_homeassistantconfig_exampleconfig.py | 56 ------------------- cookbook/models.py | 25 ++++----- cookbook/serializer.py | 6 +- cookbook/tables.py | 49 ++-------------- cookbook/templates/base.html | 2 +- .../api/test_api_home_assistant_config.py | 12 ++-- ...nfig.py => test_edits_connector_config.py} | 19 ++++--- cookbook/urls.py | 8 +-- cookbook/views/api.py | 11 ++-- cookbook/views/delete.py | 22 ++------ cookbook/views/edit.py | 35 +++--------- cookbook/views/lists.py | 20 ++++++- cookbook/views/new.py | 49 +++------------- 20 files changed, 159 insertions(+), 316 deletions(-) delete mode 100644 cookbook/connectors/example.py create mode 100644 cookbook/migrations/0208_connectorconfig.py delete mode 100644 cookbook/migrations/0208_homeassistantconfig_exampleconfig.py rename cookbook/tests/edits/{test_edits_home_assistant_config.py => test_edits_connector_config.py} (77%) diff --git a/cookbook/admin.py b/cookbook/admin.py index 64a03173..fcc007bc 100644 --- a/cookbook/admin.py +++ b/cookbook/admin.py @@ -16,7 +16,7 @@ from .models import (BookmarkletImport, Comment, CookLog, Food, ImportLog, Ingre ShoppingList, ShoppingListEntry, ShoppingListRecipe, Space, Step, Storage, Supermarket, SupermarketCategory, SupermarketCategoryRelation, Sync, SyncLog, TelegramBot, Unit, UnitConversion, UserFile, UserPreference, UserSpace, - ViewLog, HomeAssistantConfig, ExampleConfig) + ViewLog, ConnectorConfig) class CustomUserAdmin(UserAdmin): @@ -95,20 +95,12 @@ class StorageAdmin(admin.ModelAdmin): admin.site.register(Storage, StorageAdmin) -class HomeAssistantConfigAdmin(admin.ModelAdmin): - list_display = ('id', 'name', 'enabled', 'url') +class ConnectorConfigAdmin(admin.ModelAdmin): + list_display = ('id', 'name', 'type', 'enabled', 'url') search_fields = ('name', 'url') -admin.site.register(HomeAssistantConfig, HomeAssistantConfigAdmin) - - -class ExampleConfigAdmin(admin.ModelAdmin): - list_display = ('id', 'name', 'enabled', 'feed_url') - search_fields = ('name',) - - -admin.site.register(ExampleConfig, ExampleConfigAdmin) +admin.site.register(ConnectorConfig, ConnectorConfigAdmin) class SyncAdmin(admin.ModelAdmin): diff --git a/cookbook/connectors/connector.py b/cookbook/connectors/connector.py index 4ed61350..3647dc71 100644 --- a/cookbook/connectors/connector.py +++ b/cookbook/connectors/connector.py @@ -1,9 +1,13 @@ from abc import ABC, abstractmethod -from cookbook.models import ShoppingListEntry, Space +from cookbook.models import ShoppingListEntry, Space, ConnectorConfig class Connector(ABC): + @abstractmethod + def __init__(self, config: ConnectorConfig): + pass + @abstractmethod async def on_shopping_list_entry_created(self, space: Space, instance: ShoppingListEntry) -> None: pass @@ -20,5 +24,4 @@ class Connector(ABC): async def close(self) -> None: pass - # TODO: Maybe add an 'IsEnabled(self) -> Bool' to here # TODO: Add Recipes & possibly Meal Place listeners/hooks (And maybe more?) diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 178c8efb..86a60b04 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -12,15 +12,13 @@ from typing import List, Any, Dict, Optional from django_scopes import scope from cookbook.connectors.connector import Connector -from cookbook.connectors.example import Example from cookbook.connectors.homeassistant import HomeAssistant -from cookbook.models import ShoppingListEntry, Recipe, MealPlan, Space, HomeAssistantConfig, ExampleConfig +from cookbook.models import ShoppingListEntry, Recipe, MealPlan, Space, ConnectorConfig multiprocessing.set_start_method('fork') # https://code.djangoproject.com/ticket/31169 QUEUE_MAX_SIZE = 25 REGISTERED_CLASSES: UnionType = ShoppingListEntry | Recipe | MealPlan -CONNECTOR_UPDATE_CLASSES: UnionType = HomeAssistantConfig | ExampleConfig class ActionType(Enum): @@ -37,7 +35,7 @@ class Work: class ConnectorManager: _queue: JoinableQueue - _listening_to_classes = REGISTERED_CLASSES | CONNECTOR_UPDATE_CLASSES + _listening_to_classes = REGISTERED_CLASSES | ConnectorConfig def __init__(self): self._queue = multiprocessing.JoinableQueue(maxsize=QUEUE_MAX_SIZE) @@ -88,7 +86,7 @@ class ConnectorManager: break # If a Connector was changed/updated, refresh connector from the database for said space - refresh_connector_cache = isinstance(item.instance, CONNECTOR_UPDATE_CLASSES) + refresh_connector_cache = isinstance(item.instance, ConnectorConfig) space: Space = item.instance.space connectors: Optional[List[Connector]] = _connectors.get(space.name) @@ -98,10 +96,11 @@ class ConnectorManager: loop.run_until_complete(close_connectors(connectors)) with scope(space=space): - connectors: List[Connector] = [ - *(HomeAssistant(config) for config in space.homeassistantconfig_set.all() if config.enabled), - *(Example(config) for config in space.exampleconfig_set.all() if config.enabled) - ] + connectors: List[Connector] = list( + filter( + lambda x: x is not None, + [ConnectorManager.get_connected_for_config(config) for config in space.connectorconfig_set.all() if config.enabled], + )) _connectors[space.name] = connectors if len(connectors) == 0 or refresh_connector_cache: @@ -113,6 +112,14 @@ class ConnectorManager: loop.close() + @staticmethod + def get_connected_for_config(config: ConnectorConfig) -> Optional[Connector]: + match config.type: + case ConnectorConfig.HOMEASSISTANT: + return HomeAssistant(config) + case _: + return None + async def close_connectors(connectors: List[Connector]): tasks: List[Task] = [asyncio.create_task(connector.close()) for connector in connectors] diff --git a/cookbook/connectors/example.py b/cookbook/connectors/example.py deleted file mode 100644 index ce55f476..00000000 --- a/cookbook/connectors/example.py +++ /dev/null @@ -1,27 +0,0 @@ -from cookbook.connectors.connector import Connector -from cookbook.models import ExampleConfig, Space, ShoppingListEntry - - -class Example(Connector): - _config: ExampleConfig - - def __init__(self, config: ExampleConfig): - self._config = config - - async def on_shopping_list_entry_created(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None: - if not self._config.on_shopping_list_entry_created_enabled: - return - pass - - async def on_shopping_list_entry_updated(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None: - if not self._config.on_shopping_list_entry_updated_enabled: - return - pass - - async def on_shopping_list_entry_deleted(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None: - if not self._config.on_shopping_list_entry_deleted_enabled: - return - pass - - async def close(self) -> None: - pass diff --git a/cookbook/connectors/homeassistant.py b/cookbook/connectors/homeassistant.py index e394e974..e653c3f3 100644 --- a/cookbook/connectors/homeassistant.py +++ b/cookbook/connectors/homeassistant.py @@ -4,16 +4,19 @@ from logging import Logger from homeassistant_api import Client, HomeassistantAPIError, Domain from cookbook.connectors.connector import Connector -from cookbook.models import ShoppingListEntry, HomeAssistantConfig, Space +from cookbook.models import ShoppingListEntry, ConnectorConfig, Space class HomeAssistant(Connector): _domains_cache: dict[str, Domain] - _config: HomeAssistantConfig + _config: ConnectorConfig _logger: Logger _client: Client - def __init__(self, config: HomeAssistantConfig): + def __init__(self, config: ConnectorConfig): + if not config.token or not config.url or not config.todo_entity: + raise ValueError("config for HomeAssistantConnector in incomplete") + self._domains_cache = dict() self._config = config self._logger = logging.getLogger("connector.HomeAssistant") diff --git a/cookbook/forms.py b/cookbook/forms.py index 18867958..2244acad 100644 --- a/cookbook/forms.py +++ b/cookbook/forms.py @@ -10,7 +10,7 @@ from django_scopes.forms import SafeModelChoiceField, SafeModelMultipleChoiceFie from hcaptcha.fields import hCaptchaField from .models import (Comment, Food, InviteLink, Keyword, Recipe, RecipeBook, RecipeBookEntry, - SearchPreference, Space, Storage, Sync, User, UserPreference, HomeAssistantConfig, ExampleConfig) + SearchPreference, Space, Storage, Sync, User, UserPreference, ConnectorConfig) class SelectWidget(widgets.Select): @@ -209,11 +209,6 @@ class ConnectorConfigForm(forms.ModelForm): required=False, ) - class Meta: - fields = ('name', 'enabled', 'on_shopping_list_entry_created_enabled', 'on_shopping_list_entry_updated_enabled', 'on_shopping_list_entry_deleted_enabled') - - -class HomeAssistantConfigForm(ConnectorConfigForm): update_token = forms.CharField( widget=forms.TextInput(attrs={'autocomplete': 'update-token', 'type': 'password'}), required=False, @@ -221,45 +216,23 @@ class HomeAssistantConfigForm(ConnectorConfigForm): ) url = forms.URLField( - required=True, + required=False, help_text=_('Something like http://homeassistant.local:8123/api'), ) - on_shopping_list_entry_created_enabled = forms.BooleanField( - help_text="Enable syncing ShoppingListEntry to Homeassistant Todo List -- Warning: Might have negative performance impact", - required=False, - ) - - on_shopping_list_entry_updated_enabled = forms.BooleanField( - help_text="PLACEHOLDER", - required=False, - ) - - on_shopping_list_entry_deleted_enabled = forms.BooleanField( - help_text="Enable syncing ShoppingListEntry deletion to Homeassistant Todo List -- Warning: Might have negative performance impact", - required=False, - ) - class Meta: - model = HomeAssistantConfig + model = ConnectorConfig - fields = ConnectorConfigForm.Meta.fields + ('url', 'todo_entity') + fields = ( + 'name', 'type', 'enabled', 'on_shopping_list_entry_created_enabled', 'on_shopping_list_entry_updated_enabled', + 'on_shopping_list_entry_deleted_enabled', 'url', 'todo_entity', + ) help_texts = { 'url': _('http://homeassistant.local:8123/api for example'), } -class ExampleConfigForm(ConnectorConfigForm): - feed_url = forms.URLField( - required=False, - ) - - class Meta: - model = ExampleConfig - fields = ConnectorConfigForm.Meta.fields + ('feed_url',) - - # TODO: Deprecate class RecipeBookEntryForm(forms.ModelForm): prefix = 'bookmark' diff --git a/cookbook/migrations/0208_connectorconfig.py b/cookbook/migrations/0208_connectorconfig.py new file mode 100644 index 00000000..c29c0253 --- /dev/null +++ b/cookbook/migrations/0208_connectorconfig.py @@ -0,0 +1,36 @@ +# Generated by Django 4.2.7 on 2024-01-17 21:12 + +import cookbook.models +from django.conf import settings +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('cookbook', '0207_space_logo_color_128_space_logo_color_144_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='ConnectorConfig', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=128, validators=[django.core.validators.MinLengthValidator(1)])), + ('type', models.CharField(choices=[('HomeAssistant', 'HomeAssistant')], default='HomeAssistant', max_length=128)), + ('enabled', models.BooleanField(default=True, help_text='Is Connector Enabled')), + ('on_shopping_list_entry_created_enabled', models.BooleanField(default=False)), + ('on_shopping_list_entry_updated_enabled', models.BooleanField(default=False)), + ('on_shopping_list_entry_deleted_enabled', models.BooleanField(default=False)), + ('url', models.URLField(blank=True, null=True)), + ('token', models.CharField(blank=True, max_length=512, null=True)), + ('todo_entity', models.CharField(blank=True, max_length=128, null=True)), + ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)), + ('space', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='cookbook.space')), + ], + bases=(models.Model, cookbook.models.PermissionModelMixin), + ), + ] diff --git a/cookbook/migrations/0208_homeassistantconfig_exampleconfig.py b/cookbook/migrations/0208_homeassistantconfig_exampleconfig.py deleted file mode 100644 index 789094c1..00000000 --- a/cookbook/migrations/0208_homeassistantconfig_exampleconfig.py +++ /dev/null @@ -1,56 +0,0 @@ -# Generated by Django 4.2.7 on 2024-01-14 16:00 - -import cookbook.models -from django.conf import settings -import django.core.validators -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('cookbook', '0207_space_logo_color_128_space_logo_color_144_and_more'), - ] - - operations = [ - migrations.CreateModel( - name='HomeAssistantConfig', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=128, validators=[django.core.validators.MinLengthValidator(1)])), - ('enabled', models.BooleanField(default=True, help_text='Is Connector Enabled')), - ('on_shopping_list_entry_created_enabled', models.BooleanField(default=False)), - ('on_shopping_list_entry_updated_enabled', models.BooleanField(default=False)), - ('on_shopping_list_entry_deleted_enabled', models.BooleanField(default=False)), - ('url', models.URLField(blank=True)), - ('token', models.CharField(blank=True, max_length=512)), - ('todo_entity', models.CharField(default='todo.shopping_list', max_length=128)), - ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)), - ('space', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='cookbook.space')), - ], - options={ - 'abstract': False, - }, - bases=(models.Model, cookbook.models.PermissionModelMixin), - ), - migrations.CreateModel( - name='ExampleConfig', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=128, validators=[django.core.validators.MinLengthValidator(1)])), - ('enabled', models.BooleanField(default=True, help_text='Is Connector Enabled')), - ('on_shopping_list_entry_created_enabled', models.BooleanField(default=False)), - ('on_shopping_list_entry_updated_enabled', models.BooleanField(default=False)), - ('on_shopping_list_entry_deleted_enabled', models.BooleanField(default=False)), - ('feed_url', models.URLField(blank=True)), - ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)), - ('space', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='cookbook.space')), - ], - options={ - 'abstract': False, - }, - bases=(models.Model, cookbook.models.PermissionModelMixin), - ), - ] diff --git a/cookbook/models.py b/cookbook/models.py index a41dc37f..f0bc1af6 100644 --- a/cookbook/models.py +++ b/cookbook/models.py @@ -339,8 +339,7 @@ class Space(ExportModelOperationsMixin('space'), models.Model): SyncLog.objects.filter(sync__space=self).delete() Sync.objects.filter(space=self).delete() Storage.objects.filter(space=self).delete() - HomeAssistantConfig.objects.filter(space=self).delete() - ExampleConfig.objects.filter(space=self).delete() + ConnectorConfig.objects.filter(space=self).delete() ShoppingListEntry.objects.filter(shoppinglist__space=self).delete() ShoppingListRecipe.objects.filter(shoppinglist__space=self).delete() @@ -366,30 +365,28 @@ class Space(ExportModelOperationsMixin('space'), models.Model): class ConnectorConfig(models.Model, PermissionModelMixin): + HOMEASSISTANT = 'HomeAssistant' + CONNECTER_TYPE = ((HOMEASSISTANT, 'HomeAssistant'),) + name = models.CharField(max_length=128, validators=[MinLengthValidator(1)]) + type = models.CharField( + choices=CONNECTER_TYPE, max_length=128, default=HOMEASSISTANT + ) enabled = models.BooleanField(default=True, help_text="Is Connector Enabled") on_shopping_list_entry_created_enabled = models.BooleanField(default=False) on_shopping_list_entry_updated_enabled = models.BooleanField(default=False) on_shopping_list_entry_deleted_enabled = models.BooleanField(default=False) + url = models.URLField(blank=True, null=True) + token = models.CharField(max_length=512, blank=True, null=True) + todo_entity = models.CharField(max_length=128, blank=True, null=True) + created_by = models.ForeignKey(User, on_delete=models.PROTECT) space = models.ForeignKey(Space, on_delete=models.CASCADE) objects = ScopedManager(space='space') - class Meta: - abstract = True - - -class HomeAssistantConfig(ConnectorConfig): - url = models.URLField(blank=True) - token = models.CharField(max_length=512, blank=True) - todo_entity = models.CharField(max_length=128, default='todo.shopping_list') - - -class ExampleConfig(ConnectorConfig): - feed_url = models.URLField(blank=True) class UserPreference(models.Model, PermissionModelMixin): diff --git a/cookbook/serializer.py b/cookbook/serializer.py index 53fc8eeb..e102c46f 100644 --- a/cookbook/serializer.py +++ b/cookbook/serializer.py @@ -34,7 +34,7 @@ from cookbook.models import (Automation, BookmarkletImport, Comment, CookLog, Cu ShareLink, ShoppingList, ShoppingListEntry, ShoppingListRecipe, Space, Step, Storage, Supermarket, SupermarketCategory, SupermarketCategoryRelation, Sync, SyncLog, Unit, UnitConversion, - UserFile, UserPreference, UserSpace, ViewLog, HomeAssistantConfig) + UserFile, UserPreference, UserSpace, ViewLog, ConnectorConfig) from cookbook.templatetags.custom_tags import markdown from recipes.settings import AWS_ENABLED, MEDIA_URL @@ -413,14 +413,14 @@ class StorageSerializer(SpacedModelSerializer): } -class HomeAssistantConfigSerializer(SpacedModelSerializer): +class ConnectorConfigConfigSerializer(SpacedModelSerializer): def create(self, validated_data): validated_data['created_by'] = self.context['request'].user return super().create(validated_data) class Meta: - model = HomeAssistantConfig + model = ConnectorConfig fields = ( 'id', 'name', 'url', 'token', 'todo_entity', 'enabled', 'on_shopping_list_entry_created_enabled', 'on_shopping_list_entry_updated_enabled', diff --git a/cookbook/tables.py b/cookbook/tables.py index ba14fbbe..3fcec9c3 100644 --- a/cookbook/tables.py +++ b/cookbook/tables.py @@ -1,14 +1,9 @@ -from typing import Any, Dict - import django_tables2 as tables from django.utils.html import format_html from django.utils.translation import gettext as _ -from django.views.generic import TemplateView -from django_tables2 import MultiTableMixin from django_tables2.utils import A -from .helper.permission_helper import GroupRequiredMixin -from .models import CookLog, InviteLink, RecipeImport, Storage, Sync, SyncLog, ViewLog, HomeAssistantConfig, ExampleConfig +from .models import CookLog, InviteLink, RecipeImport, Storage, Sync, SyncLog, ViewLog, ConnectorConfig class StorageTable(tables.Table): @@ -20,47 +15,13 @@ class StorageTable(tables.Table): fields = ('id', 'name', 'method') -class ExampleConfigTable(tables.Table): - id = tables.LinkColumn('edit_example_config', args=[A('id')]) +class ConnectorConfigTable(tables.Table): + id = tables.LinkColumn('edit_connector_config', args=[A('id')]) class Meta: - model = ExampleConfig + model = ConnectorConfig template_name = 'generic/table_template.html' - fields = ('id', 'name', 'enabled', 'feed_url') - attrs = {'table_name': "Example Configs", 'create_url': 'new_example_config'} - - -class HomeAssistantConfigTable(tables.Table): - id = tables.LinkColumn('edit_home_assistant_config', args=[A('id')]) - - class Meta: - model = HomeAssistantConfig - template_name = 'generic/table_template.html' - fields = ('id', 'name', 'enabled', 'url') - attrs = {'table_name': "HomeAssistant Configs", 'create_url': 'new_home_assistant_config'} - - -class ConnectorConfigTable(GroupRequiredMixin, MultiTableMixin, TemplateView): - groups_required = ['admin'] - template_name = "list_connectors.html" - - table_pagination = { - "per_page": 25 - } - - def get_context_data(self, **kwargs: Any) -> Dict[str, Any]: - kwargs = super().get_context_data(**kwargs) - kwargs['title'] = _("Connectors") - return kwargs - - def get_tables(self): - example_configs = ExampleConfig.objects.filter(space=self.request.space).all() - home_assistant_configs = HomeAssistantConfig.objects.filter(space=self.request.space).all() - - return [ - ExampleConfigTable(example_configs), - HomeAssistantConfigTable(home_assistant_configs) - ] + fields = ('id', 'name', 'type', 'enabled') class ImportLogTable(tables.Table): diff --git a/cookbook/templates/base.html b/cookbook/templates/base.html index 7fbb14db..6505b405 100644 --- a/cookbook/templates/base.html +++ b/cookbook/templates/base.html @@ -336,7 +336,7 @@ class="fas fa-server fa-fw"> {% trans 'Space Settings' %} {% endif %} {% if request.user == request.space.created_by or user.is_superuser %} - {% trans 'External Connectors' %} {% endif %} {% if user.is_superuser %} diff --git a/cookbook/tests/api/test_api_home_assistant_config.py b/cookbook/tests/api/test_api_home_assistant_config.py index a3de0f45..79f496bf 100644 --- a/cookbook/tests/api/test_api_home_assistant_config.py +++ b/cookbook/tests/api/test_api_home_assistant_config.py @@ -5,21 +5,21 @@ from django.contrib import auth from django.urls import reverse from django_scopes import scopes_disabled -from cookbook.models import HomeAssistantConfig +from cookbook.models import ConnectorConfig -LIST_URL = 'api:homeassistantconfig-list' -DETAIL_URL = 'api:homeassistantconfig-detail' +LIST_URL = 'api:connectorconfig-list' +DETAIL_URL = 'api:connectorconfig-detail' @pytest.fixture() def obj_1(space_1, u1_s1): - return HomeAssistantConfig.objects.create( + return ConnectorConfig.objects.create( name='HomeAssistant 1', token='token', url='url', todo_entity='todo.shopping_list', enabled=True, created_by=auth.get_user(u1_s1), space=space_1, ) @pytest.fixture def obj_2(space_1, u1_s1): - return HomeAssistantConfig.objects.create( + return ConnectorConfig.objects.create( name='HomeAssistant 2', token='token', url='url', todo_entity='todo.shopping_list', enabled=True, created_by=auth.get_user(u1_s1), space=space_1, ) @@ -123,4 +123,4 @@ def test_delete(a1_s1, a1_s2, obj_1): assert r.status_code == 204 with scopes_disabled(): - assert HomeAssistantConfig.objects.count() == 0 + assert ConnectorConfig.objects.count() == 0 diff --git a/cookbook/tests/edits/test_edits_home_assistant_config.py b/cookbook/tests/edits/test_edits_connector_config.py similarity index 77% rename from cookbook/tests/edits/test_edits_home_assistant_config.py rename to cookbook/tests/edits/test_edits_connector_config.py index 0df507de..aad00f8d 100644 --- a/cookbook/tests/edits/test_edits_home_assistant_config.py +++ b/cookbook/tests/edits/test_edits_connector_config.py @@ -3,16 +3,18 @@ from django.contrib import auth from django.contrib import messages from django.contrib.messages import get_messages from django.urls import reverse +from pytest_django.asserts import assertTemplateUsed -from cookbook.models import HomeAssistantConfig +from cookbook.models import ConnectorConfig -EDIT_VIEW_NAME = 'edit_home_assistant_config' +EDIT_VIEW_NAME = 'edit_connector_config' @pytest.fixture def home_assistant_config_obj(a1_s1, space_1): - return HomeAssistantConfig.objects.create( + return ConnectorConfig.objects.create( name='HomeAssistant 1', + type=ConnectorConfig.HOMEASSISTANT, token='token', url='http://localhost:8123/api', todo_entity='todo.shopping_list', @@ -22,20 +24,22 @@ def home_assistant_config_obj(a1_s1, space_1): ) -def test_edit_home_assistant_config(home_assistant_config_obj: HomeAssistantConfig, a1_s1, a1_s2): +def test_edit_connector_config_homeassistant(home_assistant_config_obj: ConnectorConfig, a1_s1, a1_s2): new_token = '1234_token' r = a1_s1.post( reverse(EDIT_VIEW_NAME, args={home_assistant_config_obj.pk}), { 'name': home_assistant_config_obj.name, + 'type': home_assistant_config_obj.type, 'url': home_assistant_config_obj.url, - 'token': new_token, + 'update_token': new_token, 'todo_entity': home_assistant_config_obj.todo_entity, 'enabled': home_assistant_config_obj.enabled, } ) - assert r.status_code == 200 + assert r.status_code == 302 + r_messages = [m for m in get_messages(r.wsgi_request)] assert not any(m.level > messages.SUCCESS for m in r_messages) @@ -46,9 +50,10 @@ def test_edit_home_assistant_config(home_assistant_config_obj: HomeAssistantConf reverse(EDIT_VIEW_NAME, args={home_assistant_config_obj.pk}), { 'name': home_assistant_config_obj.name, + 'type': home_assistant_config_obj.type, 'url': home_assistant_config_obj.url, 'todo_entity': home_assistant_config_obj.todo_entity, - 'token': new_token, + 'update_token': new_token, 'enabled': home_assistant_config_obj.enabled, } ) diff --git a/cookbook/urls.py b/cookbook/urls.py index e85cfbaf..54d29ae4 100644 --- a/cookbook/urls.py +++ b/cookbook/urls.py @@ -12,8 +12,7 @@ from recipes.settings import DEBUG, PLUGINS from .models import (Automation, Comment, CustomFilter, Food, InviteLink, Keyword, PropertyType, Recipe, RecipeBook, RecipeBookEntry, RecipeImport, ShoppingList, Space, Step, Storage, Supermarket, SupermarketCategory, Sync, SyncLog, Unit, UnitConversion, - UserFile, UserSpace, get_model_name, HomeAssistantConfig, ExampleConfig) -from .tables import ConnectorConfigTable + UserFile, UserSpace, get_model_name, ConnectorConfig) from .views import api, data, delete, edit, import_export, lists, new, telegram, views from .views.api import CustomAuthToken, ImportOpenData @@ -52,7 +51,7 @@ router.register(r'shopping-list-recipe', api.ShoppingListRecipeViewSet) router.register(r'space', api.SpaceViewSet) router.register(r'step', api.StepViewSet) router.register(r'storage', api.StorageViewSet) -router.register(r'home-assistant-config', api.HomeAssistantConfigViewSet) +router.register(r'home-assistant-config', api.ConnectorConfigConfigViewSet) router.register(r'supermarket', api.SupermarketViewSet) router.register(r'supermarket-category', api.SupermarketCategoryViewSet) router.register(r'supermarket-category-relation', api.SupermarketCategoryRelationViewSet) @@ -116,7 +115,6 @@ urlpatterns = [ path('edit/recipe/convert//', edit.convert_recipe, name='edit_convert_recipe'), path('edit/storage//', edit.edit_storage, name='edit_storage'), - path('list/connectors', ConnectorConfigTable.as_view(), name='list_connectors'), path('delete/recipe-source//', delete.delete_recipe_source, name='delete_recipe_source'), @@ -169,7 +167,7 @@ urlpatterns = [ ] generic_models = ( - Recipe, RecipeImport, Storage, HomeAssistantConfig, ExampleConfig, RecipeBook, SyncLog, Sync, + Recipe, RecipeImport, Storage, ConnectorConfig, RecipeBook, SyncLog, Sync, Comment, RecipeBookEntry, ShoppingList, InviteLink, UserSpace, Space ) diff --git a/cookbook/views/api.py b/cookbook/views/api.py index d2cf65f8..bffc3ee4 100644 --- a/cookbook/views/api.py +++ b/cookbook/views/api.py @@ -76,7 +76,7 @@ from cookbook.models import (Automation, BookmarkletImport, CookLog, CustomFilte ShoppingListEntry, ShoppingListRecipe, Space, Step, Storage, Supermarket, SupermarketCategory, SupermarketCategoryRelation, Sync, SyncLog, Unit, UnitConversion, UserFile, UserPreference, UserSpace, - ViewLog, HomeAssistantConfig) + ViewLog, ConnectorConfig) from cookbook.provider.dropbox import Dropbox from cookbook.provider.local import Local from cookbook.provider.nextcloud import Nextcloud @@ -102,7 +102,7 @@ from cookbook.serializer import (AccessTokenSerializer, AutomationSerializer, SupermarketCategorySerializer, SupermarketSerializer, SyncLogSerializer, SyncSerializer, UnitConversionSerializer, UnitSerializer, UserFileSerializer, UserPreferenceSerializer, - UserSerializer, UserSpaceSerializer, ViewLogSerializer, HomeAssistantConfigSerializer) + UserSerializer, UserSpaceSerializer, ViewLogSerializer, ConnectorConfigConfigSerializer) from cookbook.views.import_export import get_integration from recipes import settings from recipes.settings import FDC_API_KEY, DRF_THROTTLE_RECIPE_URL_IMPORT @@ -464,10 +464,9 @@ class StorageViewSet(viewsets.ModelViewSet): return self.queryset.filter(space=self.request.space) -class HomeAssistantConfigViewSet(viewsets.ModelViewSet): - # TODO handle delete protect error and adjust test - queryset = HomeAssistantConfig.objects - serializer_class = HomeAssistantConfigSerializer +class ConnectorConfigConfigViewSet(viewsets.ModelViewSet): + queryset = ConnectorConfig.objects + serializer_class = ConnectorConfigConfigSerializer permission_classes = [CustomIsAdmin & CustomTokenHasReadWriteScope] def get_queryset(self): diff --git a/cookbook/views/delete.py b/cookbook/views/delete.py index bbdc98ee..f37d8a55 100644 --- a/cookbook/views/delete.py +++ b/cookbook/views/delete.py @@ -9,7 +9,7 @@ from django.views.generic import DeleteView from cookbook.helper.permission_helper import GroupRequiredMixin, OwnerRequiredMixin, group_required from cookbook.models import (Comment, InviteLink, MealPlan, Recipe, RecipeBook, RecipeBookEntry, - RecipeImport, Space, Storage, Sync, UserSpace, HomeAssistantConfig, ExampleConfig) + RecipeImport, Space, Storage, Sync, UserSpace, ConnectorConfig) from cookbook.provider.dropbox import Dropbox from cookbook.provider.local import Local from cookbook.provider.nextcloud import Nextcloud @@ -122,27 +122,15 @@ class StorageDelete(GroupRequiredMixin, DeleteView): return HttpResponseRedirect(reverse('list_storage')) -class HomeAssistantConfigDelete(GroupRequiredMixin, DeleteView): +class ConnectorConfigDelete(GroupRequiredMixin, DeleteView): groups_required = ['admin'] template_name = "generic/delete_template.html" - model = HomeAssistantConfig - success_url = reverse_lazy('list_connectors') + model = ConnectorConfig + success_url = reverse_lazy('list_connector_config') def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - context['title'] = _("HomeAssistant Config Backend") - return context - - -class ExampleConfigDelete(GroupRequiredMixin, DeleteView): - groups_required = ['admin'] - template_name = "generic/delete_template.html" - model = ExampleConfig - success_url = reverse_lazy('list_connectors') - - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) - context['title'] = _("Example Config Backend") + context['title'] = _("Connectors Config Backend") return context diff --git a/cookbook/views/edit.py b/cookbook/views/edit.py index 2f99216b..356191ec 100644 --- a/cookbook/views/edit.py +++ b/cookbook/views/edit.py @@ -9,10 +9,10 @@ from django.utils.translation import gettext as _ from django.views.generic import UpdateView from django.views.generic.edit import FormMixin -from cookbook.forms import CommentForm, ExternalRecipeForm, StorageForm, SyncForm, HomeAssistantConfigForm, ExampleConfigForm +from cookbook.forms import CommentForm, ExternalRecipeForm, StorageForm, SyncForm, ConnectorConfigForm from cookbook.helper.permission_helper import (GroupRequiredMixin, OwnerRequiredMixin, above_space_limit, group_required) -from cookbook.models import Comment, Recipe, RecipeImport, Storage, Sync, HomeAssistantConfig, ExampleConfig +from cookbook.models import Comment, Recipe, RecipeImport, Storage, Sync, ConnectorConfig from cookbook.provider.dropbox import Dropbox from cookbook.provider.local import Local from cookbook.provider.nextcloud import Nextcloud @@ -128,11 +128,11 @@ def edit_storage(request, pk): ) -class HomeAssistantConfigUpdate(GroupRequiredMixin, UpdateView): +class ConnectorConfigUpdate(GroupRequiredMixin, UpdateView): groups_required = ['admin'] template_name = "generic/edit_template.html" - model = HomeAssistantConfig - form_class = HomeAssistantConfigForm + model = ConnectorConfig + form_class = ConnectorConfigForm def get_form_kwargs(self): kwargs = super().get_form_kwargs() @@ -143,33 +143,14 @@ class HomeAssistantConfigUpdate(GroupRequiredMixin, UpdateView): if form.cleaned_data['update_token'] != VALUE_NOT_CHANGED and form.cleaned_data['update_token'] != "": form.instance.token = form.cleaned_data['update_token'] messages.add_message(self.request, messages.SUCCESS, _('Config saved!')) - return super(HomeAssistantConfigUpdate, self).form_valid(form) + return super(ConnectorConfigUpdate, self).form_valid(form) def get_success_url(self): - return reverse('edit_home_assistant_config', kwargs={'pk': self.object.pk}) + return reverse('edit_connector_config', kwargs={'pk': self.object.pk}) def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - context['title'] = _("HomeAssistantConfig") - return context - - -class ExampleConfigUpdate(GroupRequiredMixin, UpdateView): - groups_required = ['admin'] - template_name = "generic/edit_template.html" - model = ExampleConfig - form_class = ExampleConfigForm - - def form_valid(self, form): - messages.add_message(self.request, messages.SUCCESS, _('Config saved!')) - return super().form_valid(form) - - def get_success_url(self): - return reverse('edit_example_config', kwargs={'pk': self.object.pk}) - - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) - context['title'] = _("ExampleConfig") + context['title'] = _("ConnectorConfig") return context diff --git a/cookbook/views/lists.py b/cookbook/views/lists.py index ed43c143..53b1ae79 100644 --- a/cookbook/views/lists.py +++ b/cookbook/views/lists.py @@ -6,8 +6,8 @@ from django.utils.translation import gettext as _ from django_tables2 import RequestConfig from cookbook.helper.permission_helper import group_required -from cookbook.models import InviteLink, RecipeImport, Storage, SyncLog, UserFile -from cookbook.tables import ImportLogTable, InviteLinkTable, RecipeImportTable, StorageTable +from cookbook.models import InviteLink, RecipeImport, Storage, SyncLog, UserFile, ConnectorConfig +from cookbook.tables import ImportLogTable, InviteLinkTable, RecipeImportTable, StorageTable, ConnectorConfigTable @group_required('admin') @@ -65,6 +65,22 @@ def storage(request): ) +@group_required('admin') +def connector_config(request): + table = ConnectorConfigTable(ConnectorConfig.objects.filter(space=request.space).all()) + RequestConfig(request, paginate={'per_page': 25}).configure(table) + + return render( + request, + 'generic/list_template.html', + { + 'title': _("Connector Config Backend"), + 'table': table, + 'create_url': 'new_connector_config' + } + ) + + @group_required('admin') def invite_link(request): table = InviteLinkTable( diff --git a/cookbook/views/new.py b/cookbook/views/new.py index 22bfbc84..93ad9996 100644 --- a/cookbook/views/new.py +++ b/cookbook/views/new.py @@ -5,9 +5,9 @@ from django.urls import reverse, reverse_lazy from django.utils.translation import gettext as _ from django.views.generic import CreateView -from cookbook.forms import ImportRecipeForm, Storage, StorageForm, HomeAssistantConfigForm, ExampleConfigForm +from cookbook.forms import ImportRecipeForm, Storage, StorageForm, ConnectorConfigForm from cookbook.helper.permission_helper import GroupRequiredMixin, above_space_limit, group_required -from cookbook.models import Recipe, RecipeImport, ShareLink, Step, HomeAssistantConfig, ExampleConfig +from cookbook.models import Recipe, RecipeImport, ShareLink, Step, ConnectorConfig from recipes import settings @@ -70,21 +70,12 @@ class StorageCreate(GroupRequiredMixin, CreateView): return context -class HomeAssistantConfigCreate(GroupRequiredMixin, CreateView): +class ConnectorConfigCreate(GroupRequiredMixin, CreateView): groups_required = ['admin'] template_name = "generic/new_template.html" - model = HomeAssistantConfig - form_class = HomeAssistantConfigForm - success_url = reverse_lazy('list_home_assistant_config') - - def get_form_class(self): - form_class = super().get_form_class() - - if self.request.method == 'GET': - update_token_field = form_class.base_fields['update_token'] - update_token_field.required = True - - return form_class + model = ConnectorConfig + form_class = ConnectorConfigForm + success_url = reverse_lazy('list_connector_config') def form_valid(self, form): if self.request.space.demo or settings.HOSTED: @@ -96,35 +87,11 @@ class HomeAssistantConfigCreate(GroupRequiredMixin, CreateView): obj.created_by = self.request.user obj.space = self.request.space obj.save() - return HttpResponseRedirect(reverse('edit_home_assistant_config', kwargs={'pk': obj.pk})) + return HttpResponseRedirect(reverse('edit_connector_config', kwargs={'pk': obj.pk})) def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - context['title'] = _("HomeAssistant Config Backend") - return context - - -class ExampleConfigCreate(GroupRequiredMixin, CreateView): - groups_required = ['admin'] - template_name = "generic/new_template.html" - model = ExampleConfig - form_class = ExampleConfigForm - success_url = reverse_lazy('list_connectors') - - def form_valid(self, form): - if self.request.space.demo or settings.HOSTED: - messages.add_message(self.request, messages.ERROR, _('This feature is not yet available in the hosted version of tandoor!')) - return redirect('index') - - obj = form.save(commit=False) - obj.created_by = self.request.user - obj.space = self.request.space - obj.save() - return HttpResponseRedirect(reverse('edit_example_config', kwargs={'pk': obj.pk})) - - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) - context['title'] = _("Example Config Backend") + context['title'] = _("Connector Config Backend") return context From 578bb2af252fc3dfd63172fc00644b95e975cfdf Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Wed, 24 Jan 2024 08:57:24 +0100 Subject: [PATCH 21/41] better error handling during connector initilization --- cookbook/connectors/connector_manager.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 86a60b04..bb74aaf0 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -82,6 +82,7 @@ class ConnectorManager: item: Optional[Work] = worker_queue.get() except KeyboardInterrupt: break + if item is None: break @@ -96,11 +97,20 @@ class ConnectorManager: loop.run_until_complete(close_connectors(connectors)) with scope(space=space): - connectors: List[Connector] = list( - filter( - lambda x: x is not None, - [ConnectorManager.get_connected_for_config(config) for config in space.connectorconfig_set.all() if config.enabled], - )) + connectors: List[Connector] = list() + for config in space.connectorconfig_set.all(): + config: ConnectorConfig = config + if not config.enabled: + continue + + try: + connector: Optional[Connector] = ConnectorManager.get_connected_for_config(config) + except BaseException: + logging.exception(f"failed to initialize {config.name}") + continue + + connectors.append(connector) + _connectors[space.name] = connectors if len(connectors) == 0 or refresh_connector_cache: From ba169ba38db0ebef641f29df1f2696a31c00719f Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Wed, 24 Jan 2024 08:59:31 +0100 Subject: [PATCH 22/41] better logging on skipped action --- cookbook/connectors/connector_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index bb74aaf0..64c55265 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -59,7 +59,7 @@ class ConnectorManager: try: self._queue.put_nowait(Work(instance, action_type)) except queue.Full: - logging.info("queue was full, so skipping %s", instance) + logging.info(f"queue was full, so skipping {action_type} of type {type(instance)}") return def stop(self): From 502a606534fe27000524eaa223f4a3bff940fb0f Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Sun, 28 Jan 2024 22:59:51 +0100 Subject: [PATCH 23/41] Update the code based on feedback. set Default to enabled, add to documentation how to disable it. Add extra documentation --- .gitignore | 1 - cookbook/connectors/connector.py | 2 ++ cookbook/connectors/connector_manager.py | 28 +++++++++++++++++------- cookbook/signals.py | 3 +-- cookbook/views/api.py | 13 +++++------ cookbook/views/new.py | 6 ++++- docs/system/configuration.md | 10 +++++++++ recipes/settings.py | 3 ++- 8 files changed, 46 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index 4c8df7a7..1c5e40a4 100644 --- a/.gitignore +++ b/.gitignore @@ -54,7 +54,6 @@ docs/_build/ target/ \.idea/dataSources/ -.idea \.idea/dataSources\.xml \.idea/dataSources\.local\.xml diff --git a/cookbook/connectors/connector.py b/cookbook/connectors/connector.py index 3647dc71..27e9408d 100644 --- a/cookbook/connectors/connector.py +++ b/cookbook/connectors/connector.py @@ -3,6 +3,7 @@ from abc import ABC, abstractmethod from cookbook.models import ShoppingListEntry, Space, ConnectorConfig +# A Connector is 'destroyed' & recreated each time 'any' ConnectorConfig in a space changes. class Connector(ABC): @abstractmethod def __init__(self, config: ConnectorConfig): @@ -12,6 +13,7 @@ class Connector(ABC): async def on_shopping_list_entry_created(self, space: Space, instance: ShoppingListEntry) -> None: pass + # This method might not trigger on 'direct' entry updates: https://stackoverflow.com/a/35238823 @abstractmethod async def on_shopping_list_entry_updated(self, space: Space, instance: ShoppingListEntry) -> None: pass diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 64c55265..fe42c1a7 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -7,18 +7,18 @@ from dataclasses import dataclass from enum import Enum from multiprocessing import JoinableQueue from types import UnionType -from typing import List, Any, Dict, Optional +from typing import List, Any, Dict, Optional, Type from django_scopes import scope +from django.conf import settings from cookbook.connectors.connector import Connector from cookbook.connectors.homeassistant import HomeAssistant -from cookbook.models import ShoppingListEntry, Recipe, MealPlan, Space, ConnectorConfig +from cookbook.models import ShoppingListEntry, Space, ConnectorConfig multiprocessing.set_start_method('fork') # https://code.djangoproject.com/ticket/31169 -QUEUE_MAX_SIZE = 25 -REGISTERED_CLASSES: UnionType = ShoppingListEntry | Recipe | MealPlan +REGISTERED_CLASSES: UnionType | Type = ShoppingListEntry class ActionType(Enum): @@ -33,12 +33,20 @@ class Work: actionType: ActionType +# The way ConnectionManager works is as follows: +# 1. On init, it starts a worker & creates a queue for 'Work' +# 2. Then any time its called, it verifies the type of action (create/update/delete) and if the item is of interest, pushes the Work (non blocking) to the queue. +# 3. The worker consumes said work from the queue. +# 3.1 If the work is of type ConnectorConfig, it flushes its cache of known connectors (per space.id) +# 3.2 If work is of type REGISTERED_CLASSES, it asynchronously fires of all connectors and wait for them to finish (runtime should depend on the 'slowest' connector) +# 4. Work is marked as consumed, and next entry of the queue is consumed. +# Each 'Work' is processed in sequential by the worker, so the throughput is about [workers * the slowest connector] class ConnectorManager: _queue: JoinableQueue _listening_to_classes = REGISTERED_CLASSES | ConnectorConfig def __init__(self): - self._queue = multiprocessing.JoinableQueue(maxsize=QUEUE_MAX_SIZE) + self._queue = multiprocessing.JoinableQueue(maxsize=settings.EXTERNAL_CONNECTORS_QUEUE_SIZE) self._worker = multiprocessing.Process(target=self.worker, args=(self._queue,), daemon=True) self._worker.start() @@ -75,7 +83,7 @@ class ConnectorManager: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - _connectors: Dict[str, List[Connector]] = dict() + _connectors: Dict[int, List[Connector]] = dict() while True: try: @@ -90,7 +98,7 @@ class ConnectorManager: refresh_connector_cache = isinstance(item.instance, ConnectorConfig) space: Space = item.instance.space - connectors: Optional[List[Connector]] = _connectors.get(space.name) + connectors: Optional[List[Connector]] = _connectors.get(space.id) if connectors is None or refresh_connector_cache: if connectors is not None: @@ -111,7 +119,7 @@ class ConnectorManager: connectors.append(connector) - _connectors[space.name] = connectors + _connectors[space.id] = connectors if len(connectors) == 0 or refresh_connector_cache: worker_queue.task_done() @@ -134,6 +142,9 @@ class ConnectorManager: async def close_connectors(connectors: List[Connector]): tasks: List[Task] = [asyncio.create_task(connector.close()) for connector in connectors] + if len(tasks) == 0: + return + try: await asyncio.gather(*tasks, return_exceptions=False) except BaseException: @@ -161,6 +172,7 @@ async def run_connectors(connectors: List[Connector], space: Space, instance: RE return try: + # Wait for all async tasks to finish, if one fails, the others still continue. await asyncio.gather(*tasks, return_exceptions=False) except BaseException: logging.exception("received an exception from one of the connectors") diff --git a/cookbook/signals.py b/cookbook/signals.py index ce957bce..cd183c58 100644 --- a/cookbook/signals.py +++ b/cookbook/signals.py @@ -15,7 +15,6 @@ from cookbook.helper.shopping_helper import RecipeShoppingEditor from cookbook.managers import DICTIONARY from cookbook.models import (Food, MealPlan, PropertyType, Recipe, SearchFields, SearchPreference, Step, Unit, UserPreference) -from recipes.settings import ENABLE_EXTERNAL_CONNECTORS SQLITE = True if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.postgresql': @@ -165,7 +164,7 @@ def clear_property_type_cache(sender, instance=None, created=False, **kwargs): caches['default'].delete(CacheHelper(instance.space).PROPERTY_TYPE_CACHE_KEY) -if ENABLE_EXTERNAL_CONNECTORS: +if not settings.DISABLE_EXTERNAL_CONNECTORS: handler = ConnectorManager() post_save.connect(handler, dispatch_uid="connector_manager") post_delete.connect(handler, dispatch_uid="connector_manager") diff --git a/cookbook/views/api.py b/cookbook/views/api.py index bffc3ee4..446f0b7f 100644 --- a/cookbook/views/api.py +++ b/cookbook/views/api.py @@ -643,13 +643,12 @@ class FoodViewSet(viewsets.ModelViewSet, TreeMixin): if pt.fdc_id: for fn in data['foodNutrients']: if fn['nutrient']['id'] == pt.fdc_id: - food_property_list.append( - Property( - property_type_id=pt.id, - property_amount=round(fn['amount'], 2), - import_food_id=food.id, - space=self.request.space, - )) + food_property_list.append(Property( + property_type_id=pt.id, + property_amount=round(fn['amount'], 2), + import_food_id=food.id, + space=self.request.space, + )) Property.objects.bulk_create(food_property_list, ignore_conflicts=True, unique_fields=('space', 'import_food_id', 'property_type',)) diff --git a/cookbook/views/new.py b/cookbook/views/new.py index 93ad9996..7e973251 100644 --- a/cookbook/views/new.py +++ b/cookbook/views/new.py @@ -78,10 +78,14 @@ class ConnectorConfigCreate(GroupRequiredMixin, CreateView): success_url = reverse_lazy('list_connector_config') def form_valid(self, form): - if self.request.space.demo or settings.HOSTED: + if self.request.space.demo: messages.add_message(self.request, messages.ERROR, _('This feature is not yet available in the hosted version of tandoor!')) return redirect('index') + if settings.DISABLE_EXTERNAL_CONNECTORS: + messages.add_message(self.request, messages.ERROR, _('This feature is not enabled by the server admin!')) + return redirect('index') + obj = form.save(commit=False) obj.token = form.cleaned_data['update_token'] obj.created_by = self.request.user diff --git a/docs/system/configuration.md b/docs/system/configuration.md index f6e64d24..dc64d41d 100644 --- a/docs/system/configuration.md +++ b/docs/system/configuration.md @@ -437,6 +437,16 @@ key [here](https://fdc.nal.usda.gov/api-key-signup.html). FDC_API_KEY=DEMO_KEY ``` +#### External Connectors + +`DISABLE_EXTERNAL_CONNECTORS` is a global switch to disable External Connectors entirely (e.g. HomeAssistant). +`EXTERNAL_CONNECTORS_QUEUE_SIZE` is the amount of changes that are kept in memory if the worker cannot keep up. + +```env +DISABLE_EXTERNAL_CONNECTORS=0 // 0 = connectors enabled, 1 = connectors enabled +EXTERNAL_CONNECTORS_QUEUE_SIZE=25 +``` + ### Debugging/Development settings !!! warning diff --git a/recipes/settings.py b/recipes/settings.py index 319271ea..6439f727 100644 --- a/recipes/settings.py +++ b/recipes/settings.py @@ -555,6 +555,7 @@ DEFAULT_FROM_EMAIL = os.getenv('DEFAULT_FROM_EMAIL', 'webmaster@localhost') ACCOUNT_EMAIL_SUBJECT_PREFIX = os.getenv( 'ACCOUNT_EMAIL_SUBJECT_PREFIX', '[Tandoor Recipes] ') # allauth sender prefix -ENABLE_EXTERNAL_CONNECTORS = bool(int(os.getenv('ENABLE_EXTERNAL_CONNECTORS', False))) +DISABLE_EXTERNAL_CONNECTORS = bool(int(os.getenv('DISABLE_EXTERNAL_CONNECTORS', False))) +EXTERNAL_CONNECTORS_QUEUE_SIZE = int(os.getenv('EXTERNAL_CONNECTORS_QUEUE_SIZE', 25)) mimetypes.add_type("text/javascript", ".js", True) From a8983a4b8a302a39d034a88fec6060bd28abb592 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Mon, 29 Jan 2024 09:56:40 +0100 Subject: [PATCH 24/41] undo workflow changes --- .github/workflows/ci.yml | 75 +++++++--------------------------------- 1 file changed, 13 insertions(+), 62 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb72bc0c..d5da9d82 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,80 +10,31 @@ jobs: max-parallel: 4 matrix: python-version: ['3.10'] - node-version: ['18'] steps: - uses: actions/checkout@v3 - - uses: awalsh128/cache-apt-pkgs-action@v1.3.1 + - name: Set up Python 3.10 + uses: actions/setup-python@v5 with: - packages: libsasl2-dev python3-dev libldap2-dev libssl-dev - version: 1.0 - - # Setup python & dependencies - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + python-version: '3.10' + # Build Vue frontend + - uses: actions/setup-node@v4 with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - - - name: Install Python Dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - - name: Cache StaticFiles - uses: actions/cache@v3 - id: django_cache - with: - path: | - ./cookbook/static - ./vue/webpack-stats.json - ./staticfiles - key: | - ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}-collectstatic-${{ hashFiles('**/*.css', '**/*.js', 'vue/src/*') }} - - # Build Vue frontend & Dependencies - - name: Set up Node ${{ matrix.node-version }} - if: steps.django_cache.outputs.cache-hit != 'true' - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - cache: 'yarn' - cache-dependency-path: ./vue/yarn.lock - + node-version: '18' - name: Install Vue dependencies - if: steps.django_cache.outputs.cache-hit != 'true' working-directory: ./vue run: yarn install - - name: Build Vue dependencies - if: steps.django_cache.outputs.cache-hit != 'true' working-directory: ./vue run: yarn build - - - name: Compile Django StatisFiles - if: steps.django_cache.outputs.cache-hit != 'true' + - name: Install Django dependencies run: | + sudo apt-get -y update + sudo apt-get install -y libsasl2-dev python3-dev libldap2-dev libssl-dev + python -m pip install --upgrade pip + pip install -r requirements.txt python3 manage.py collectstatic --noinput python3 manage.py collectstatic_js_reverse - - - uses: actions/cache/save@v3 - if: steps.django_cache.outputs.cache-hit != 'true' - with: - path: | - ./cookbook/static - ./vue/webpack-stats.json - ./staticfiles - key: | - ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}-collectstatic-${{ hashFiles('**/*.css', '**/*.js', 'vue/src/*') }} - - name: Django Testing project - run: pytest --junitxml=junit/test-results-${{ matrix.python-version }}.xml - - - name: Publish Test Results - uses: EnricoMi/publish-unit-test-result-action@v2 - if: always() - with: - comment_mode: off - files: | - junit/test-results-${{ matrix.python-version }}.xml + run: | + pytest From 75c0ca8a9e7d0d86ff06fbddd1406a91abcf779a Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Fri, 2 Feb 2024 20:52:05 +0100 Subject: [PATCH 25/41] bunp migration --- .../{0209_connectorconfig.py => 0210_connectorconfig.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename cookbook/migrations/{0209_connectorconfig.py => 0210_connectorconfig.py} (93%) diff --git a/cookbook/migrations/0209_connectorconfig.py b/cookbook/migrations/0210_connectorconfig.py similarity index 93% rename from cookbook/migrations/0209_connectorconfig.py rename to cookbook/migrations/0210_connectorconfig.py index 4f672e81..3d88816f 100644 --- a/cookbook/migrations/0209_connectorconfig.py +++ b/cookbook/migrations/0210_connectorconfig.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.7 on 2024-01-17 21:49 +# Generated by Django 4.2.7 on 2024-02-02 19:51 import cookbook.models from django.conf import settings @@ -11,7 +11,7 @@ class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('cookbook', '0208_space_app_name_userpreference_max_owned_spaces'), + ('cookbook', '0209_remove_space_use_plural'), ] operations = [ From 247907ef254b3ae4f0b5f9d4e139fcc40cb79918 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Mon, 5 Feb 2024 23:26:33 +0100 Subject: [PATCH 26/41] move from signals to apps, add dedicated feature docs, add config toggle to menu item, undo unnecessary changes --- .github/workflows/ci.yml | 1 + cookbook/apps.py | 7 + cookbook/connectors/connector_manager.py | 16 +- cookbook/helper/context_processors.py | 1 + cookbook/signals.py | 9 +- cookbook/templates/base.html | 2 +- cookbook/templatetags/custom_tags.py | 3 + .../tests/other/test_connector_manager.py | 2 - cookbook/views/api.py | 199 ++++++++---------- docs/features/connectors.md | 43 ++++ docs/system/configuration.md | 10 - recipes/settings.py | 2 +- 12 files changed, 151 insertions(+), 144 deletions(-) create mode 100644 docs/features/connectors.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 202de7b2..56148b3b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,6 +78,7 @@ jobs: ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}-collectstatic-${{ hashFiles('**/*.css', '**/*.js', 'vue/src/*') }} - name: Django Testing project + timeout-minutes: 15 run: pytest --junitxml=junit/test-results-${{ matrix.python-version }}.xml - name: Publish Test Results diff --git a/cookbook/apps.py b/cookbook/apps.py index e551319d..3bd8d847 100644 --- a/cookbook/apps.py +++ b/cookbook/apps.py @@ -3,6 +3,7 @@ import traceback from django.apps import AppConfig from django.conf import settings from django.db import OperationalError, ProgrammingError +from django.db.models.signals import post_save, post_delete from django_scopes import scopes_disabled from recipes.settings import DEBUG @@ -14,6 +15,12 @@ class CookbookConfig(AppConfig): def ready(self): import cookbook.signals # noqa + if not settings.DISABLE_EXTERNAL_CONNECTORS: + from cookbook.connectors.connector_manager import ConnectorManager # Needs to be here to prevent loading race condition of oauth2 modules in models.py + handler = ConnectorManager() + post_save.connect(handler, dispatch_uid="connector_manager") + post_delete.connect(handler, dispatch_uid="connector_manager") + # if not settings.DISABLE_TREE_FIX_STARTUP: # # when starting up run fix_tree to: # # a) make sure that nodes are sorted when switching between sort modes diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index fe42c1a7..90ad1bfd 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -29,13 +29,13 @@ class ActionType(Enum): @dataclass class Work: - instance: REGISTERED_CLASSES + instance: REGISTERED_CLASSES | ConnectorConfig actionType: ActionType # The way ConnectionManager works is as follows: # 1. On init, it starts a worker & creates a queue for 'Work' -# 2. Then any time its called, it verifies the type of action (create/update/delete) and if the item is of interest, pushes the Work (non blocking) to the queue. +# 2. Then any time its called, it verifies the type of action (create/update/delete) and if the item is of interest, pushes the Work (non-blocking) to the queue. # 3. The worker consumes said work from the queue. # 3.1 If the work is of type ConnectorConfig, it flushes its cache of known connectors (per space.id) # 3.2 If work is of type REGISTERED_CLASSES, it asynchronously fires of all connectors and wait for them to finish (runtime should depend on the 'slowest' connector) @@ -50,6 +50,7 @@ class ConnectorManager: self._worker = multiprocessing.Process(target=self.worker, args=(self._queue,), daemon=True) self._worker.start() + # Called by post save & post delete signals def __call__(self, instance: Any, **kwargs) -> None: if not isinstance(instance, self._listening_to_classes) or not hasattr(instance, "space"): return @@ -77,13 +78,15 @@ class ConnectorManager: @staticmethod def worker(worker_queue: JoinableQueue): + # https://stackoverflow.com/a/10684672 Close open connections after starting a new process to prevent re-use of same connections from django.db import connections connections.close_all() loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - _connectors: Dict[int, List[Connector]] = dict() + # + _connectors_cache: Dict[int, List[Connector]] = dict() while True: try: @@ -98,7 +101,7 @@ class ConnectorManager: refresh_connector_cache = isinstance(item.instance, ConnectorConfig) space: Space = item.instance.space - connectors: Optional[List[Connector]] = _connectors.get(space.id) + connectors: Optional[List[Connector]] = _connectors_cache.get(space.id) if connectors is None or refresh_connector_cache: if connectors is not None: @@ -117,9 +120,10 @@ class ConnectorManager: logging.exception(f"failed to initialize {config.name}") continue - connectors.append(connector) + if connector is not None: + connectors.append(connector) - _connectors[space.id] = connectors + _connectors_cache[space.id] = connectors if len(connectors) == 0 or refresh_connector_cache: worker_queue.task_done() diff --git a/cookbook/helper/context_processors.py b/cookbook/helper/context_processors.py index 30e704c1..9beb8b9b 100644 --- a/cookbook/helper/context_processors.py +++ b/cookbook/helper/context_processors.py @@ -11,4 +11,5 @@ def context_settings(request): 'PRIVACY_URL': settings.PRIVACY_URL, 'IMPRINT_URL': settings.IMPRINT_URL, 'SHOPPING_MIN_AUTOSYNC_INTERVAL': settings.SHOPPING_MIN_AUTOSYNC_INTERVAL, + 'DISABLE_EXTERNAL_CONNECTORS': settings.DISABLE_EXTERNAL_CONNECTORS, } diff --git a/cookbook/signals.py b/cookbook/signals.py index cd183c58..a93ffba1 100644 --- a/cookbook/signals.py +++ b/cookbook/signals.py @@ -4,12 +4,11 @@ from django.conf import settings from django.contrib.auth.models import User from django.contrib.postgres.search import SearchVector from django.core.cache import caches -from django.db.models.signals import post_save, post_delete +from django.db.models.signals import post_save from django.dispatch import receiver from django.utils import translation from django_scopes import scope, scopes_disabled -from cookbook.connectors.connector_manager import ConnectorManager from cookbook.helper.cache_helper import CacheHelper from cookbook.helper.shopping_helper import RecipeShoppingEditor from cookbook.managers import DICTIONARY @@ -162,9 +161,3 @@ def clear_unit_cache(sender, instance=None, created=False, **kwargs): def clear_property_type_cache(sender, instance=None, created=False, **kwargs): if instance: caches['default'].delete(CacheHelper(instance.space).PROPERTY_TYPE_CACHE_KEY) - - -if not settings.DISABLE_EXTERNAL_CONNECTORS: - handler = ConnectorManager() - post_save.connect(handler, dispatch_uid="connector_manager") - post_delete.connect(handler, dispatch_uid="connector_manager") diff --git a/cookbook/templates/base.html b/cookbook/templates/base.html index 4cbc1544..f1d6c2f7 100644 --- a/cookbook/templates/base.html +++ b/cookbook/templates/base.html @@ -335,7 +335,7 @@ {% trans 'Space Settings' %} {% endif %} - {% if request.user == request.space.created_by or user.is_superuser %} + {% if not DISABLE_EXTERNAL_CONNECTORS and request.user == request.space.created_by or not DISABLE_EXTERNAL_CONNECTORS and user.is_superuser %} {% trans 'External Connectors' %} {% endif %} diff --git a/cookbook/templatetags/custom_tags.py b/cookbook/templatetags/custom_tags.py index cdacd8e7..45985510 100644 --- a/cookbook/templatetags/custom_tags.py +++ b/cookbook/templatetags/custom_tags.py @@ -112,6 +112,9 @@ def recipe_last(recipe, user): def page_help(page_name): help_pages = { 'edit_storage': 'https://docs.tandoor.dev/features/external_recipes/', + 'list_connector_config': 'https://docs.tandoor.dev/features/connectors/', + 'new_connector_config': 'https://docs.tandoor.dev/features/connectors/', + 'edit_connector_config': 'https://docs.tandoor.dev/features/connectors/', 'view_shopping': 'https://docs.tandoor.dev/features/shopping/', 'view_import': 'https://docs.tandoor.dev/features/import_export/', 'view_export': 'https://docs.tandoor.dev/features/import_export/', diff --git a/cookbook/tests/other/test_connector_manager.py b/cookbook/tests/other/test_connector_manager.py index c6df778f..b8610082 100644 --- a/cookbook/tests/other/test_connector_manager.py +++ b/cookbook/tests/other/test_connector_manager.py @@ -1,5 +1,3 @@ -from dataclasses import dataclass - import pytest from django.contrib import auth from mock.mock import Mock diff --git a/cookbook/views/api.py b/cookbook/views/api.py index 5b44c2f8..1a91a33c 100644 --- a/cookbook/views/api.py +++ b/cookbook/views/api.py @@ -181,10 +181,9 @@ class FuzzyFilterMixin(ViewSetMixin, ExtendedRecipeMixin): self.queryset = self.queryset.filter(space=self.request.space).order_by(Lower('name').asc()) query = self.request.query_params.get('query', None) if self.request.user.is_authenticated: - fuzzy = self.request.user.searchpreference.lookup or any( - [self.model.__name__.lower() in x for x in - self.request.user.searchpreference.trigram.values_list( - 'field', flat=True)]) + fuzzy = self.request.user.searchpreference.lookup or any([self.model.__name__.lower() in x for x in + self.request.user.searchpreference.trigram.values_list( + 'field', flat=True)]) else: fuzzy = True @@ -204,10 +203,8 @@ class FuzzyFilterMixin(ViewSetMixin, ExtendedRecipeMixin): filter |= Q(name__unaccent__icontains=query) self.queryset = ( - self.queryset.annotate( - starts=Case( - When(name__istartswith=query, then=(Value(100))), - default=Value(0))) # put exact matches at the top of the result set + self.queryset.annotate(starts=Case(When(name__istartswith=query, then=(Value(100))), + default=Value(0))) # put exact matches at the top of the result set .filter(filter).order_by('-starts', Lower('name').asc()) ) @@ -329,9 +326,8 @@ class TreeMixin(MergeMixin, FuzzyFilterMixin, ExtendedRecipeMixin): return self.annotate_recipe(queryset=super().get_queryset(), request=self.request, serializer=self.serializer_class, tree=True) self.queryset = self.queryset.filter(space=self.request.space).order_by(Lower('name').asc()) - return self.annotate_recipe( - queryset=self.queryset, request=self.request, serializer=self.serializer_class, - tree=True) + return self.annotate_recipe(queryset=self.queryset, request=self.request, serializer=self.serializer_class, + tree=True) @decorators.action(detail=True, url_path='move/(?P[^/.]+)', methods=['PUT'], ) @decorators.renderer_classes((TemplateHTMLRenderer, JSONRenderer)) @@ -575,9 +571,8 @@ class FoodViewSet(viewsets.ModelViewSet, TreeMixin): pass self.queryset = super().get_queryset() - shopping_status = ShoppingListEntry.objects.filter( - space=self.request.space, food=OuterRef('id'), - checked=False).values('id') + shopping_status = ShoppingListEntry.objects.filter(space=self.request.space, food=OuterRef('id'), + checked=False).values('id') # onhand_status = self.queryset.annotate(onhand_status=Exists(onhand_users_set__in=[shared_users])) return self.queryset \ .annotate(shopping_status=Exists(shopping_status)) \ @@ -598,9 +593,8 @@ class FoodViewSet(viewsets.ModelViewSet, TreeMixin): shared_users = list(self.request.user.get_shopping_share()) shared_users.append(request.user) if request.data.get('_delete', False) == 'true': - ShoppingListEntry.objects.filter( - food=obj, checked=False, space=request.space, - created_by__in=shared_users).delete() + ShoppingListEntry.objects.filter(food=obj, checked=False, space=request.space, + created_by__in=shared_users).delete() content = {'msg': _(f'{obj.name} was removed from the shopping list.')} return Response(content, status=status.HTTP_204_NO_CONTENT) @@ -608,9 +602,8 @@ class FoodViewSet(viewsets.ModelViewSet, TreeMixin): unit = request.data.get('unit', None) content = {'msg': _(f'{obj.name} was added to the shopping list.')} - ShoppingListEntry.objects.create( - food=obj, amount=amount, unit=unit, space=request.space, - created_by=request.user) + ShoppingListEntry.objects.create(food=obj, amount=amount, unit=unit, space=request.space, + created_by=request.user) return Response(content, status=status.HTTP_204_NO_CONTENT) @decorators.action(detail=True, methods=['POST'], ) @@ -623,11 +616,8 @@ class FoodViewSet(viewsets.ModelViewSet, TreeMixin): response = requests.get(f'https://api.nal.usda.gov/fdc/v1/food/{food.fdc_id}?api_key={FDC_API_KEY}') if response.status_code == 429: - return JsonResponse( - {'msg', - 'API Key Rate Limit reached/exceeded, see https://api.data.gov/docs/rate-limits/ for more information. Configure your key in Tandoor using environment FDC_API_KEY variable.'}, - status=429, - json_dumps_params={'indent': 4}) + return JsonResponse({'msg', 'API Key Rate Limit reached/exceeded, see https://api.data.gov/docs/rate-limits/ for more information. Configure your key in Tandoor using environment FDC_API_KEY variable.'}, status=429, + json_dumps_params={'indent': 4}) try: data = json.loads(response.content) @@ -883,14 +873,12 @@ class RecipePagination(PageNumberPagination): return super().paginate_queryset(queryset, request, view) def get_paginated_response(self, data): - return Response( - OrderedDict( - [ - ('count', self.page.paginator.count), - ('next', self.get_next_link()), - ('previous', self.get_previous_link()), - ('results', data), - ])) + return Response(OrderedDict([ + ('count', self.page.paginator.count), + ('next', self.get_next_link()), + ('previous', self.get_previous_link()), + ('results', data), + ])) class RecipeViewSet(viewsets.ModelViewSet): @@ -976,10 +964,9 @@ class RecipeViewSet(viewsets.ModelViewSet): def list(self, request, *args, **kwargs): if self.request.GET.get('debug', False): - return JsonResponse( - { - 'new': str(self.get_queryset().query), - }) + return JsonResponse({ + 'new': str(self.get_queryset().query), + }) return super().list(request, *args, **kwargs) def get_serializer_class(self): @@ -1149,10 +1136,8 @@ class ShoppingListEntryViewSet(viewsets.ModelViewSet): permission_classes = [(CustomIsOwner | CustomIsShared) & CustomTokenHasReadWriteScope] query_params = [ QueryParam(name='id', description=_('Returns the shopping list entry with a primary key of id. Multiple values allowed.'), qtype='int'), - QueryParam( - name='checked', description=_( - 'Filter shopping list entries on checked. [''true'', ''false'', ''both'', ''recent'']
- ''recent'' includes unchecked items and recently completed items.') - ), + QueryParam(name='checked', description=_('Filter shopping list entries on checked. [''true'', ''false'', ''both'', ''recent'']
- ''recent'' includes unchecked items and recently completed items.') + ), QueryParam(name='supermarket', description=_('Returns the shopping list entries sorted by supermarket category order.'), qtype='int'), ] schema = QueryParamAutoSchema() @@ -1343,28 +1328,25 @@ class CustomAuthToken(ObtainAuthToken): throttle_classes = [AuthTokenThrottle] def post(self, request, *args, **kwargs): - serializer = self.serializer_class( - data=request.data, - context={'request': request}) + serializer = self.serializer_class(data=request.data, + context={'request': request}) serializer.is_valid(raise_exception=True) user = serializer.validated_data['user'] if token := AccessToken.objects.filter(user=user, expires__gt=timezone.now(), scope__contains='read').filter( scope__contains='write').first(): access_token = token else: - access_token = AccessToken.objects.create( - user=user, token=f'tda_{str(uuid.uuid4()).replace("-", "_")}', - expires=(timezone.now() + timezone.timedelta(days=365 * 5)), - scope='read write app') - return Response( - { - 'id': access_token.id, - 'token': access_token.token, - 'scope': access_token.scope, - 'expires': access_token.expires, - 'user_id': access_token.user.pk, - 'test': user.pk - }) + access_token = AccessToken.objects.create(user=user, token=f'tda_{str(uuid.uuid4()).replace("-", "_")}', + expires=(timezone.now() + timezone.timedelta(days=365 * 5)), + scope='read write app') + return Response({ + 'id': access_token.id, + 'token': access_token.token, + 'scope': access_token.scope, + 'expires': access_token.expires, + 'user_id': access_token.user.pk, + 'test': user.pk + }) class RecipeUrlImportView(APIView): @@ -1393,71 +1375,61 @@ class RecipeUrlImportView(APIView): url = serializer.validated_data.get('url', None) data = unquote(serializer.validated_data.get('data', None)) if not url and not data: - return Response( - { - 'error': True, - 'msg': _('Nothing to do.') - }, status=status.HTTP_400_BAD_REQUEST) + return Response({ + 'error': True, + 'msg': _('Nothing to do.') + }, status=status.HTTP_400_BAD_REQUEST) elif url and not data: if re.match('^(https?://)?(www\\.youtube\\.com|youtu\\.be)/.+$', url): if validators.url(url, public=True): - return Response( - { - 'recipe_json': get_from_youtube_scraper(url, request), - 'recipe_images': [], - }, status=status.HTTP_200_OK) + return Response({ + 'recipe_json': get_from_youtube_scraper(url, request), + 'recipe_images': [], + }, status=status.HTTP_200_OK) if re.match( '^(.)*/view/recipe/[0-9]+/[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$', url): recipe_json = requests.get( - url.replace('/view/recipe/', '/api/recipe/').replace( - re.split('/view/recipe/[0-9]+', url)[1], - '') + '?share=' + + url.replace('/view/recipe/', '/api/recipe/').replace(re.split('/view/recipe/[0-9]+', url)[1], + '') + '?share=' + re.split('/view/recipe/[0-9]+', url)[1].replace('/', '')).json() recipe_json = clean_dict(recipe_json, 'id') serialized_recipe = RecipeExportSerializer(data=recipe_json, context={'request': request}) if serialized_recipe.is_valid(): recipe = serialized_recipe.save() if validators.url(recipe_json['image'], public=True): - recipe.image = File( - handle_image( - request, - File( - io.BytesIO(requests.get(recipe_json['image']).content), - name='image'), - filetype=pathlib.Path(recipe_json['image']).suffix), - name=f'{uuid.uuid4()}_{recipe.pk}{pathlib.Path(recipe_json["image"]).suffix}') + recipe.image = File(handle_image(request, + File(io.BytesIO(requests.get(recipe_json['image']).content), + name='image'), + filetype=pathlib.Path(recipe_json['image']).suffix), + name=f'{uuid.uuid4()}_{recipe.pk}{pathlib.Path(recipe_json["image"]).suffix}') recipe.save() - return Response( - { - 'link': request.build_absolute_uri(reverse('view_recipe', args={recipe.pk})) - }, status=status.HTTP_201_CREATED) + return Response({ + 'link': request.build_absolute_uri(reverse('view_recipe', args={recipe.pk})) + }, status=status.HTTP_201_CREATED) else: try: if validators.url(url, public=True): scrape = scrape_me(url_path=url, wild_mode=True) else: - return Response( - { - 'error': True, - 'msg': _('Invalid Url') - }, status=status.HTTP_400_BAD_REQUEST) + return Response({ + 'error': True, + 'msg': _('Invalid Url') + }, status=status.HTTP_400_BAD_REQUEST) except NoSchemaFoundInWildMode: pass except requests.exceptions.ConnectionError: - return Response( - { - 'error': True, - 'msg': _('Connection Refused.') - }, status=status.HTTP_400_BAD_REQUEST) + return Response({ + 'error': True, + 'msg': _('Connection Refused.') + }, status=status.HTTP_400_BAD_REQUEST) except requests.exceptions.MissingSchema: - return Response( - { - 'error': True, - 'msg': _('Bad URL Schema.') - }, status=status.HTTP_400_BAD_REQUEST) + return Response({ + 'error': True, + 'msg': _('Bad URL Schema.') + }, status=status.HTTP_400_BAD_REQUEST) else: try: data_json = json.loads(data) @@ -1473,18 +1445,16 @@ class RecipeUrlImportView(APIView): scrape = text_scraper(text=data, url=found_url) if scrape: - return Response( - { - 'recipe_json': helper.get_from_scraper(scrape, request), - 'recipe_images': list(dict.fromkeys(get_images_from_soup(scrape.soup, url))), - }, status=status.HTTP_200_OK) + return Response({ + 'recipe_json': helper.get_from_scraper(scrape, request), + 'recipe_images': list(dict.fromkeys(get_images_from_soup(scrape.soup, url))), + }, status=status.HTTP_200_OK) else: - return Response( - { - 'error': True, - 'msg': _('No usable data could be found.') - }, status=status.HTTP_400_BAD_REQUEST) + return Response({ + 'error': True, + 'msg': _('No usable data could be found.') + }, status=status.HTTP_400_BAD_REQUEST) else: return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) @@ -1576,9 +1546,8 @@ def import_files(request): return Response({'import_id': il.pk}, status=status.HTTP_200_OK) except NotImplementedError: - return Response( - {'error': True, 'msg': _('Importing is not implemented for this provider')}, - status=status.HTTP_400_BAD_REQUEST) + return Response({'error': True, 'msg': _('Importing is not implemented for this provider')}, + status=status.HTTP_400_BAD_REQUEST) else: return Response({'error': True, 'msg': form.errors}, status=status.HTTP_400_BAD_REQUEST) @@ -1654,9 +1623,8 @@ def get_recipe_file(request, recipe_id): @group_required('user') def sync_all(request): if request.space.demo or settings.HOSTED: - messages.add_message( - request, messages.ERROR, - _('This feature is not yet available in the hosted version of tandoor!')) + messages.add_message(request, messages.ERROR, + _('This feature is not yet available in the hosted version of tandoor!')) return redirect('index') monitors = Sync.objects.filter(active=True).filter(space=request.user.userspace_set.filter(active=1).first().space) @@ -1695,9 +1663,8 @@ def share_link(request, pk): if request.space.allow_sharing and has_group_permission(request.user, ('user',)): recipe = get_object_or_404(Recipe, pk=pk, space=request.space) link = ShareLink.objects.create(recipe=recipe, created_by=request.user, space=request.space) - return JsonResponse( - {'pk': pk, 'share': link.uuid, - 'link': request.build_absolute_uri(reverse('view_recipe', args=[pk, link.uuid]))}) + return JsonResponse({'pk': pk, 'share': link.uuid, + 'link': request.build_absolute_uri(reverse('view_recipe', args=[pk, link.uuid]))}) else: return JsonResponse({'error': 'sharing_disabled'}, status=403) diff --git a/docs/features/connectors.md b/docs/features/connectors.md new file mode 100644 index 00000000..2687f7af --- /dev/null +++ b/docs/features/connectors.md @@ -0,0 +1,43 @@ +!!! warning + Connectors are currently in a beta stage. + +## Connectors + +Connectors are a powerful add-on component to TandoorRecipes. +They allow for certain actions to be translated to api calls to external services. + +### General Config + +!!! danger + In order for this application to push data to external providers it needs to store authentication information. + Please use read only/separate accounts or app passwords wherever possible. + +- `DISABLE_EXTERNAL_CONNECTORS` is a global switch to disable External Connectors entirely. +- `EXTERNAL_CONNECTORS_QUEUE_SIZE` is the amount of changes that are kept in memory if the worker cannot keep up. + +Example Config +```env +DISABLE_EXTERNAL_CONNECTORS=0 // 0 = connectors enabled, 1 = connectors enabled +EXTERNAL_CONNECTORS_QUEUE_SIZE=100 +``` + +## Current Connectors + +### HomeAssistant + +The current HomeAssistant connector supports the following features: +1. Push newly created shopping list items. +2. Pushes all shopping list items if a recipe is added to the shopping list. +3. Removed todo's from HomeAssistant IF they are unchanged and are removed through TandoorRecipes. + +#### How to configure: + +Step 1: +1. Generate a HomeAssistant Long-Lived Access Tokens +![Profile Page](https://github.com/TandoorRecipes/recipes/assets/7824786/15ebeec5-5be3-48db-97d1-c698405db533) +2. Get/create a todo list entry you want to sync too. +![Todos Viewer](https://github.com/TandoorRecipes/recipes/assets/7824786/506c4c34-3d40-48ae-a80c-e50374c5c618) +3. Create a connector +![New Connector Config](https://github.com/TandoorRecipes/recipes/assets/7824786/7f14f181-1341-4cab-959b-a6bef79e2159) +4. ??? +5. Profit diff --git a/docs/system/configuration.md b/docs/system/configuration.md index 0eb1b783..8d93f985 100644 --- a/docs/system/configuration.md +++ b/docs/system/configuration.md @@ -437,16 +437,6 @@ key [here](https://fdc.nal.usda.gov/api-key-signup.html). FDC_API_KEY=DEMO_KEY ``` -#### External Connectors - -`DISABLE_EXTERNAL_CONNECTORS` is a global switch to disable External Connectors entirely (e.g. HomeAssistant). -`EXTERNAL_CONNECTORS_QUEUE_SIZE` is the amount of changes that are kept in memory if the worker cannot keep up. - -```env -DISABLE_EXTERNAL_CONNECTORS=0 // 0 = connectors enabled, 1 = connectors enabled -EXTERNAL_CONNECTORS_QUEUE_SIZE=25 -``` - ### Debugging/Development settings !!! warning diff --git a/recipes/settings.py b/recipes/settings.py index 009c6a5a..f14d2701 100644 --- a/recipes/settings.py +++ b/recipes/settings.py @@ -557,6 +557,6 @@ ACCOUNT_EMAIL_SUBJECT_PREFIX = os.getenv( 'ACCOUNT_EMAIL_SUBJECT_PREFIX', '[Tandoor Recipes] ') # allauth sender prefix DISABLE_EXTERNAL_CONNECTORS = bool(int(os.getenv('DISABLE_EXTERNAL_CONNECTORS', False))) -EXTERNAL_CONNECTORS_QUEUE_SIZE = int(os.getenv('EXTERNAL_CONNECTORS_QUEUE_SIZE', 25)) +EXTERNAL_CONNECTORS_QUEUE_SIZE = int(os.getenv('EXTERNAL_CONNECTORS_QUEUE_SIZE', 100)) mimetypes.add_type("text/javascript", ".js", True) From 074244ee127e99a246469ae736b3f0e9e98c5daa Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Mon, 5 Feb 2024 23:35:39 +0100 Subject: [PATCH 27/41] add timeout to async test --- cookbook/tests/other/test_connector_manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cookbook/tests/other/test_connector_manager.py b/cookbook/tests/other/test_connector_manager.py index b8610082..e6009f6b 100644 --- a/cookbook/tests/other/test_connector_manager.py +++ b/cookbook/tests/other/test_connector_manager.py @@ -15,6 +15,7 @@ def obj_1(space_1, u1_s1): return e +@pytest.mark.timeout(10) @pytest.mark.asyncio async def test_run_connectors(space_1, u1_s1, obj_1) -> None: connector_mock = Mock(spec=Connector) From 0279013f72493b2e3b364dfdcf3992ae4af0b7e7 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Mon, 5 Feb 2024 23:37:18 +0100 Subject: [PATCH 28/41] remove loop closing --- cookbook/connectors/connector_manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 90ad1bfd..8a35ba74 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -132,7 +132,6 @@ class ConnectorManager: loop.run_until_complete(run_connectors(connectors, space, item.instance, item.actionType)) worker_queue.task_done() - loop.close() @staticmethod def get_connected_for_config(config: ConnectorConfig) -> Optional[Connector]: From 0e945f4bd77c99de95291b27954493496c2ebe60 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Mon, 5 Feb 2024 23:40:50 +0100 Subject: [PATCH 29/41] add startup & termination log to worker --- cookbook/connectors/connector_manager.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 8a35ba74..bc3293ff 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -9,8 +9,8 @@ from multiprocessing import JoinableQueue from types import UnionType from typing import List, Any, Dict, Optional, Type -from django_scopes import scope from django.conf import settings +from django_scopes import scope from cookbook.connectors.connector import Connector from cookbook.connectors.homeassistant import HomeAssistant @@ -47,7 +47,7 @@ class ConnectorManager: def __init__(self): self._queue = multiprocessing.JoinableQueue(maxsize=settings.EXTERNAL_CONNECTORS_QUEUE_SIZE) - self._worker = multiprocessing.Process(target=self.worker, args=(self._queue,), daemon=True) + self._worker = multiprocessing.Process(target=self.worker, args=(0, self._queue,), daemon=True) self._worker.start() # Called by post save & post delete signals @@ -77,7 +77,7 @@ class ConnectorManager: self._worker.join() @staticmethod - def worker(worker_queue: JoinableQueue): + def worker(worker_id: int, worker_queue: JoinableQueue): # https://stackoverflow.com/a/10684672 Close open connections after starting a new process to prevent re-use of same connections from django.db import connections connections.close_all() @@ -85,7 +85,9 @@ class ConnectorManager: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - # + logging.info(f"started ConnectionManager worker {worker_id}") + + # When multiple workers are used, please make sure the cache is shared across all threads, otherwise it might lead to un-expected behavior. _connectors_cache: Dict[int, List[Connector]] = dict() while True: @@ -132,6 +134,7 @@ class ConnectorManager: loop.run_until_complete(run_connectors(connectors, space, item.instance, item.actionType)) worker_queue.task_done() + logging.info(f"terminating ConnectionManager worker {worker_id}") @staticmethod def get_connected_for_config(config: ConnectorConfig) -> Optional[Connector]: From 408c2271a696a4a07850310b46d5b31246c51c94 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Mon, 5 Feb 2024 23:43:13 +0100 Subject: [PATCH 30/41] reduce timeout, remove report generation --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 56148b3b..ca7ea735 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,8 +78,8 @@ jobs: ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}-collectstatic-${{ hashFiles('**/*.css', '**/*.js', 'vue/src/*') }} - name: Django Testing project - timeout-minutes: 15 - run: pytest --junitxml=junit/test-results-${{ matrix.python-version }}.xml + timeout-minutes: 6 + run: pytest - name: Publish Test Results uses: EnricoMi/publish-unit-test-result-action@v2 From 2a6c13fc5ca0cde8a84b12fe9af0dd32790c9b4a Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Mon, 5 Feb 2024 23:50:57 +0100 Subject: [PATCH 31/41] add finalizer to stop worker on terminate --- .github/workflows/ci.yml | 2 +- cookbook/connectors/connector_manager.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca7ea735..6eb6195d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,7 +79,7 @@ jobs: - name: Django Testing project timeout-minutes: 6 - run: pytest + run: pytest --junitxml=junit/test-results-${{ matrix.python-version }}.xml - name: Publish Test Results uses: EnricoMi/publish-unit-test-result-action@v2 diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index bc3293ff..132bc56e 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -2,6 +2,7 @@ import asyncio import logging import multiprocessing import queue +import weakref from asyncio import Task from dataclasses import dataclass from enum import Enum @@ -49,6 +50,7 @@ class ConnectorManager: self._queue = multiprocessing.JoinableQueue(maxsize=settings.EXTERNAL_CONNECTORS_QUEUE_SIZE) self._worker = multiprocessing.Process(target=self.worker, args=(0, self._queue,), daemon=True) self._worker.start() + self._finalizer = weakref.finalize(self, self.stop) # Called by post save & post delete signals def __call__(self, instance: Any, **kwargs) -> None: From 16e8c1e8e337050a02180067060ee5476bca6f60 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Mon, 5 Feb 2024 23:59:40 +0100 Subject: [PATCH 32/41] disable connector in tests --- cookbook/connectors/connector_manager.py | 5 +++-- pytest.ini | 4 +++- requirements.txt | 1 + 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 132bc56e..60a0d5e2 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -2,7 +2,6 @@ import asyncio import logging import multiprocessing import queue -import weakref from asyncio import Task from dataclasses import dataclass from enum import Enum @@ -50,7 +49,6 @@ class ConnectorManager: self._queue = multiprocessing.JoinableQueue(maxsize=settings.EXTERNAL_CONNECTORS_QUEUE_SIZE) self._worker = multiprocessing.Process(target=self.worker, args=(0, self._queue,), daemon=True) self._worker.start() - self._finalizer = weakref.finalize(self, self.stop) # Called by post save & post delete signals def __call__(self, instance: Any, **kwargs) -> None: @@ -138,6 +136,9 @@ class ConnectorManager: logging.info(f"terminating ConnectionManager worker {worker_id}") + asyncio.set_event_loop(None) + loop.close() + @staticmethod def get_connected_for_config(config: ConnectorConfig) -> Optional[Connector]: match config.type: diff --git a/pytest.ini b/pytest.ini index d766a842..3738a6c8 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,5 @@ [pytest] DJANGO_SETTINGS_MODULE = recipes.settings -python_files = tests.py test_*.py *_tests.py \ No newline at end of file +python_files = tests.py test_*.py *_tests.py +env = + DISABLE_EXTERNAL_CONNECTORS = 1 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 7f08008a..a9d1029b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,6 +36,7 @@ django-scopes==2.0.0 pytest==7.4.3 pytest-asyncio==0.23.3 pytest-django==4.6.0 +pytest-env==1.1.3 django-treebeard==4.7 django-cors-headers==4.2.0 django-storages==1.14.2 From 2bfc8b07171a83f696971b989b165de43bff9d38 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Tue, 6 Feb 2024 00:11:46 +0100 Subject: [PATCH 33/41] format --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 3738a6c8..bdfb66bc 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,4 +2,4 @@ DJANGO_SETTINGS_MODULE = recipes.settings python_files = tests.py test_*.py *_tests.py env = - DISABLE_EXTERNAL_CONNECTORS = 1 \ No newline at end of file + DISABLE_EXTERNAL_CONNECTORS=1 \ No newline at end of file From 65a7c82af9e733d622f1e86142de042528f3124b Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Tue, 6 Feb 2024 00:17:23 +0100 Subject: [PATCH 34/41] terminate worker on finalize --- cookbook/connectors/connector_manager.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index 60a0d5e2..f32e49c8 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -2,6 +2,7 @@ import asyncio import logging import multiprocessing import queue +import weakref from asyncio import Task from dataclasses import dataclass from enum import Enum @@ -49,6 +50,7 @@ class ConnectorManager: self._queue = multiprocessing.JoinableQueue(maxsize=settings.EXTERNAL_CONNECTORS_QUEUE_SIZE) self._worker = multiprocessing.Process(target=self.worker, args=(0, self._queue,), daemon=True) self._worker.start() + weakref.finalize(self, self._worker.terminate) # Called by post save & post delete signals def __call__(self, instance: Any, **kwargs) -> None: From 962d61783991cea382855b794c8fe1779e966da7 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Tue, 6 Feb 2024 00:37:37 +0100 Subject: [PATCH 35/41] switch to threading, f multiprocessing in python --- cookbook/connectors/connector_manager.py | 20 +++++--------------- pytest.ini | 2 -- requirements.txt | 1 - 3 files changed, 5 insertions(+), 18 deletions(-) diff --git a/cookbook/connectors/connector_manager.py b/cookbook/connectors/connector_manager.py index f32e49c8..e247db30 100644 --- a/cookbook/connectors/connector_manager.py +++ b/cookbook/connectors/connector_manager.py @@ -1,12 +1,10 @@ import asyncio import logging -import multiprocessing import queue -import weakref +import threading from asyncio import Task from dataclasses import dataclass from enum import Enum -from multiprocessing import JoinableQueue from types import UnionType from typing import List, Any, Dict, Optional, Type @@ -17,8 +15,6 @@ from cookbook.connectors.connector import Connector from cookbook.connectors.homeassistant import HomeAssistant from cookbook.models import ShoppingListEntry, Space, ConnectorConfig -multiprocessing.set_start_method('fork') # https://code.djangoproject.com/ticket/31169 - REGISTERED_CLASSES: UnionType | Type = ShoppingListEntry @@ -43,14 +39,13 @@ class Work: # 4. Work is marked as consumed, and next entry of the queue is consumed. # Each 'Work' is processed in sequential by the worker, so the throughput is about [workers * the slowest connector] class ConnectorManager: - _queue: JoinableQueue + _queue: queue.Queue _listening_to_classes = REGISTERED_CLASSES | ConnectorConfig def __init__(self): - self._queue = multiprocessing.JoinableQueue(maxsize=settings.EXTERNAL_CONNECTORS_QUEUE_SIZE) - self._worker = multiprocessing.Process(target=self.worker, args=(0, self._queue,), daemon=True) + self._queue = queue.Queue(maxsize=settings.EXTERNAL_CONNECTORS_QUEUE_SIZE) + self._worker = threading.Thread(target=self.worker, args=(0, self._queue,), daemon=True) self._worker.start() - weakref.finalize(self, self._worker.terminate) # Called by post save & post delete signals def __call__(self, instance: Any, **kwargs) -> None: @@ -75,15 +70,10 @@ class ConnectorManager: def stop(self): self._queue.join() - self._queue.close() self._worker.join() @staticmethod - def worker(worker_id: int, worker_queue: JoinableQueue): - # https://stackoverflow.com/a/10684672 Close open connections after starting a new process to prevent re-use of same connections - from django.db import connections - connections.close_all() - + def worker(worker_id: int, worker_queue: queue.Queue): loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) diff --git a/pytest.ini b/pytest.ini index bdfb66bc..79e7a4e2 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,3 @@ [pytest] DJANGO_SETTINGS_MODULE = recipes.settings python_files = tests.py test_*.py *_tests.py -env = - DISABLE_EXTERNAL_CONNECTORS=1 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index a9d1029b..7f08008a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,7 +36,6 @@ django-scopes==2.0.0 pytest==7.4.3 pytest-asyncio==0.23.3 pytest-django==4.6.0 -pytest-env==1.1.3 django-treebeard==4.7 django-cors-headers==4.2.0 django-storages==1.14.2 From 1dc9244ac2a5b3ca0e8452d0f507a78a19722dfd Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Tue, 6 Feb 2024 00:47:46 +0100 Subject: [PATCH 36/41] dont use timezone in test --- cookbook/tests/other/test_recipe_full_text_search.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cookbook/tests/other/test_recipe_full_text_search.py b/cookbook/tests/other/test_recipe_full_text_search.py index eed33887..b6b7a858 100644 --- a/cookbook/tests/other/test_recipe_full_text_search.py +++ b/cookbook/tests/other/test_recipe_full_text_search.py @@ -1,8 +1,9 @@ import itertools import json -from datetime import timedelta +from datetime import timedelta, datetime import pytest +import pytz from django.conf import settings from django.contrib import auth from django.urls import reverse @@ -343,7 +344,7 @@ def test_search_date(found_recipe, recipes, param_type, result, u1_s1, u2_s1, sp Recipe.objects.filter(id=recipe.id).update( updated_at=recipe.created_at) - date = (timezone.now() - timedelta(days=15)).strftime("%Y-%m-%d") + date = (datetime.now() - timedelta(days=15)).strftime("%Y-%m-%d") param1 = f"?{param_type}={date}" param2 = f"?{param_type}=-{date}" r = json.loads(u1_s1.get(reverse(LIST_URL) + f'{param1}').content) From 20e1435abf71716d1500b8587e389b76ce3dcf11 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Thu, 8 Feb 2024 17:28:33 +0100 Subject: [PATCH 37/41] remove migration --- cookbook/migrations/0210_connectorconfig.py | 36 --------------------- 1 file changed, 36 deletions(-) delete mode 100644 cookbook/migrations/0210_connectorconfig.py diff --git a/cookbook/migrations/0210_connectorconfig.py b/cookbook/migrations/0210_connectorconfig.py deleted file mode 100644 index 3d88816f..00000000 --- a/cookbook/migrations/0210_connectorconfig.py +++ /dev/null @@ -1,36 +0,0 @@ -# Generated by Django 4.2.7 on 2024-02-02 19:51 - -import cookbook.models -from django.conf import settings -import django.core.validators -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('cookbook', '0209_remove_space_use_plural'), - ] - - operations = [ - migrations.CreateModel( - name='ConnectorConfig', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=128, validators=[django.core.validators.MinLengthValidator(1)])), - ('type', models.CharField(choices=[('HomeAssistant', 'HomeAssistant')], default='HomeAssistant', max_length=128)), - ('enabled', models.BooleanField(default=True, help_text='Is Connector Enabled')), - ('on_shopping_list_entry_created_enabled', models.BooleanField(default=False)), - ('on_shopping_list_entry_updated_enabled', models.BooleanField(default=False)), - ('on_shopping_list_entry_deleted_enabled', models.BooleanField(default=False)), - ('url', models.URLField(blank=True, null=True)), - ('token', models.CharField(blank=True, max_length=512, null=True)), - ('todo_entity', models.CharField(blank=True, max_length=128, null=True)), - ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)), - ('space', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='cookbook.space')), - ], - bases=(models.Model, cookbook.models.PermissionModelMixin), - ), - ] From 6ce95fb393ab75b30539e1459f656f38fcafe487 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Tue, 20 Feb 2024 09:25:04 +0100 Subject: [PATCH 38/41] add reference to the feature configuration in configuration.md --- docs/system/configuration.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/system/configuration.md b/docs/system/configuration.md index 8d93f985..2d7b288c 100644 --- a/docs/system/configuration.md +++ b/docs/system/configuration.md @@ -218,7 +218,18 @@ SESSION_COOKIE_NAME=sessionid # use this only to not interfere with non unified ### Features Some features can be enabled/disabled on a server level because they might change the user experience significantly, -they might be unstable/beta or they have performance/security implications. +they might be unstable/beta, or they have performance/security implications. + +For more see configurations: +- [Authentication](https://docs.tandoor.dev/features/authentication/) +- [Automation](https://docs.tandoor.dev/features/automation/) +- [Connectors](https://docs.tandoor.dev/features/connectors/) +- [External Recipes](https://docs.tandoor.dev/features/external_recipes/) +- [Import/Export](https://docs.tandoor.dev/features/import_export/) +- [Shopping](https://docs.tandoor.dev/features/shopping/) +- [Telegram Bot](https://docs.tandoor.dev/features/telegram_bot/) +- [Templating](https://docs.tandoor.dev/features/templating/) + #### Captcha From 5e508944a32c889009a4e67c65f5b6649c9fda64 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Tue, 20 Feb 2024 17:48:35 +0100 Subject: [PATCH 39/41] move env settings to configuration with backlink from connectors page --- docs/features/connectors.md | 11 +---------- docs/system/configuration.md | 25 +++++++++++++------------ 2 files changed, 14 insertions(+), 22 deletions(-) diff --git a/docs/features/connectors.md b/docs/features/connectors.md index 2687f7af..1f9a1b9c 100644 --- a/docs/features/connectors.md +++ b/docs/features/connectors.md @@ -6,20 +6,11 @@ Connectors are a powerful add-on component to TandoorRecipes. They allow for certain actions to be translated to api calls to external services. -### General Config - !!! danger In order for this application to push data to external providers it needs to store authentication information. Please use read only/separate accounts or app passwords wherever possible. -- `DISABLE_EXTERNAL_CONNECTORS` is a global switch to disable External Connectors entirely. -- `EXTERNAL_CONNECTORS_QUEUE_SIZE` is the amount of changes that are kept in memory if the worker cannot keep up. - -Example Config -```env -DISABLE_EXTERNAL_CONNECTORS=0 // 0 = connectors enabled, 1 = connectors enabled -EXTERNAL_CONNECTORS_QUEUE_SIZE=100 -``` +for the configuration please see [Configuration](https://docs.tandoor.dev/system/configuration/#connectors) ## Current Connectors diff --git a/docs/system/configuration.md b/docs/system/configuration.md index 2d7b288c..6d5f1ce6 100644 --- a/docs/system/configuration.md +++ b/docs/system/configuration.md @@ -218,18 +218,7 @@ SESSION_COOKIE_NAME=sessionid # use this only to not interfere with non unified ### Features Some features can be enabled/disabled on a server level because they might change the user experience significantly, -they might be unstable/beta, or they have performance/security implications. - -For more see configurations: -- [Authentication](https://docs.tandoor.dev/features/authentication/) -- [Automation](https://docs.tandoor.dev/features/automation/) -- [Connectors](https://docs.tandoor.dev/features/connectors/) -- [External Recipes](https://docs.tandoor.dev/features/external_recipes/) -- [Import/Export](https://docs.tandoor.dev/features/import_export/) -- [Shopping](https://docs.tandoor.dev/features/shopping/) -- [Telegram Bot](https://docs.tandoor.dev/features/telegram_bot/) -- [Templating](https://docs.tandoor.dev/features/templating/) - +they might be unstable/beta or they have performance/security implications. #### Captcha @@ -448,6 +437,18 @@ key [here](https://fdc.nal.usda.gov/api-key-signup.html). FDC_API_KEY=DEMO_KEY ``` +#### Connectors + +- `DISABLE_EXTERNAL_CONNECTORS` is a global switch to disable External Connectors entirely. +- `EXTERNAL_CONNECTORS_QUEUE_SIZE` is the amount of changes that are kept in memory if the worker cannot keep up. + +(External) Connectors are used to sync the status from Tandoor to other services. More info can be found [here](https://docs.tandoor.dev/features/connectors/). + +```env +DISABLE_EXTERNAL_CONNECTORS=0 # Default 0 (false), set to 1 (true) to disable connectors +EXTERNAL_CONNECTORS_QUEUE_SIZE=100 # Defaults to 100, set to any number >1 +``` + ### Debugging/Development settings !!! warning From 8f3effe194c6c58668e30f9c7962fec64e433a09 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Tue, 20 Feb 2024 17:53:26 +0100 Subject: [PATCH 40/41] bump pytest-asyncio for pytest 8.0.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ae4ded83..5d3dbab0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,7 +34,7 @@ django-allauth==0.61.1 recipe-scrapers==14.52.0 django-scopes==2.0.0 pytest==8.0.0 -pytest-asyncio==0.23.3 +pytest-asyncio==0.23.5 pytest-django==4.8.0 django-treebeard==4.7 django-cors-headers==4.2.0 From 4e43a7a325fa6d2e47a314dcfa492143bdcad168 Mon Sep 17 00:00:00 2001 From: Mikhail Epifanov Date: Tue, 20 Feb 2024 17:54:58 +0100 Subject: [PATCH 41/41] add connectors to mkdocs --- mkdocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/mkdocs.yml b/mkdocs.yml index b7f50de2..644396fd 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -42,6 +42,7 @@ nav: - Shopping: features/shopping.md - Authentication: features/authentication.md - Automation: features/automation.md + - Connectors: features/connectors.md - Storages and Sync: features/external_recipes.md - Import/Export: features/import_export.md - System: