Merge pull request #2730 from smilerz/orphan_file_cleanup

view and delete orphaned files
This commit is contained in:
vabene1111 2023-12-16 07:36:08 +01:00 committed by GitHub
commit 6214176fe5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 250 additions and 109 deletions

View File

@ -7,7 +7,7 @@ class Round(Func):
def str2bool(v):
if type(v) == bool or v is None:
if isinstance(v, bool) or v is None:
return v
else:
return v.lower() in ("yes", "true", "1")

View File

@ -309,7 +309,7 @@ class RecipeSearch():
def _favorite_recipes(self, times_cooked=None):
if self._sort_includes('favorite') or times_cooked:
less_than = '-' in (times_cooked or []) and not self._sort_includes('-favorite')
less_than = '-' in (str(times_cooked) or []) and not self._sort_includes('-favorite')
if less_than:
default = 1000
else:

View File

@ -83,22 +83,50 @@
{% trans 'Everything is fine!' %}
{% endif %}
<h4 class="mt-3">{% trans 'Database' %} <span
class="badge badge-{% if postgres %}warning{% else %}success{% endif %}">{% if postgres %}
{% trans 'Info' %}{% else %}{% trans 'Ok' %}{% endif %}</span></h4>
{% if postgres %}
{% blocktrans %}
This application is not running with a Postgres database backend. This is ok but not recommended as some
features only work with postgres databases.
{% endblocktrans %}
<h4 class="mt-3">{% trans 'Database' %}
<span class="badge badge-{{postgres_status}}">
{% if postgres_status == 'warning' %}
{% trans 'Info' %}
{% elif postgres_status == 'danger'%}
{% trans 'Warning' %}
{% else %}
{% trans 'Everything is fine!' %}
{% trans 'Ok' %}
{% endif %}
</span>
</h4>
{{postgres_message}}
<h4 class="mt-3">
{% trans 'Orphaned Files' %}
<span class="badge badge-{% if orphans|length == 0 %}success{% elif orphans|length <= 25 %}warning{% else %}danger{% endif %}">
{% if orphans|length == 0 %}{% trans 'Success' %}
{% elif orphans|length <= 25 %}{% trans 'Warning' %}
{% else %}{% trans 'Danger' %}
{% endif %}
</span>
</h4>
{% if orphans|length == 0 %}
{% trans 'Everything is fine!' %}
{% else %}
{% blocktrans with orphan_count=orphans|length %}
There are currently {{ orphan_count }} orphaned files.
{% endblocktrans %}
<br>
<button id="toggle-button" class="btn btn-info btn-sm" onclick="toggleOrphans()">{% trans 'Show' %}</button>
<button class="btn btn-info btn-sm" onclick="deleteOrphans()">{% trans 'Delete' %}</button>
{% endif %}
<textarea id="orphans-list" style="display:none;" class="form-control" rows="20">
{% for orphan in orphans %}{{ orphan }}
{% endfor %}
</textarea>
<h4 class="mt-3">Debug</h4>
<textarea class="form-control" rows="20">
Gunicorn Media: {{ gunicorn_media }}
Sqlite: {{ postgres }}
Sqlite: {% if postgres %} {% trans 'False' %} {% else %} {% trans 'True' %} {% endif %}
{% if postgres %}PostgreSQL: {{postgres_version}} {% endif %}
Debug: {{ debug }}
{% for key,value in request.META.items %}{% if key in 'SERVER_PORT,REMOTE_HOST,REMOTE_ADDR,SERVER_PROTOCOL' %}{{ key }}:{{ value }}
@ -110,4 +138,30 @@ Debug: {{ debug }}
</textarea>
<br/>
<br/>
<form method="POST" id="delete-form">
{% csrf_token %}
<input type="hidden" name="delete_orphans" value="false">
</form>
{% block script %}
<script>
function toggleOrphans() {
var orphansList = document.getElementById('orphans-list');
var button = document.getElementById('toggle-button');
if (orphansList.style.display === 'none') {
orphansList.style.display = 'block';
button.innerText = "{% trans 'Hide' %}";
} else {
orphansList.style.display = 'none';
button.innerText = "{% trans 'Show' %}";
}
}
function deleteOrphans() {
document.getElementById('delete-form').delete_orphans.value = 'true';
document.getElementById('delete-form').submit();
}
</script>
{% endblock script %}
{% endblock %}

View File

@ -1,3 +1,3 @@
TANDOOR_VERSION = ""
TANDOOR_REF = ""
VERSION_INFO = []
TANDOOR_REF = "abf8f791360b2bc4a5c7d011877668679bcbb3f2"
VERSION_INFO = [{'name': 'Tandoor ', 'version': "commit abf8f791360b2bc4a5c7d011877668679bcbb3f2\nMerge: 4723a7ec fd028047\nAuthor: vabene1111 \nDate: Sun Dec 3 14:10:28 2023 +0100\n\n Merge branch 'develop'\n \n # Conflicts:\n # docs/faq.md\n", 'website': 'https://github.com/TandoorRecipes/recipes', 'commit_link': 'https://github.com/TandoorRecipes/recipes/commit/abf8f791360b2bc4a5c7d011877668679bcbb3f2', 'ref': 'abf8f791360b2bc4a5c7d011877668679bcbb3f2', 'branch': 'HEAD', 'tag': ''}]

View File

@ -3,12 +3,14 @@ import re
from datetime import datetime
from uuid import UUID
from django.apps import apps
from django.conf import settings
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.contrib.auth.models import Group
from django.contrib.auth.password_validation import validate_password
from django.core.exceptions import ValidationError
from django.db import models
from django.http import HttpResponseRedirect
from django.shortcuts import get_object_or_404, redirect, render
from django.urls import reverse, reverse_lazy
@ -18,6 +20,7 @@ from django_scopes import scopes_disabled
from cookbook.forms import (CommentForm, Recipe, SearchPreferenceForm, SpaceCreateForm,
SpaceJoinForm, User, UserCreateForm, UserPreference)
from cookbook.helper.HelperFunctions import str2bool
from cookbook.helper.permission_helper import (group_required, has_group_permission,
share_link_valid, switch_user_active_space)
from cookbook.models import (Comment, CookLog, InviteLink, SearchFields, SearchPreference,
@ -314,17 +317,48 @@ def system(request):
if not request.user.is_superuser:
return HttpResponseRedirect(reverse('index'))
postgres_ver = None
postgres = settings.DATABASES['default']['ENGINE'] == 'django.db.backends.postgresql'
if postgres:
postgres_current = 16 # will need to be updated as PostgreSQL releases new major versions
from decimal import Decimal
from django.db import connection
postgres_ver = Decimal(str(connection.pg_version).replace('00', '.'))
if postgres_ver >= postgres_current:
database_status = 'success'
database_message = _('Everything is fine!')
elif postgres_ver < postgres_current - 2:
database_status = 'danger'
database_message = _('PostgreSQL %(v)s is deprecated. Upgrade to a fully supported version!') % {'v': postgres_ver}
else:
database_status = 'info'
database_message = _('You are running PostgreSQL %(v1)s. PostgreSQL %(v2)s is recommended') % {'v1': postgres_ver, 'v2': postgres_current}
else:
database_status = 'info'
database_message = _('This application is not running with a Postgres database backend. This is ok but not recommended as some features only work with postgres databases.')
secret_key = False if os.getenv('SECRET_KEY') else True
if request.method == "POST":
del_orphans = request.POST.get('delete_orphans')
orphans = get_orphan_files(delete_orphans=str2bool(del_orphans))
else:
orphans = get_orphan_files()
return render(request, 'system.html', {
'gunicorn_media': settings.GUNICORN_MEDIA,
'debug': settings.DEBUG,
'postgres': postgres,
'postgres_version': postgres_ver,
'postgres_status': database_status,
'postgres_message': database_message,
'version_info': VERSION_INFO,
'plugins': PLUGINS,
'secret_key': secret_key
'secret_key': secret_key,
'orphans': orphans
})
@ -448,3 +482,47 @@ def test(request):
def test2(request):
if not settings.DEBUG:
return HttpResponseRedirect(reverse('index'))
def get_orphan_files(delete_orphans=False):
# Get list of all image files in media folder
media_dir = settings.MEDIA_ROOT
def find_orphans():
image_files = []
for root, dirs, files in os.walk(media_dir):
for file in files:
if not file.lower().endswith(('.db')) and not root.lower().endswith(('@eadir')):
full_path = os.path.join(root, file)
relative_path = os.path.relpath(full_path, media_dir)
image_files.append((relative_path, full_path))
# Get list of all image fields in models
image_fields = []
for model in apps.get_models():
for field in model._meta.get_fields():
if isinstance(field, models.ImageField) or isinstance(field, models.FileField):
image_fields.append((model, field.name))
# get all images in the database
# TODO I don't know why, but this completely bypasses scope limitations
image_paths = []
for model, field in image_fields:
image_field_paths = model.objects.values_list(field, flat=True)
image_paths.extend(image_field_paths)
# Check each image file against model image fields
return [img for img in image_files if img[0] not in image_paths]
orphans = find_orphans()
if delete_orphans:
for f in [img[1] for img in orphans]:
try:
os.remove(f)
except FileNotFoundError:
print(f"File not found: {f}")
except Exception as e:
print(f"Error deleting file {f}: {e}")
orphans = find_orphans()
return [img[1] for img in orphans]

View File

@ -438,7 +438,7 @@ for p in PLUGINS:
if p['bundle_name'] != '':
WEBPACK_LOADER[p['bundle_name']] = {
'CACHE': not DEBUG,
'BUNDLE_DIR_NAME': f'vue/', # must end with slash
'BUNDLE_DIR_NAME': 'vue/', # must end with slash
'STATS_FILE': os.path.join(p["base_path"], 'vue', 'webpack-stats.json'),
'POLL_INTERVAL': 0.1,
'TIMEOUT': None,

View File

@ -13,12 +13,11 @@ tandoor_hash = ''
try:
print('getting tandoor version')
r = subprocess.check_output(['git', 'show', '-s'], cwd=BASE_DIR).decode()
tandoor_branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], cwd=BASE_DIR).decode()
tandoor_branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], cwd=BASE_DIR).decode().replace('\n', '')
tandoor_hash = r.split('\n')[0].split(' ')[1]
try:
tandoor_tag = subprocess.check_output(['git', 'describe', '--exact-match', tandoor_hash], cwd=BASE_DIR).decode().replace('\n', '')
except:
tandoor_tag = subprocess.check_output(['git', 'describe', '--exact-match', '--tags', tandoor_hash], cwd=BASE_DIR).decode().replace('\n', '')
except BaseException:
pass
version_info.append({
@ -47,8 +46,9 @@ try:
commit_hash = r.split('\n')[0].split(' ')[1]
try:
print('running describe')
tag = subprocess.check_output(['git', 'describe', '--exact-match', commit_hash], cwd=os.path.join(BASE_DIR, 'recipes', 'plugins', d)).decode().replace('\n', '')
except:
tag = subprocess.check_output(['git', 'describe', '--exact-match', commit_hash],
cwd=os.path.join(BASE_DIR, 'recipes', 'plugins', d)).decode().replace('\n', '')
except BaseException:
tag = ''
version_info.append({
@ -66,7 +66,7 @@ try:
traceback.print_exc()
except subprocess.CalledProcessError as e:
print("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output))
except:
except BaseException:
traceback.print_exc()
with open('cookbook/version_info.py', 'w+', encoding='UTF-8') as f:

View File

@ -558,16 +558,26 @@
<b-input-group-append v-if="ui.show_makenow">
<b-input-group-text>
{{ $t("make_now") }}
<b-form-checkbox v-model="search.makenow" name="check-button"
<b-form-checkbox
v-model="search.makenow"
name="check-button"
@change="refreshData(false)"
class="shadow-none" switch style="width: 4em"/>
class="shadow-none"
switch
style="width: 4em"
/>
</b-input-group-text>
<b-input-group-text>
<span>{{ $t("make_now_count") }}</span>
<b-form-input type="number" min="0" max="20" v-model="search.makenow_count"
<b-form-input
type="number"
min="0"
max="20"
v-model="search.makenow_count"
@change="refreshData(false)"
size="sm" class="mt-1"></b-form-input>
size="sm"
class="mt-1"
></b-form-input>
</b-input-group-text>
</b-input-group-append>
</b-input-group>
@ -853,24 +863,23 @@
</template>
<script>
import Vue from "vue"
import { BootstrapVue } from "bootstrap-vue"
import VueCookies from "vue-cookies"
import "bootstrap-vue/dist/bootstrap-vue.css"
import Vue from "vue"
import VueCookies from "vue-cookies"
import moment from "moment"
import _debounce from "lodash/debounce"
import moment from "moment"
import Multiselect from "vue-multiselect"
import {ApiMixin, ResolveUrlMixin, StandardToasts, ToastMixin} from "@/utils/utils"
import LoadingSpinner from "@/components/LoadingSpinner" // TODO: is this deprecated?
import RecipeCard from "@/components/RecipeCard"
import GenericMultiselect from "@/components/GenericMultiselect"
import RecipeSwitcher from "@/components/Buttons/RecipeSwitcher"
import {ApiApiFactory} from "@/utils/openapi/api"
import {useMealPlanStore} from "@/stores/MealPlanStore"
import BottomNavigationBar from "@/components/BottomNavigationBar.vue"
import RecipeSwitcher from "@/components/Buttons/RecipeSwitcher"
import GenericMultiselect from "@/components/GenericMultiselect"
import MealPlanEditModal from "@/components/MealPlanEditModal.vue"
import RecipeCard from "@/components/RecipeCard"
import { useMealPlanStore } from "@/stores/MealPlanStore"
import { ApiApiFactory } from "@/utils/openapi/api"
import { ApiMixin, ResolveUrlMixin, StandardToasts, ToastMixin } from "@/utils/utils"
Vue.use(VueCookies)
Vue.use(BootstrapVue)
@ -986,7 +995,7 @@ export default {
date: moment_date,
create_default_date: moment_date.format("YYYY-MM-DD"), // improve meal plan edit modal to do formatting itself and accept dates
date_label: moment_date.format("dd") + " " + moment_date.format("ll"),
plan_entries: this.meal_plan_store.plan_list.filter((m) => moment_date.isBetween(moment(m.from_date), moment(m.to_date), 'day', '[]'))
plan_entries: this.meal_plan_store.plan_list.filter((m) => moment_date.isBetween(moment(m.from_date), moment(m.to_date), "day", "[]")),
})
}
}
@ -1063,7 +1072,7 @@ export default {
[this.$t("Name"), "name", "A-z", "Z-a"],
[this.$t("last_cooked"), "lastcooked", "↑", "↓"],
[this.$t("Rating"), "rating", "1-5", "5-1"],
[this.$t("times_cooked"), "favorite", "*-x", "x-*"],
[this.$t("times_cooked"), "favorite", "x-X", "X-x"],
[this.$t("date_created"), "created_at", "↑", "↓"],
[this.$t("date_viewed"), "lastviewed", "↑", "↓"],
]