diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 6da6f4bae0..dce0ccafd9 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -243,6 +243,7 @@ # timeout for a query to an individual connector QUERY_TIMEOUT = env.int("INTERACTIVE_QUERY_TIMEOUT", env.int("QUERY_TIMEOUT", 5)) +CACHE_KEY_PREFIX = "django_cache" # Redis cache backend if env.bool("USE_DUMMY_CACHE", False): CACHES = { @@ -259,6 +260,7 @@ "default": { "BACKEND": "django.core.cache.backends.redis.RedisCache", "LOCATION": REDIS_ACTIVITY_URL, + "KEY_PREFIX": CACHE_KEY_PREFIX, }, "file_resubmit": { "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", diff --git a/bookwyrm/templates/settings/layout.html b/bookwyrm/templates/settings/layout.html index 70c7ef0f47..e61900e338 100644 --- a/bookwyrm/templates/settings/layout.html +++ b/bookwyrm/templates/settings/layout.html @@ -85,6 +85,10 @@

{% trans "System" %}

{% url 'settings-celery' as url %} {% trans "Celery status" %} +
  • + {% url 'settings-redis' as url %} + {% trans "Redis status" %} +
  • {% url 'settings-schedules' as url %} {% trans "Scheduled tasks" %} diff --git a/bookwyrm/templates/settings/redis.html b/bookwyrm/templates/settings/redis.html new file mode 100644 index 0000000000..3915544987 --- /dev/null +++ b/bookwyrm/templates/settings/redis.html @@ -0,0 +1,125 @@ +{% extends 'settings/layout.html' %} +{% load humanize %} +{% load i18n %} + +{% block title %}{% trans "Redis Status" %}{% endblock %} + +{% block header %}{% trans "Redis Status" %}{% endblock %} + +{% block panel %} + +{% if info %} +
    +

    {% trans "Info" %}

    +
    +
    +
    +

    {% trans "Used memory" %}

    +

    {{ info.used_memory_human }}

    +
    +
    +
    +
    +

    {% trans "Total system memory" %}

    +

    {{ info.total_system_memory_human }}

    +
    +
    +
    +
    +

    {% trans "Keys" %}

    +

    {{ info.db0.keys | intcomma }}

    +
    +
    +
    +
    + +
    +

    {% trans "Outdated cache keys" %}

    +
    +

    + {% blocktrans trimmed %} + This will scan for keys in the Django redis cache that use no prefix (the current prefix is {{ prefix }}), and identify Activity Streams for users with deleted accounts. + {% endblocktrans %} +

    + + {% if outdated_identified is not None %} +

    + {% blocktrans trimmed with keys=outdated_identified|intcomma %} + {{ keys }} identified + {% endblocktrans %} +

    + + {% if outdated_identified > 0 %} +
    + {% csrf_token %} + +
    + {% endif %} + {% else %} +
    + {% csrf_token %} + + +
    + {% endif %} +
    +
    + +
    +

    {% trans "Advanced" %}

    +
    + + + {% trans "Clear Django Cache" %} + + + + +

    + {% blocktrans trimmed %} + This is NOT recommended and should only be used if something has gone very wrong with your cache. All sessions will be cleared and users will be logged out of their accounts. + {% endblocktrans %} +

    + + {% if cache_deleted %} +

    + {% blocktrans trimmed with keys=cache_deleted|intcomma %} + {{ keys }} keys deleted + {% endblocktrans %} +

    + {% else %} +
    + {% csrf_token %} + +
    + +
    +
    + {% endif %} +
    +
    + +{% else %} + +
    + + + {% trans "Could not connect to Redis Activity" %} + +
    + +{% endif %} + +{% if errors %} +
    +

    {% trans "Errors" %}

    + {% for error in errors %} + 
    {{ error }}
    +{% endfor %} + +
    +{% endif %} + +{% endblock %} diff --git a/bookwyrm/tests/views/admin/test_redis.py b/bookwyrm/tests/views/admin/test_redis.py new file mode 100644 index 0000000000..e59bf32dec --- /dev/null +++ b/bookwyrm/tests/views/admin/test_redis.py @@ -0,0 +1,93 @@ +""" test for app action functionality """ +from unittest.mock import patch + +from django.contrib.auth.models import Group +from django.template.response import TemplateResponse +from django.test import TestCase +from django.test.client import RequestFactory + +from bookwyrm import models, views +from bookwyrm.management.commands import initdb +from bookwyrm.tests.validate_html import validate_html + + +class RedisStatusViews(TestCase): + """every response to a get request, html or json""" + + @classmethod + def setUpTestData(cls): + """we need basic test data and mocks""" + with ( + patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), + patch("bookwyrm.activitystreams.populate_stream_task.delay"), + patch("bookwyrm.lists_stream.populate_lists_task.delay"), + ): + cls.local_user = models.User.objects.create_user( + "mouse@local.com", + "mouse@mouse.mouse", + "password", + local=True, + localname="mouse", + ) + initdb.init_groups() + initdb.init_permissions() + group = Group.objects.get(name="admin") + cls.local_user.groups.set([group]) + models.SiteSettings.objects.create() + + def setUp(self): + """individual test setup""" + self.factory = RequestFactory() + + def test_redis_status_get(self): + """there are so many views, this just makes sure it LOADS""" + view = views.RedisStatus.as_view() + request = self.factory.get("") + request.user = self.local_user + + result = view(request) + self.assertIsInstance(result, TemplateResponse) + validate_html(result.render()) + self.assertEqual(result.status_code, 200) + + def test_redis_status_post_scan_keys(self): + """count keys in redis""" + view = views.RedisStatus.as_view() + request = self.factory.post("", {"dry_run": True}) + request.user = self.local_user + + result = view(request) + self.assertIsInstance(result, TemplateResponse) + validate_html(result.render()) + self.assertEqual(result.status_code, 200) + self.assertTrue("outdated_identified" in result.context_data) + self.assertFalse("outdated_deleted" in result.context_data) + self.assertFalse("cache_deleted" in result.context_data) + + def test_redis_status_post_erase_keys(self): + """count keys in redis""" + view = views.RedisStatus.as_view() + request = self.factory.post("") + request.user = self.local_user + + result = view(request) + self.assertIsInstance(result, TemplateResponse) + validate_html(result.render()) + self.assertEqual(result.status_code, 200) + self.assertFalse("outdated_identified" in result.context_data) + self.assertTrue("outdated_deleted" in result.context_data) + self.assertFalse("cache_deleted" in result.context_data) + + def test_redis_status_post_erase_cache(self): + """count keys in redis""" + view = views.RedisStatus.as_view() + request = self.factory.post("", {"erase_cache": True}) + request.user = self.local_user + + result = view(request) + self.assertIsInstance(result, TemplateResponse) + validate_html(result.render()) + self.assertEqual(result.status_code, 200) + self.assertFalse("outdated_identified" in result.context_data) + self.assertFalse("outdated_deleted" in result.context_data) + self.assertTrue("cache_deleted" in result.context_data) diff --git a/bookwyrm/urls.py b/bookwyrm/urls.py index cd75eb0c02..18d485a443 100644 --- a/bookwyrm/urls.py +++ b/bookwyrm/urls.py @@ -369,6 +369,7 @@ re_path( r"^settings/celery/ping/?$", views.celery_ping, name="settings-celery-ping" ), + re_path(r"^settings/redis/?$", views.RedisStatus.as_view(), name="settings-redis"), re_path( r"^settings/schedules/(?P\d+)?$", views.ScheduledTasks.as_view(), diff --git a/bookwyrm/views/__init__.py b/bookwyrm/views/__init__.py index ebc851847a..24b00fffff 100644 --- a/bookwyrm/views/__init__.py +++ b/bookwyrm/views/__init__.py @@ -5,6 +5,7 @@ from .admin.automod import AutoMod, automod_delete, run_automod from .admin.automod import schedule_automod_task, unschedule_automod_task from .admin.celery_status import CeleryStatus, celery_ping +from .admin.redis import RedisStatus from .admin.schedule import ScheduledTasks from .admin.dashboard import Dashboard from .admin.federation import Federation, FederatedServer diff --git a/bookwyrm/views/admin/redis.py b/bookwyrm/views/admin/redis.py new file mode 100644 index 0000000000..0395a15f6a --- /dev/null +++ b/bookwyrm/views/admin/redis.py @@ -0,0 +1,76 @@ +""" redis cache status """ +from django.contrib.auth.decorators import login_required, permission_required +from django.template.response import TemplateResponse +from django.utils.decorators import method_decorator +from django.views import View +import redis + +from bookwyrm import models, settings + +r = redis.from_url(settings.REDIS_ACTIVITY_URL) + +# pylint: disable= no-self-use +@method_decorator(login_required, name="dispatch") +@method_decorator( + permission_required("bookwyrm.edit_instance_settings", raise_exception=True), + name="dispatch", +) +class RedisStatus(View): + """Are your tasks running? Well you'd better go catch them""" + + def get(self, request): + """See workers and active tasks""" + data = view_data() + + return TemplateResponse(request, "settings/redis.html", data) + + def post(self, request): + """Erase invalid keys""" + dry_run = request.POST.get("dry_run") + erase_cache = request.POST.get("erase_cache") + data_key = "cache" if erase_cache else "outdated" + + if erase_cache: + patterns = [f"{settings.CACHE_KEY_PREFIX}:*:*"] + else: + patterns = [":*:*"] # this pattern is a django cache with no prefix + for user_id in models.User.objects.filter( + is_deleted=True, local=True + ).values_list("id", flat=True): + patterns.append(f"{user_id}-*") + + deleted_count = 0 + for pattern in patterns: + deleted_count += erase_keys(pattern, dry_run=dry_run) + + data = view_data() + if dry_run: + data[f"{data_key}_identified"] = deleted_count + else: + data[f"{data_key}_deleted"] = deleted_count + return TemplateResponse(request, "settings/redis.html", data) + + +def view_data(): + """Helper function to load basic info for the view""" + data = {"errors": [], "prefix": settings.CACHE_KEY_PREFIX} + try: + data["info"] = r.info() + # pylint: disable=broad-except + except Exception as err: + data["errors"].append(err) + return data + + +def erase_keys(pattern, count=1000, dry_run=False): + """Delete all redis activity keys according to a provided regex pattern""" + pipeline = r.pipeline() + key_count = 0 + for key in r.scan_iter(match=pattern, count=count): + key_count += 1 + if dry_run: + continue + pipeline.delete(key) + if not dry_run: + pipeline.execute() + return key_count