From c20e4d7c9ab365918d4523692245faed82ce5eaf Mon Sep 17 00:00:00 2001
From: Eliot Berriot <contact@eliotberriot.com>
Date: Fri, 13 Apr 2018 21:31:40 +0200
Subject: [PATCH] Added task to delete unused cached files
---
.../dynamic_preferences_registry.py | 14 +++++++++
api/funkwhale_api/federation/factories.py | 5 ++++
api/funkwhale_api/federation/tasks.py | 24 +++++++++++++++
api/tests/federation/test_tasks.py | 30 +++++++++++++++++++
4 files changed, 73 insertions(+)
diff --git a/api/funkwhale_api/federation/dynamic_preferences_registry.py b/api/funkwhale_api/federation/dynamic_preferences_registry.py
index c7cb015a..43877c75 100644
--- a/api/funkwhale_api/federation/dynamic_preferences_registry.py
+++ b/api/funkwhale_api/federation/dynamic_preferences_registry.py
@@ -4,3 +4,17 @@ from dynamic_preferences import types
from dynamic_preferences.registries import global_preferences_registry
federation = types.Section('federation')
+
+
+@global_preferences_registry.register
+class MusicCacheDuration(types.IntPreference):
+ show_in_api = True
+ section = federation
+ name = 'music_cache_duration'
+ default = 60 * 24 * 2
+ verbose_name = 'Music cache duration'
+ help_text = (
+ 'How much minutes do you want to keep a copy of federated tracks'
+ 'locally? Federated files that were not listened in this interval '
+ 'will be erased and refetched from the remote on the next listening.'
+ )
diff --git a/api/funkwhale_api/federation/factories.py b/api/funkwhale_api/federation/factories.py
index 1aeb733c..0754c4b2 100644
--- a/api/funkwhale_api/federation/factories.py
+++ b/api/funkwhale_api/federation/factories.py
@@ -185,6 +185,11 @@ class LibraryTrackFactory(factory.DjangoModelFactory):
class Meta:
model = models.LibraryTrack
+ class Params:
+ with_audio_file = factory.Trait(
+ audio_file=factory.django.FileField()
+ )
+
@registry.register(name='federation.Note')
class NoteFactory(factory.Factory):
diff --git a/api/funkwhale_api/federation/tasks.py b/api/funkwhale_api/federation/tasks.py
index c6a70174..adc354c4 100644
--- a/api/funkwhale_api/federation/tasks.py
+++ b/api/funkwhale_api/federation/tasks.py
@@ -1,3 +1,4 @@
+import datetime
import json
import logging
@@ -5,8 +6,10 @@ from django.conf import settings
from django.utils import timezone
from requests.exceptions import RequestException
+from dynamic_preferences.registries import global_preferences_registry
from funkwhale_api.common import session
+from funkwhale_api.history.models import Listening
from funkwhale_api.taskapp import celery
from . import actors
@@ -85,3 +88,24 @@ def scan_library_page(library, page_url, until=None):
next_page = data.get('next')
if next_page and next_page != page_url:
scan_library_page.delay(library_id=library.id, page_url=next_page)
+
+
+@celery.app.task(name='federation.clean_music_cache')
+def clean_music_cache():
+ preferences = global_preferences_registry.manager()
+ delay = preferences['federation__music_cache_duration']
+ if delay < 1:
+ return # cache clearing disabled
+
+ candidates = models.LibraryTrack.objects.filter(
+ audio_file__isnull=False
+ ).values_list('local_track_file__track', flat=True)
+ listenings = Listening.objects.filter(
+ creation_date__gte=timezone.now() - datetime.timedelta(minutes=delay),
+ track__pk__in=candidates).values_list('track', flat=True)
+ too_old = set(candidates) - set(listenings)
+
+ to_remove = models.LibraryTrack.objects.filter(
+ local_track_file__track__pk__in=too_old).only('audio_file')
+ for lt in to_remove:
+ lt.audio_file.delete()
diff --git a/api/tests/federation/test_tasks.py b/api/tests/federation/test_tasks.py
index d164d62a..506fbc1f 100644
--- a/api/tests/federation/test_tasks.py
+++ b/api/tests/federation/test_tasks.py
@@ -1,3 +1,5 @@
+import datetime
+
from django.core.paginator import Paginator
from django.utils import timezone
@@ -108,3 +110,31 @@ def test_scan_page_stops_once_until_is_reached(
assert len(lts) == 2
for i, tf in enumerate(tfs[:1]):
assert tf.creation_date == lts[i].published_date
+
+
+def test_clean_federation_music_cache_if_no_listen(preferences, factories):
+ preferences['federation__music_cache_duration'] = 60
+ lt1 = factories['federation.LibraryTrack'](with_audio_file=True)
+ lt2 = factories['federation.LibraryTrack'](with_audio_file=True)
+ lt3 = factories['federation.LibraryTrack'](with_audio_file=True)
+ tf1 = factories['music.TrackFile'](library_track=lt1)
+ tf2 = factories['music.TrackFile'](library_track=lt2)
+ tf3 = factories['music.TrackFile'](library_track=lt3)
+
+ # we listen to the first one, and the second one (but weeks ago)
+ listening1 = factories['history.Listening'](
+ track=tf1.track,
+ creation_date=timezone.now())
+ listening2 = factories['history.Listening'](
+ track=tf2.track,
+ creation_date=timezone.now() - datetime.timedelta(minutes=61))
+
+ tasks.clean_music_cache()
+
+ lt1.refresh_from_db()
+ lt2.refresh_from_db()
+ lt3.refresh_from_db()
+
+ assert bool(lt1.audio_file) is True
+ assert bool(lt2.audio_file) is False
+ assert bool(lt3.audio_file) is False
--
GitLab