From 8489c79c89293202b92db00f70e0c75b25a4cc80 Mon Sep 17 00:00:00 2001
From: Eliot Berriot <contact@eliotberriot.com>
Date: Wed, 24 Oct 2018 19:44:31 +0200
Subject: [PATCH] See #272: clean transcoding files task
---
api/config/settings/common.py | 7 ++++++-
.../music/dynamic_preferences_registry.py | 15 +++++++++++++++
api/funkwhale_api/music/factories.py | 12 ++++++++++++
api/funkwhale_api/music/tasks.py | 19 ++++++++++++++++++-
api/tests/music/test_tasks.py | 17 +++++++++++++++++
5 files changed, 68 insertions(+), 2 deletions(-)
diff --git a/api/config/settings/common.py b/api/config/settings/common.py
index 4759d7aa..a445e102 100644
--- a/api/config/settings/common.py
+++ b/api/config/settings/common.py
@@ -412,7 +412,12 @@ CELERY_BEAT_SCHEDULE = {
"task": "federation.clean_music_cache",
"schedule": crontab(hour="*/2"),
"options": {"expires": 60 * 2},
- }
+ },
+ "music.clean_transcoding_cache": {
+ "task": "music.clean_transcoding_cache",
+ "schedule": crontab(hour="*"),
+ "options": {"expires": 60 * 2},
+ },
}
JWT_AUTH = {
diff --git a/api/funkwhale_api/music/dynamic_preferences_registry.py b/api/funkwhale_api/music/dynamic_preferences_registry.py
index d4122577..0a7d781a 100644
--- a/api/funkwhale_api/music/dynamic_preferences_registry.py
+++ b/api/funkwhale_api/music/dynamic_preferences_registry.py
@@ -17,3 +17,18 @@ class MaxTracks(types.BooleanPreference):
"load on the server."
)
default = True
+
+
+@global_preferences_registry.register
+class MusicCacheDuration(types.IntPreference):
+ show_in_api = True
+ section = music
+ name = "transcoding_cache_duration"
+ default = 60 * 24 * 7
+ verbose_name = "Transcoding cache duration"
+ help_text = (
+ "How much minutes do you want to keep a copy of transcoded tracks"
+ "locally? Transcoded files that were not listened in this interval "
+ "will be erased and retranscoded from the remote on the next listening."
+ )
+ field_kwargs = {"required": False}
diff --git a/api/funkwhale_api/music/factories.py b/api/funkwhale_api/music/factories.py
index 9571f978..0ec3fdb2 100644
--- a/api/funkwhale_api/music/factories.py
+++ b/api/funkwhale_api/music/factories.py
@@ -95,6 +95,18 @@ class UploadFactory(factory.django.DjangoModelFactory):
)
+@registry.register
+class UploadVersionFactory(factory.django.DjangoModelFactory):
+ upload = factory.SubFactory(UploadFactory, bitrate=200000)
+ bitrate = factory.SelfAttribute("upload.bitrate")
+ mimetype = "audio/mpeg"
+ audio_file = factory.django.FileField()
+ size = 2000000
+
+ class Meta:
+ model = "music.UploadVersion"
+
+
@registry.register
class WorkFactory(factory.django.DjangoModelFactory):
mbid = factory.Faker("uuid4")
diff --git a/api/funkwhale_api/music/tasks.py b/api/funkwhale_api/music/tasks.py
index d96471b9..7008b12f 100644
--- a/api/funkwhale_api/music/tasks.py
+++ b/api/funkwhale_api/music/tasks.py
@@ -1,4 +1,5 @@
import collections
+import datetime
import logging
import os
@@ -10,7 +11,7 @@ from django.dispatch import receiver
from musicbrainzngs import ResponseError
from requests.exceptions import RequestException
-from funkwhale_api.common import channels
+from funkwhale_api.common import channels, preferences
from funkwhale_api.federation import routes
from funkwhale_api.federation import library as lb
from funkwhale_api.taskapp import celery
@@ -526,3 +527,19 @@ def broadcast_import_status_update_to_owner(old_status, new_status, upload, **kw
},
},
)
+
+
+@celery.app.task(name="music.clean_transcoding_cache")
+def clean_transcoding_cache():
+ delay = preferences.get("music__transcoding_cache_duration")
+ if delay < 1:
+ return # cache clearing disabled
+ limit = timezone.now() - datetime.timedelta(minutes=delay)
+ candidates = (
+ models.UploadVersion.objects.filter(
+ (Q(accessed_date__lt=limit) | Q(accessed_date=None))
+ )
+ .only("audio_file", "id")
+ .order_by("id")
+ )
+ return candidates.delete()
diff --git a/api/tests/music/test_tasks.py b/api/tests/music/test_tasks.py
index efa0e801..10d9fba7 100644
--- a/api/tests/music/test_tasks.py
+++ b/api/tests/music/test_tasks.py
@@ -546,3 +546,20 @@ def test_scan_page_trigger_next_page_scan_skip_if_same(mocker, factories, r_mock
scan.refresh_from_db()
assert scan.status == "finished"
+
+
+def test_clean_transcoding_cache(preferences, now, factories):
+ preferences['music__transcoding_cache_duration'] = 60
+ u1 = factories['music.UploadVersion'](
+ accessed_date=now - datetime.timedelta(minutes=61)
+ )
+ u2 = factories['music.UploadVersion'](
+ accessed_date=now - datetime.timedelta(minutes=59)
+ )
+
+ tasks.clean_transcoding_cache()
+
+ u2.refresh_from_db()
+
+ with pytest.raises(u1.__class__.DoesNotExist):
+ u1.refresh_from_db()
\ No newline at end of file
--
GitLab