From 6a04779125d8d54ad5ff218f78146a84a4af0799 Mon Sep 17 00:00:00 2001
From: Eliot Berriot <contact@eliotberriot.com>
Date: Fri, 13 Apr 2018 21:04:55 +0200
Subject: [PATCH] Will now fetch and cache federated tracks
---
.../migrations/0005_auto_20180413_1723.py | 26 ++++++++++++
api/funkwhale_api/federation/models.py | 41 +++++++++++++++++++
api/funkwhale_api/music/views.py | 36 +++++-----------
api/tests/music/test_views.py | 10 +++--
4 files changed, 85 insertions(+), 28 deletions(-)
create mode 100644 api/funkwhale_api/federation/migrations/0005_auto_20180413_1723.py
diff --git a/api/funkwhale_api/federation/migrations/0005_auto_20180413_1723.py b/api/funkwhale_api/federation/migrations/0005_auto_20180413_1723.py
new file mode 100644
index 0000000000..00ba5c83de
--- /dev/null
+++ b/api/funkwhale_api/federation/migrations/0005_auto_20180413_1723.py
@@ -0,0 +1,26 @@
+# Generated by Django 2.0.3 on 2018-04-13 17:23
+
+import django.contrib.postgres.fields.jsonb
+import django.core.serializers.json
+from django.db import migrations, models
+import funkwhale_api.federation.models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('federation', '0004_auto_20180410_2025'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='librarytrack',
+ name='audio_file',
+ field=models.FileField(blank=True, null=True, upload_to=funkwhale_api.federation.models.get_file_path),
+ ),
+ migrations.AlterField(
+ model_name='librarytrack',
+ name='metadata',
+ field=django.contrib.postgres.fields.jsonb.JSONField(default={}, encoder=django.core.serializers.json.DjangoJSONEncoder, max_length=10000),
+ ),
+ ]
diff --git a/api/funkwhale_api/federation/models.py b/api/funkwhale_api/federation/models.py
index e841b63948..066c5847b0 100644
--- a/api/funkwhale_api/federation/models.py
+++ b/api/funkwhale_api/federation/models.py
@@ -1,4 +1,6 @@
+import os
import uuid
+import tempfile
from django.conf import settings
from django.contrib.postgres.fields import JSONField
@@ -6,6 +8,9 @@ from django.core.serializers.json import DjangoJSONEncoder
from django.db import models
from django.utils import timezone
+from funkwhale_api.common import session
+from funkwhale_api.music import utils as music_utils
+
TYPE_CHOICES = [
('Person', 'Person'),
('Application', 'Application'),
@@ -147,10 +152,23 @@ class Library(models.Model):
)
+def get_file_path(instance, filename):
+ uid = str(uuid.uuid4())
+ chunk_size = 2
+ chunks = [uid[i:i+chunk_size] for i in range(0, len(uid), chunk_size)]
+ parts = chunks[:3] + [filename]
+ return os.path.join('federation_cache', *parts)
+
+
class LibraryTrack(models.Model):
url = models.URLField(unique=True)
audio_url = models.URLField()
audio_mimetype = models.CharField(max_length=200)
+ audio_file = models.FileField(
+ upload_to=get_file_path,
+ null=True,
+ blank=True)
+
creation_date = models.DateTimeField(default=timezone.now)
modification_date = models.DateTimeField(
auto_now=True)
@@ -170,3 +188,26 @@ class LibraryTrack(models.Model):
return self.metadata['recording']['musicbrainz_id']
except KeyError:
pass
+
+ def download_audio(self):
+ from . import actors
+ auth = actors.SYSTEM_ACTORS['library'].get_request_auth()
+ remote_response = session.get_session().get(
+ self.audio_url,
+ auth=auth,
+ stream=True,
+ timeout=20,
+ verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
+ headers={
+ 'Content-Type': 'application/activity+json'
+ }
+ )
+ with remote_response as r:
+ remote_response.raise_for_status()
+ extension = music_utils.get_ext_from_type(self.audio_mimetype)
+ title = ' - '.join([self.title, self.album_title, self.artist_name])
+ filename = '{}.{}'.format(title, extension)
+ tmp_file = tempfile.TemporaryFile()
+ for chunk in r.iter_content(chunk_size=512):
+ tmp_file.write(chunk)
+ self.audio_file.save(filename, tmp_file)
diff --git a/api/funkwhale_api/music/views.py b/api/funkwhale_api/music/views.py
index d5247fbf60..e8ace1b3ab 100644
--- a/api/funkwhale_api/music/views.py
+++ b/api/funkwhale_api/music/views.py
@@ -23,7 +23,6 @@ from rest_framework import permissions
from musicbrainzngs import ResponseError
from funkwhale_api.common import utils as funkwhale_utils
-from funkwhale_api.common import session
from funkwhale_api.federation import actors
from funkwhale_api.requests.models import ImportRequest
from funkwhale_api.musicbrainz import api
@@ -206,35 +205,22 @@ class TrackFileViewSet(viewsets.ReadOnlyModelViewSet):
return Response(status=404)
mt = f.mimetype
+ audio_file = f.audio_file
try:
library_track = f.library_track
except ObjectDoesNotExist:
library_track = None
- if library_track and not f.audio_file:
- # we proxy the response to the remote library
- # since we did not mirror the file locally
+ if library_track and not audio_file:
+ if not library_track.audio_file:
+ # we need to populate from cache
+ library_track.download_audio()
+ audio_file = library_track.audio_file
mt = library_track.audio_mimetype
- file_extension = utils.get_ext_from_type(mt)
- filename = '{}.{}'.format(f.track.full_name, file_extension)
- auth = actors.SYSTEM_ACTORS['library'].get_request_auth()
- remote_response = session.get_session().get(
- library_track.audio_url,
- auth=auth,
- stream=True,
- timeout=20,
- verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
- headers={
- 'Content-Type': 'application/activity+json'
- })
- logger.debug(
- 'Proxying media request to %s', library_track.audio_url)
- response = StreamingHttpResponse(remote_response.iter_content())
- else:
- response = Response()
- filename = f.filename
- response['X-Accel-Redirect'] = "{}{}".format(
- settings.PROTECT_FILES_PATH,
- f.audio_file.url)
+ response = Response()
+ filename = f.filename
+ response['X-Accel-Redirect'] = "{}{}".format(
+ settings.PROTECT_FILES_PATH,
+ audio_file.url)
filename = "filename*=UTF-8''{}".format(
urllib.parse.quote(filename))
response["Content-Disposition"] = "attachment; {}".format(filename)
diff --git a/api/tests/music/test_views.py b/api/tests/music/test_views.py
index f18d18c861..4a7ca69c47 100644
--- a/api/tests/music/test_views.py
+++ b/api/tests/music/test_views.py
@@ -79,12 +79,16 @@ def test_can_proxy_remote_track(
settings.PROTECT_AUDIO_FILES = False
track_file = factories['music.TrackFile'](federation=True)
- r_mock.get(track_file.library_track.audio_url, body=io.StringIO('test'))
+ r_mock.get(track_file.library_track.audio_url, body=io.BytesIO(b'test'))
response = api_client.get(track_file.path)
+ library_track = track_file.library_track
+ library_track.refresh_from_db()
assert response.status_code == 200
- assert list(response.streaming_content) == [b't', b'e', b's', b't']
- assert response['Content-Type'] == track_file.library_track.audio_mimetype
+ assert response['X-Accel-Redirect'] == "{}{}".format(
+ settings.PROTECT_FILES_PATH,
+ library_track.audio_file.url)
+ assert library_track.audio_file.read() == b'test'
def test_can_create_import_from_federation_tracks(
--
GitLab