Verified Commit 6a047791 authored by Eliot Berriot's avatar Eliot Berriot
Browse files

Will now fetch and cache federated tracks

parent 3a31248a
# Generated by Django 2.0.3 on 2018-04-13 17:23
import django.contrib.postgres.fields.jsonb
import django.core.serializers.json
from django.db import migrations, models
import funkwhale_api.federation.models
class Migration(migrations.Migration):
dependencies = [
('federation', '0004_auto_20180410_2025'),
]
operations = [
migrations.AddField(
model_name='librarytrack',
name='audio_file',
field=models.FileField(blank=True, null=True, upload_to=funkwhale_api.federation.models.get_file_path),
),
migrations.AlterField(
model_name='librarytrack',
name='metadata',
field=django.contrib.postgres.fields.jsonb.JSONField(default={}, encoder=django.core.serializers.json.DjangoJSONEncoder, max_length=10000),
),
]
import os
import uuid
import tempfile
from django.conf import settings
from django.contrib.postgres.fields import JSONField
......@@ -6,6 +8,9 @@ from django.core.serializers.json import DjangoJSONEncoder
from django.db import models
from django.utils import timezone
from funkwhale_api.common import session
from funkwhale_api.music import utils as music_utils
TYPE_CHOICES = [
('Person', 'Person'),
('Application', 'Application'),
......@@ -147,10 +152,23 @@ class Library(models.Model):
)
def get_file_path(instance, filename):
uid = str(uuid.uuid4())
chunk_size = 2
chunks = [uid[i:i+chunk_size] for i in range(0, len(uid), chunk_size)]
parts = chunks[:3] + [filename]
return os.path.join('federation_cache', *parts)
class LibraryTrack(models.Model):
url = models.URLField(unique=True)
audio_url = models.URLField()
audio_mimetype = models.CharField(max_length=200)
audio_file = models.FileField(
upload_to=get_file_path,
null=True,
blank=True)
creation_date = models.DateTimeField(default=timezone.now)
modification_date = models.DateTimeField(
auto_now=True)
......@@ -170,3 +188,26 @@ class LibraryTrack(models.Model):
return self.metadata['recording']['musicbrainz_id']
except KeyError:
pass
def download_audio(self):
from . import actors
auth = actors.SYSTEM_ACTORS['library'].get_request_auth()
remote_response = session.get_session().get(
self.audio_url,
auth=auth,
stream=True,
timeout=20,
verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
headers={
'Content-Type': 'application/activity+json'
}
)
with remote_response as r:
remote_response.raise_for_status()
extension = music_utils.get_ext_from_type(self.audio_mimetype)
title = ' - '.join([self.title, self.album_title, self.artist_name])
filename = '{}.{}'.format(title, extension)
tmp_file = tempfile.TemporaryFile()
for chunk in r.iter_content(chunk_size=512):
tmp_file.write(chunk)
self.audio_file.save(filename, tmp_file)
......@@ -23,7 +23,6 @@ from rest_framework import permissions
from musicbrainzngs import ResponseError
from funkwhale_api.common import utils as funkwhale_utils
from funkwhale_api.common import session
from funkwhale_api.federation import actors
from funkwhale_api.requests.models import ImportRequest
from funkwhale_api.musicbrainz import api
......@@ -206,35 +205,22 @@ class TrackFileViewSet(viewsets.ReadOnlyModelViewSet):
return Response(status=404)
mt = f.mimetype
audio_file = f.audio_file
try:
library_track = f.library_track
except ObjectDoesNotExist:
library_track = None
if library_track and not f.audio_file:
# we proxy the response to the remote library
# since we did not mirror the file locally
if library_track and not audio_file:
if not library_track.audio_file:
# we need to populate from cache
library_track.download_audio()
audio_file = library_track.audio_file
mt = library_track.audio_mimetype
file_extension = utils.get_ext_from_type(mt)
filename = '{}.{}'.format(f.track.full_name, file_extension)
auth = actors.SYSTEM_ACTORS['library'].get_request_auth()
remote_response = session.get_session().get(
library_track.audio_url,
auth=auth,
stream=True,
timeout=20,
verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
headers={
'Content-Type': 'application/activity+json'
})
logger.debug(
'Proxying media request to %s', library_track.audio_url)
response = StreamingHttpResponse(remote_response.iter_content())
else:
response = Response()
filename = f.filename
response['X-Accel-Redirect'] = "{}{}".format(
settings.PROTECT_FILES_PATH,
f.audio_file.url)
response = Response()
filename = f.filename
response['X-Accel-Redirect'] = "{}{}".format(
settings.PROTECT_FILES_PATH,
audio_file.url)
filename = "filename*=UTF-8''{}".format(
urllib.parse.quote(filename))
response["Content-Disposition"] = "attachment; {}".format(filename)
......
......@@ -79,12 +79,16 @@ def test_can_proxy_remote_track(
settings.PROTECT_AUDIO_FILES = False
track_file = factories['music.TrackFile'](federation=True)
r_mock.get(track_file.library_track.audio_url, body=io.StringIO('test'))
r_mock.get(track_file.library_track.audio_url, body=io.BytesIO(b'test'))
response = api_client.get(track_file.path)
library_track = track_file.library_track
library_track.refresh_from_db()
assert response.status_code == 200
assert list(response.streaming_content) == [b't', b'e', b's', b't']
assert response['Content-Type'] == track_file.library_track.audio_mimetype
assert response['X-Accel-Redirect'] == "{}{}".format(
settings.PROTECT_FILES_PATH,
library_track.audio_file.url)
assert library_track.audio_file.read() == b'test'
def test_can_create_import_from_federation_tracks(
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment