Can now serve track from remote library

api/funkwhale_api/federation/activity.py

@@ -58,21 +58,12 @@ OBJECT_TYPES = [
     'Video',
 ] + ACTIVITY_TYPES
 
+
 def deliver(activity, on_behalf_of, to=[]):
     from . import actors
     logger.info('Preparing activity delivery to %s', to)
-    auth = requests_http_signature.HTTPSignatureAuth(
-        use_auth_header=False,
-        headers=[
-            '(request-target)',
-            'user-agent',
-            'host',
-            'date',
-            'content-type',],
-        algorithm='rsa-sha256',
-        key=on_behalf_of.private_key.encode('utf-8'),
-        key_id=on_behalf_of.private_key_id,
-    )
+    auth = signing.get_auth(
+        on_behalf_of.private_key, on_behalf_of.private_key_id)
     for url in to:
         recipient_actor = actors.get_actor(url)
         logger.debug('delivering to %s', recipient_actor.inbox_url)

api/funkwhale_api/federation/actors.py

@@ -13,8 +13,10 @@ from rest_framework.exceptions import PermissionDenied
 from dynamic_preferences.registries import global_preferences_registry
 
 from . import activity
+from . import keys
 from . import models
 from . import serializers
+from . import signing
 from . import utils
 
 logger = logging.getLogger(__name__)
@@ -51,24 +53,37 @@ class SystemActor(object):
     additional_attributes = {}
     manually_approves_followers = False
 
+    def get_request_auth(self):
+        actor = self.get_actor_instance()
+        return signing.get_auth(
+            actor.private_key, actor.private_key_id)
+
     def serialize(self):
         actor = self.get_actor_instance()
         serializer = serializers.ActorSerializer(actor)
         return serializer.data
 
     def get_actor_instance(self):
+        try:
+            return models.Actor.objects.get(url=self.get_actor_url())
+        except models.Actor.DoesNotExist:
+            pass
+        private, public = keys.get_key_pair()
         args = self.get_instance_argument(
             self.id,
             name=self.name,
             summary=self.summary,
             **self.additional_attributes
         )
-        url = args.pop('url')
-        a, created = models.Actor.objects.get_or_create(
-            url=url,
-            defaults=args,
-        )
-        return a
+        args['private_key'] = private.decode('utf-8')
+        args['public_key'] = public.decode('utf-8')
+        return models.Actor.objects.create(**args)
+
+    def get_actor_url(self):
+        return utils.full_url(
+            reverse(
+                'federation:instance-actors-detail',
+                kwargs={'actor': self.id}))
 
     def get_instance_argument(self, id, name, summary, **kwargs):
         preferences = global_preferences_registry.manager()
@@ -78,10 +93,7 @@ class SystemActor(object):
             'type': 'Person',
             'name': name.format(host=settings.FEDERATION_HOSTNAME),
             'manually_approves_followers': True,
-            'url': utils.full_url(
-                reverse(
-                    'federation:instance-actors-detail',
-                    kwargs={'actor': id})),
+            'url': self.get_actor_url(),
             'shared_inbox_url': utils.full_url(
                 reverse(
                     'federation:instance-actors-inbox',

api/funkwhale_api/federation/authentication.py

@@ -51,6 +51,8 @@ class SignatureAuthentication(authentication.BaseAuthentication):
     def authenticate(self, request):
         setattr(request, 'actor', None)
         actor = self.authenticate_actor(request)
+        if not actor:
+            return
         user = AnonymousUser()
         setattr(request, 'actor', actor)
         return (user, None)

api/funkwhale_api/federation/management/commands/generate_keys.py

-from django.core.management.base import BaseCommand, CommandError
-from django.db import transaction
-
-from dynamic_preferences.registries import global_preferences_registry
-
-from funkwhale_api.federation import keys
-
-
-class Command(BaseCommand):
-    help = (
-        'Generate a public/private key pair for your instance,'
-        ' for federation purposes. If a key pair already exists, does nothing.'
-    )
-
-    def add_arguments(self, parser):
-        parser.add_argument(
-            '--replace',
-            action='store_true',
-            dest='replace',
-            default=False,
-            help='Replace existing key pair, if any',
-        )
-        parser.add_argument(
-            '--noinput', '--no-input', action='store_false', dest='interactive',
-            help="Do NOT prompt the user for input of any kind.",
-        )
-
-    @transaction.atomic
-    def handle(self, *args, **options):
-        preferences = global_preferences_registry.manager()
-        existing_public = preferences['federation__public_key']
-        existing_private = preferences['federation__public_key']
-
-        if existing_public or existing_private and not options['replace']:
-            raise CommandError(
-                'Keys are already present! '
-                'Replace them with --replace if you know what you are doing.')
-
-        if options['interactive']:
-            message = (
-                'Are you sure you want to do this?\n\n'
-                "Type 'yes' to continue, or 'no' to cancel: "
-            )
-            if input(''.join(message)) != 'yes':
-                raise CommandError("Operation cancelled.")
-        private, public = keys.get_key_pair()
-        preferences['federation__public_key'] = public.decode('utf-8')
-        preferences['federation__private_key'] = private.decode('utf-8')
-
-        self.stdout.write(
-            'Your new key pair was generated.'
-            'Your public key is now:\n\n{}'.format(public.decode('utf-8'))
-        )

api/funkwhale_api/federation/migrations/0003_auto_20180407_0852.py → api/funkwhale_api/federation/migrations/0003_auto_20180407_1010.py

-# Generated by Django 2.0.3 on 2018-04-07 08:52
+# Generated by Django 2.0.3 on 2018-04-07 10:10
 
 import django.contrib.postgres.fields.jsonb
 from django.db import migrations, models
@@ -10,7 +10,6 @@ import uuid
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('music', '0022_importbatch_import_request'),
         ('federation', '0002_auto_20180403_1620'),
     ]
 
@@ -70,7 +69,6 @@ class Migration(migrations.Migration):
                 ('title', models.CharField(max_length=500)),
                 ('metadata', django.contrib.postgres.fields.jsonb.JSONField(default={}, max_length=10000)),
                 ('library', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tracks', to='federation.Library')),
-                ('local_track_file', models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='library_track', to='music.TrackFile')),
             ],
         ),
         migrations.AddField(

api/funkwhale_api/federation/models.py

@@ -192,13 +192,6 @@ class LibraryTrack(models.Model):
     published_date = models.DateTimeField(null=True, blank=True)
     library = models.ForeignKey(
         Library, related_name='tracks', on_delete=models.CASCADE)
-    local_track_file = models.OneToOneField(
-        'music.TrackFile',
-        related_name='library_track',
-        on_delete=models.CASCADE,
-        null=True,
-        blank=True,
-    )
     artist_name = models.CharField(max_length=500)
     album_title = models.CharField(max_length=500)
     title = models.CharField(max_length=500)

api/funkwhale_api/federation/signing.py

@@ -53,3 +53,18 @@ def verify_django(django_request, public_key):
             request.headers[h] = str(v)
     prepared_request = request.prepare()
     return verify(request, public_key)
+
+
+def get_auth(private_key, private_key_id):
+    return requests_http_signature.HTTPSignatureAuth(
+        use_auth_header=False,
+        headers=[
+            '(request-target)',
+            'user-agent',
+            'host',
+            'date',
+            'content-type'],
+        algorithm='rsa-sha256',
+        key=private_key.encode('utf-8'),
+        key_id=private_key_id,
+    )

api/funkwhale_api/music/factories.py

@@ -56,6 +56,18 @@ class TrackFileFactory(factory.django.DjangoModelFactory):
     class Meta:
         model = 'music.TrackFile'
 
+    class Params:
+        federation = factory.Trait(
+            audio_file=None,
+            library_track=factory.SubFactory(LibraryTrackFactory),
+            mimetype=factory.LazyAttribute(
+                lambda o: o.library_track.audio_mimetype
+            ),
+            source=factory.LazyAttribute(
+                lambda o: o.library_track.audio_url
+            ),
+        )
+
 
 @registry.register
 class ImportBatchFactory(factory.django.DjangoModelFactory):

api/funkwhale_api/music/migrations/0023_auto_20180407_0852.py → api/funkwhale_api/music/migrations/0023_auto_20180407_1010.py

-# Generated by Django 2.0.3 on 2018-04-07 08:52
+# Generated by Django 2.0.3 on 2018-04-07 10:10
 
 from django.conf import settings
 from django.db import migrations, models
@@ -10,7 +10,7 @@ import uuid
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('federation', '0003_auto_20180407_0852'),
+        ('federation', '0003_auto_20180407_1010'),
         ('music', '0022_importbatch_import_request'),
     ]
 
@@ -55,6 +55,11 @@ class Migration(migrations.Migration):
             name='creation_date',
             field=models.DateTimeField(default=django.utils.timezone.now),
         ),
+        migrations.AddField(
+            model_name='trackfile',
+            name='library_track',
+            field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='local_track_file', to='federation.LibraryTrack'),
+        ),
         migrations.AddField(
             model_name='trackfile',
             name='modification_date',

api/funkwhale_api/music/models.py

@@ -419,6 +419,14 @@ class TrackFile(models.Model):
     acoustid_track_id = models.UUIDField(null=True, blank=True)
     mimetype = models.CharField(null=True, blank=True, max_length=200)
 
+    library_track = models.OneToOneField(
+        'federation.LibraryTrack',
+        related_name='local_track_file',
+        on_delete=models.CASCADE,
+        null=True,
+        blank=True,
+    )
+
     def download_file(self):
         # import the track file, since there is not any
         # we create a tmp dir for the download
@@ -441,10 +449,8 @@ class TrackFile(models.Model):
 
     @property
     def path(self):
-        if settings.PROTECT_AUDIO_FILES:
-            return reverse(
-                'api:v1:trackfiles-serve', kwargs={'pk': self.pk})
-        return self.audio_file.url
+        return reverse(
+            'api:v1:trackfiles-serve', kwargs={'pk': self.pk})
 
     @property
     def filename(self):

api/funkwhale_api/music/permissions.py

+from django.conf import settings
+
+from rest_framework.permissions import BasePermission
+
+from funkwhale_api.federation import actors
+
+
+class Listen(BasePermission):
+
+    def has_permission(self, request, view):
+        if not settings.PROTECT_AUDIO_FILES:
+            return True
+
+        user = getattr(request, 'user', None)
+        if user and user.is_authenticated:
+            return True
+
+        actor = getattr(request, 'actor', None)
+        if actor is None:
+            return False
+
+        library = actors.SYSTEM_ACTORS['library'].get_actor_instance()
+        return library.followers.filter(url=actor.url).exists()

api/funkwhale_api/music/tasks.py

@@ -112,6 +112,7 @@ def _do_import(import_job, replace, use_acoustid=True):
         track_file.audio_file.name = import_job.audio_file.name
         track_file.duration = duration
     elif import_job.library_track:
+        track_file.library_track = import_job.library_track
         track_file.mimetype = import_job.library_track.audio_mimetype
         if import_job.library_track.library.download_files:
             raise NotImplementedError()
@@ -121,10 +122,6 @@ def _do_import(import_job, replace, use_acoustid=True):
     else:
         track_file.download_file()
     track_file.save()
-    if import_job.library_track:
-        import_job.library_track.local_track_file = track_file
-        import_job.library_track.save(
-            update_fields=['modification_date', 'local_track_file'])
     import_job.status = 'finished'
     import_job.track_file = track_file
     if import_job.audio_file:

api/funkwhale_api/music/utils.py

@@ -60,3 +60,10 @@ def compute_status(jobs):
     if pending:
         return 'pending'
     return 'finished'
+
+
+def get_ext_from_type(mimetype):
+    mapping = {
+        'audio/ogg': 'ogg',
+        'audio/mpeg': 'mp3',
+    }

api/funkwhale_api/music/views.py

 import ffmpeg
 import os
 import json
+import requests
 import subprocess
 import unicodedata
 import urllib
 
-from django.urls import reverse
+from django.contrib.auth.decorators import login_required
+from django.core.exceptions import ObjectDoesNotExist
+from django.conf import settings
 from django.db import models, transaction
 from django.db.models.functions import Length
-from django.conf import settings
 from django.http import StreamingHttpResponse
+from django.urls import reverse
+from django.utils.decorators import method_decorator
 
 from rest_framework import viewsets, views, mixins
 from rest_framework.decorators import detail_route, list_route
 from rest_framework.response import Response
+from rest_framework import settings as rest_settings
 from rest_framework import permissions
 from musicbrainzngs import ResponseError
-from django.contrib.auth.decorators import login_required
-from django.utils.decorators import method_decorator
 
 from funkwhale_api.common import utils as funkwhale_utils
+from funkwhale_api.federation import actors
 from funkwhale_api.requests.models import ImportRequest
 from funkwhale_api.musicbrainz import api
 from funkwhale_api.common.permissions import (
     ConditionalAuthentication, HasModelPermission)
 from taggit.models import Tag
+from funkwhale_api.federation.authentication import SignatureAuthentication
 
+from . import filters
 from . import forms
+from . import importers
 from . import models
+from . import permissions as music_permissions
 from . import serializers
-from . import importers
-from . import filters
 from . import tasks
 from . import utils
 
@@ -45,6 +51,7 @@ class SearchMixin(object):
         serializer = self.serializer_class(queryset, many=True)
         return Response(serializer.data)
 
+
 class TagViewSetMixin(object):
 
     def get_queryset(self):
@@ -179,22 +186,54 @@ class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet):
 class TrackFileViewSet(viewsets.ReadOnlyModelViewSet):
     queryset = (models.TrackFile.objects.all().order_by('-id'))
     serializer_class = serializers.TrackFileSerializer
-    permission_classes = [ConditionalAuthentication]
+    authentication_classes = rest_settings.api_settings.DEFAULT_AUTHENTICATION_CLASSES + [
+        SignatureAuthentication
+    ]
+    permission_classes = [music_permissions.Listen]
 
     @detail_route(methods=['get'])
     def serve(self, request, *args, **kwargs):
         try:
-            f = models.TrackFile.objects.get(pk=kwargs['pk'])
+            f = models.TrackFile.objects.select_related(
+                'library_track',
+                'track__album__artist',
+                'track__artist',
+            ).get(pk=kwargs['pk'])
         except models.TrackFile.DoesNotExist:
             return Response(status=404)
 
-        response = Response()
+        mt = f.mimetype
+        try:
+            library_track = f.library_track
+        except ObjectDoesNotExist:
+            library_track = None
+        if library_track and not f.audio_file:
+            # we proxy the response to the remote library
+            # since we did not mirror the file locally
+            mt = library_track.audio_mimetype
+            file_extension = utils.get_ext_from_type(mt)
+            filename = '{}.{}'.format(f.track.full_name, file_extension)
+            auth = actors.SYSTEM_ACTORS['library'].get_request_auth()
+            remote_response = requests.get(
+                library_track.audio_url,
+                auth=auth,
+                stream=True,
+                headers={
+                    'Content-Type': 'application/activity+json'
+                })
+            response = StreamingHttpResponse(remote_response.iter_content())
+        else:
+            response = Response()
+            filename = f.filename
+            response['X-Accel-Redirect'] = "{}{}".format(
+                settings.PROTECT_FILES_PATH,
+                f.audio_file.url)
         filename = "filename*=UTF-8''{}".format(
-            urllib.parse.quote(f.filename))
+            urllib.parse.quote(filename))
         response["Content-Disposition"] = "attachment; {}".format(filename)
-        response['X-Accel-Redirect'] = "{}{}".format(
-            settings.PROTECT_FILES_PATH,
-            f.audio_file.url)
+        if mt:
+            response["Content-Type"] = mt
+
         return response
 
     @list_route(methods=['get'])

api/tests/conftest.py

@@ -162,3 +162,12 @@ def media_root(settings):
 def r_mock():
     with requests_mock.mock() as m:
         yield m
+
+
+@pytest.fixture
+def authenticated_actor(factories, mocker):
+    actor = factories['federation.Actor']()
+    mocker.patch(
+        'funkwhale_api.federation.authentication.SignatureAuthentication.authenticate_actor',
+        return_value=actor)
+    yield actor

api/tests/federation/conftest.py

-import pytest
-
-
-@pytest.fixture
-def authenticated_actor(nodb_factories, mocker):
-    actor = nodb_factories['federation.Actor']()
-    mocker.patch(
-        'funkwhale_api.federation.authentication.SignatureAuthentication.authenticate_actor',
-        return_value=actor)
-    yield actor

api/tests/federation/test_actors.py

@@ -26,14 +26,17 @@ def test_actor_fetching(r_mock):
     assert r == payload
 
 
-def test_get_library(settings, preferences):
-    preferences['federation__public_key'] = 'public_key'
+def test_get_library(db, settings, mocker):
+    get_key_pair = mocker.patch(
+        'funkwhale_api.federation.keys.get_key_pair',
+        return_value=(b'private', b'public'))
     expected = {
         'preferred_username': 'library',
         'domain': settings.FEDERATION_HOSTNAME,
         'type': 'Person',
         'name': '{}\'s library'.format(settings.FEDERATION_HOSTNAME),
         'manually_approves_followers': True,
+        'public_key': 'public',
         'url': utils.full_url(
             reverse(
                 'federation:instance-actors-detail',
@@ -50,7 +53,6 @@ def test_get_library(settings, preferences):
             reverse(
                 'federation:instance-actors-outbox',
                 kwargs={'actor': 'library'})),
-        'public_key': 'public_key',
         'summary': 'Bot account to federate with {}\'s library'.format(
         settings.FEDERATION_HOSTNAME),
     }
@@ -59,14 +61,17 @@ def test_get_library(settings, preferences):
         assert getattr(actor, key) == value
 
 
-def test_get_test(settings, preferences):
-    preferences['federation__public_key'] = 'public_key'
+def test_get_test(db, mocker, settings):
+    get_key_pair = mocker.patch(
+        'funkwhale_api.federation.keys.get_key_pair',
+        return_value=(b'private', b'public'))
     expected = {
         'preferred_username': 'test',
         'domain': settings.FEDERATION_HOSTNAME,
         'type': 'Person',
         'name': '{}\'s test account'.format(settings.FEDERATION_HOSTNAME),
         'manually_approves_followers': False,
+        'public_key': 'public',
         'url': utils.full_url(
             reverse(
                 'federation:instance-actors-detail',
@@ -83,7 +88,6 @@ def test_get_test(settings, preferences):
             reverse(
                 'federation:instance-actors-outbox',
                 kwargs={'actor': 'test'})),
-        'public_key': 'public_key',
         'summary': 'Bot account to test federation with {}. Send me /ping and I\'ll answer you.'.format(
         settings.FEDERATION_HOSTNAME),
     }

api/tests/federation/test_commands.py

-from django.core.management import call_command
-
-
-def test_generate_instance_key_pair(preferences, mocker):
-    mocker.patch(
-        'funkwhale_api.federation.keys.get_key_pair',
-        return_value=(b'private', b'public'))
-    assert preferences['federation__public_key'] == ''
-    assert preferences['federation__private_key'] == ''
-
-    call_command('generate_keys', interactive=False)
-
-    assert preferences['federation__private_key'] == 'private'
-    assert preferences['federation__public_key'] == 'public'

api/tests/music/test_permissions.py

+from rest_framework.views import APIView
+
+from funkwhale_api.federation import actors
+from funkwhale_api.music import permissions
+
+
+def test_list_permission_no_protect(anonymous_user, api_request, settings):
+    settings.PROTECT_AUDIO_FILES = False
+    view = APIView.as_view()
+    permission = permissions.Listen()
+    request = api_request.get('/')
+    assert permission.has_permission(request, view) is True
+
+
+def test_list_permission_protect_anonymous(
+        anonymous_user, api_request, settings):
+    settings.PROTECT_AUDIO_FILES = True
+    view = APIView.as_view()
+    permission = permissions.Listen()
+    request = api_request.get('/')
+    assert permission.has_permission(request, view) is False
+
+
+def test_list_permission_protect_authenticated(
+        factories, api_request, settings):
+    settings.PROTECT_AUDIO_FILES = True
+    user = factories['users.User']()
+    view = APIView.as_view()
+    permission = permissions.Listen()
+    request = api_request.get('/')
+    setattr(request, 'user', user)
+    assert permission.has_permission(request, view) is True
+
+
+def test_list_permission_protect_not_following_actor(
+        factories, api_request, settings):
+    settings.PROTECT_AUDIO_FILES = True
+    actor = factories['federation.Actor']()
+    view = APIView.as_view()
+    permission = permissions.Listen()
+    request = api_request.get('/')
+    setattr(request, 'actor', actor)
+    assert permission.has_permission(request, view) is False
+
+
+def test_list_permission_protect_following_actor(
+        factories, api_request, settings):
+    settings.PROTECT_AUDIO_FILES = True
+    library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
+    follow = factories['federation.Follow'](target=library_actor)
+    view = APIView.as_view()
+    permission = permissions.Listen()
+    request = api_request.get('/')
+    setattr(request, 'actor', follow.actor)
+
+    assert permission.has_permission(request, view) is True

api/tests/music/test_views.py

+import io
 import pytest
 
 from funkwhale_api.music import views
+from funkwhale_api.federation import actors