Merge branch 'remote-scan' into 'develop'

Remote scan/import

Closes #137 and #136

See merge request funkwhale/funkwhale!126

.env.dev

 API_AUTHENTICATION_REQUIRED=True
 RAVEN_ENABLED=false
 RAVEN_DSN=https://44332e9fdd3d42879c7d35bf8562c6a4:0062dc16a22b41679cd5765e5342f716@sentry.eliotberriot.com/5
-DJANGO_ALLOWED_HOSTS=localhost,nginx
+DJANGO_ALLOWED_HOSTS=.funkwhale.test,localhost,nginx,0.0.0.0,127.0.0.1
 DJANGO_SETTINGS_MODULE=config.settings.local
 DJANGO_SECRET_KEY=dev
 C_FORCE_ROOT=true
-FUNKWHALE_URL=http://localhost
+FUNKWHALE_HOSTNAME=localhost
+FUNKWHALE_PROTOCOL=http
 PYTHONDONTWRITEBYTECODE=true
+WEBPACK_DEVSERVER_PORT=8080

README.rst

@@ -206,3 +206,80 @@ Typical workflow for a merge request
 6. Push your branch
 7. Create your merge request
 8. Take a step back and enjoy, we're really grateful you did all of this and took the time to contribute!
+
+
+Working with federation locally
+-------------------------------
+
+To achieve that, you'll need:
+
+1. to update your dns resolver to resolve all your .dev hostnames locally
+2. a reverse proxy (such as traefik) to catch those .dev requests and
+   and with https certificate
+3. two instances (or more) running locally, following the regular dev setup
+
+Resolve .dev names locally
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you use dnsmasq, this is as simple as doing::
+
+    echo "address=/test/172.17.0.1" | sudo tee /etc/dnsmasq.d/test.conf
+    sudo systemctl restart dnsmasq
+
+If you use NetworkManager with dnsmasq integration, use this instead::
+
+    echo "address=/test/172.17.0.1" | sudo tee /etc/NetworkManager/dnsmasq.d/test.conf
+    sudo systemctl restart NetworkManager
+
+Add wildcard certificate to the trusted certificates
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Simply copy bundled certificates::
+
+    sudo cp docker/ssl/test.crt /usr/local/share/ca-certificates/
+    sudo update-ca-certificates
+
+This certificate is a wildcard for ``*.funkwhale.test``
+
+Run a reverse proxy for your instances
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+
+Crete docker network
+^^^^^^^^^^^^^^^^^^^^
+
+Create the federation network::
+
+    docker network create federation
+
+Launch everything
+^^^^^^^^^^^^^^^^^
+
+Launch the traefik proxy::
+
+    docker-compose -f docker/traefik.yml up -d
+
+Then, in separate terminals, you can setup as many different instances as you
+need::
+
+    export COMPOSE_PROJECT_NAME=node2
+    docker-compose -f dev.yml run --rm api python manage.py migrate
+    docker-compose -f dev.yml run --rm api python manage.py createsuperuser
+    docker-compose -f dev.yml up nginx api front
+
+Note that by default, if you don't export the COMPOSE_PROJECT_NAME,
+we will default to node1 as the name of your instance.
+
+Assuming your project name is ``node1``, your server will be reachable
+at ``https://node1.funkwhale.test/``. Not that you'll have to trust
+the SSL Certificate as it's self signed.
+
+When working on federation with traefik, ensure you have this in your ``env``::
+
+    # This will ensure we don't bind any port on the host, and thus enable
+    # multiple instances of funkwhale to be spawned concurrently.
+    WEBPACK_DEVSERVER_PORT_BINDING=
+    # This disable certificate verification
+    EXTERNAL_REQUESTS_VERIFY_SSL=false
+    # this ensure you don't have incorrect urls pointing to http resources
+    FUNKWHALE_PROTOCOL=https

api/config/api_urls.py

@@ -32,6 +32,10 @@ v1_patterns += [
         include(
             ('funkwhale_api.instance.urls', 'instance'),
             namespace='instance')),
+    url(r'^federation/',
+        include(
+            ('funkwhale_api.federation.api_urls', 'federation'),
+            namespace='federation')),
     url(r'^providers/',
         include(
             ('funkwhale_api.providers.urls', 'providers'),

api/config/settings/common.py

@@ -25,8 +25,26 @@ try:
 except FileNotFoundError:
     pass
 
-FUNKWHALE_URL = env('FUNKWHALE_URL')
-FUNKWHALE_HOSTNAME = urlsplit(FUNKWHALE_URL).netloc
+FUNKWHALE_HOSTNAME = None
+FUNKWHALE_HOSTNAME_SUFFIX = env('FUNKWHALE_HOSTNAME_SUFFIX', default=None)
+FUNKWHALE_HOSTNAME_PREFIX = env('FUNKWHALE_HOSTNAME_PREFIX', default=None)
+if FUNKWHALE_HOSTNAME_PREFIX and FUNKWHALE_HOSTNAME_SUFFIX:
+    # We're in traefik case, in development
+    FUNKWHALE_HOSTNAME = '{}.{}'.format(
+        FUNKWHALE_HOSTNAME_PREFIX, FUNKWHALE_HOSTNAME_SUFFIX)
+    FUNKWHALE_PROTOCOL = env('FUNKWHALE_PROTOCOL', default='https')
+else:
+    try:
+        FUNKWHALE_HOSTNAME = env('FUNKWHALE_HOSTNAME')
+        FUNKWHALE_PROTOCOL = env('FUNKWHALE_PROTOCOL', default='https')
+    except Exception:
+        FUNKWHALE_URL = env('FUNKWHALE_URL')
+        _parsed = urlsplit(FUNKWHALE_URL)
+        FUNKWHALE_HOSTNAME = _parsed.netloc
+        FUNKWHALE_PROTOCOL = _parsed.scheme
+
+FUNKWHALE_URL = '{}://{}'.format(FUNKWHALE_PROTOCOL, FUNKWHALE_HOSTNAME)
+
 
 FEDERATION_ENABLED = env.bool('FEDERATION_ENABLED', default=True)
 FEDERATION_HOSTNAME = env('FEDERATION_HOSTNAME', default=FUNKWHALE_HOSTNAME)
@@ -406,3 +424,8 @@ ACCOUNT_USERNAME_BLACKLIST = [
     'staff',
     'service',
 ] + env.list('ACCOUNT_USERNAME_BLACKLIST', default=[])
+
+EXTERNAL_REQUESTS_VERIFY_SSL = env.bool(
+    'EXTERNAL_REQUESTS_VERIFY_SSL',
+    default=True
+)

api/funkwhale_api/common/fields.py

+import django_filters
+
 from django.db import models
 
+from funkwhale_api.music import utils
+
 
 PRIVACY_LEVEL_CHOICES = [
     ('me', 'Only me'),
@@ -25,3 +29,15 @@ def privacy_level_query(user, lookup_field='privacy_level'):
             'followers', 'instance', 'everyone'
         ]
     })
+
+
+class SearchFilter(django_filters.CharFilter):
+    def __init__(self, *args, **kwargs):
+        self.search_fields = kwargs.pop('search_fields')
+        super().__init__(*args, **kwargs)
+
+    def filter(self, qs, value):
+        if not value:
+            return qs
+        query = utils.get_query(value, self.search_fields)
+        return qs.filter(query)

api/funkwhale_api/federation/activity.py

-import logging
-import json
-import requests_http_signature
-import uuid
-
-from funkwhale_api.common import session
-
-from . import models
-from . import signing
-
-logger = logging.getLogger(__name__)
+from . import serializers
+from . import tasks
 
 ACTIVITY_TYPES = [
     'Accept',
@@ -61,86 +52,16 @@ OBJECT_TYPES = [
 
 
 def deliver(activity, on_behalf_of, to=[]):
-    from . import actors
-    logger.info('Preparing activity delivery to %s', to)
-    auth = signing.get_auth(
-        on_behalf_of.private_key, on_behalf_of.private_key_id)
-    for url in to:
-        recipient_actor = actors.get_actor(url)
-        logger.debug('delivering to %s', recipient_actor.inbox_url)
-        logger.debug('activity content: %s', json.dumps(activity))
-        response = session.get_session().post(
-            auth=auth,
-            json=activity,
-            url=recipient_actor.inbox_url,
-            headers={
-                'Content-Type': 'application/activity+json'
-            }
-        )
-        response.raise_for_status()
-        logger.debug('Remote answered with %s', response.status_code)
-
-
-def get_follow(follow_id, follower, followed):
-    return {
-        '@context': [
-            'https://www.w3.org/ns/activitystreams',
-            'https://w3id.org/security/v1',
-            {}
-        ],
-        'actor': follower.url,
-        'id': follower.url + '#follows/{}'.format(follow_id),
-        'object': followed.url,
-        'type': 'Follow'
-    }
-
-
-def get_undo(id, actor, object):
-    return {
-        '@context': [
-            'https://www.w3.org/ns/activitystreams',
-            'https://w3id.org/security/v1',
-            {}
-        ],
-        'type': 'Undo',
-        'id': id + '/undo',
-        'actor': actor.url,
-        'object': object,
-    }
-
-
-def get_accept_follow(accept_id, accept_actor, follow, follow_actor):
-    return {
-        "@context": [
-            "https://www.w3.org/ns/activitystreams",
-            "https://w3id.org/security/v1",
-            {}
-        ],
-        "id": accept_actor.url + '#accepts/follows/{}'.format(
-            accept_id),
-        "type": "Accept",
-        "actor": accept_actor.url,
-        "object": {
-            "id": follow['id'],
-            "type": "Follow",
-            "actor": follow_actor.url,
-            "object": accept_actor.url
-        },
-    }
+    return tasks.send.delay(
+        activity=activity,
+        actor_id=on_behalf_of.pk,
+        to=to
+    )
 
 
-def accept_follow(target, follow, actor):
-    accept_uuid = uuid.uuid4()
-    accept = get_accept_follow(
-        accept_id=accept_uuid,
-        accept_actor=target,
-        follow=follow,
-        follow_actor=actor)
-    deliver(
-        accept,
-        to=[actor.url],
-        on_behalf_of=target)
-    return models.Follow.objects.get_or_create(
-        actor=actor,
-        target=target,
-    )
+def accept_follow(follow):
+    serializer = serializers.AcceptFollowSerializer(follow)
+    return deliver(
+        serializer.data,
+        to=[follow.actor.url],
+        on_behalf_of=follow.target)

api/funkwhale_api/federation/actors.py

@@ -31,6 +31,8 @@ def remove_tags(text):
 def get_actor_data(actor_url):
     response = session.get_session().get(
         actor_url,
+        timeout=5,
+        verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
         headers={
             'Accept': 'application/activity+json',
         }
@@ -42,6 +44,7 @@ def get_actor_data(actor_url):
         raise ValueError(
             'Invalid actor payload: {}'.format(response.text))
 
+
 def get_actor(actor_url):
     data = get_actor_data(actor_url)
     serializer = serializers.ActorSerializer(data=data)
@@ -150,24 +153,32 @@ class SystemActor(object):
 
     def handle_follow(self, ac, sender):
         system_actor = self.get_actor_instance()
-        if self.manually_approves_followers:
-            fr, created = models.FollowRequest.objects.get_or_create(
-                actor=sender,
-                target=system_actor,
-                approved=None,
-            )
-            return fr
+        serializer = serializers.FollowSerializer(
+            data=ac, context={'follow_actor': sender})
+        if not serializer.is_valid():
+            return logger.info('Invalid follow payload')
+        approved = True if not self.manually_approves_followers else None
+        follow = serializer.save(approved=approved)
+        if follow.approved:
+            return activity.accept_follow(follow)
+
+    def handle_accept(self, ac, sender):
+        system_actor = self.get_actor_instance()
+        serializer = serializers.AcceptFollowSerializer(
+            data=ac,
+            context={'follow_target': sender, 'follow_actor': system_actor})
+        if not serializer.is_valid(raise_exception=True):
+            return logger.info('Received invalid payload')
 
-        return activity.accept_follow(
-            system_actor, ac, sender
-        )
+        return serializer.save()
 
     def handle_undo_follow(self, ac, sender):
-        actor = self.get_actor_instance()
-        models.Follow.objects.filter(
-            actor=sender,
-            target=actor,
-        ).delete()
+        system_actor = self.get_actor_instance()
+        serializer = serializers.UndoFollowSerializer(
+            data=ac, context={'actor': sender, 'target': system_actor})
+        if not serializer.is_valid():
+            return logger.info('Received invalid payload')
+        serializer.save()
 
     def handle_undo(self, ac, sender):
         if ac['object']['type'] != 'Follow':
@@ -343,15 +354,15 @@ class TestActor(SystemActor):
         super().handle_follow(ac, sender)
         # also, we follow back
         test_actor = self.get_actor_instance()
-        follow_uuid = uuid.uuid4()
-        follow = activity.get_follow(
-            follow_id=follow_uuid,
-            follower=test_actor,
-            followed=sender)
+        follow_back = models.Follow.objects.get_or_create(
+            actor=test_actor,
+            target=sender,
+            approved=None,
+        )[0]
         activity.deliver(
-            follow,
-            to=[ac['actor']],
-            on_behalf_of=test_actor)
+            serializers.FollowSerializer(follow_back).data,
+            to=[follow_back.target.url],
+            on_behalf_of=follow_back.actor)
 
     def handle_undo_follow(self, ac, sender):
         super().handle_undo_follow(ac, sender)
@@ -364,11 +375,7 @@ class TestActor(SystemActor):
             )
         except models.Follow.DoesNotExist:
             return
-        undo = activity.get_undo(
-            id=follow.get_federation_url(),
-            actor=actor,
-            object=serializers.FollowSerializer(follow).data,
-        )
+        undo = serializers.UndoFollowSerializer(follow).data
         follow.delete()
         activity.deliver(
             undo,

api/funkwhale_api/federation/admin.py

+from django.contrib import admin
+
+from . import models
+
+
+@admin.register(models.Actor)
+class ActorAdmin(admin.ModelAdmin):
+    list_display = [
+        'url',
+        'domain',
+        'preferred_username',
+        'type',
+        'creation_date',
+        'last_fetch_date']
+    search_fields = ['url', 'domain', 'preferred_username']
+    list_filter = [
+        'type'
+    ]
+
+
+@admin.register(models.Follow)
+class FollowAdmin(admin.ModelAdmin):
+    list_display = [
+        'actor',
+        'target',
+        'approved',
+        'creation_date'
+    ]
+    list_filter = [
+        'approved'
+    ]
+    search_fields = ['actor__url', 'target__url']
+    list_select_related = True
+
+
+@admin.register(models.Library)
+class LibraryAdmin(admin.ModelAdmin):
+    list_display = [
+        'actor',
+        'url',
+        'creation_date',
+        'fetched_date',
+        'tracks_count']
+    search_fields = ['actor__url', 'url']
+    list_filter = [
+        'federation_enabled',
+        'download_files',
+        'autoimport',
+    ]
+    list_select_related = True
+
+
+@admin.register(models.LibraryTrack)
+class LibraryTrackAdmin(admin.ModelAdmin):
+    list_display = [
+        'title',
+        'artist_name',
+        'album_title',
+        'url',
+        'library',
+        'creation_date',
+        'published_date',
+    ]
+    search_fields = [
+        'library__url', 'url', 'artist_name', 'title', 'album_title']
+    list_select_related = True

api/funkwhale_api/federation/api_urls.py

+from rest_framework import routers
+
+from . import views
+
+router = routers.SimpleRouter()
+router.register(
+    r'libraries',
+    views.LibraryViewSet,
+    'libraries')
+router.register(
+    r'library-tracks',
+    views.LibraryTrackViewSet,
+    'library-tracks')
+
+urlpatterns = router.urls

api/funkwhale_api/federation/dynamic_preferences_registry.py

@@ -4,3 +4,17 @@ from dynamic_preferences import types
 from dynamic_preferences.registries import global_preferences_registry
 
 federation = types.Section('federation')
+
+
+@global_preferences_registry.register
+class MusicCacheDuration(types.IntPreference):
+    show_in_api = True
+    section = federation
+    name = 'music_cache_duration'
+    default = 60 * 24 * 2
+    verbose_name = 'Music cache duration'
+    help_text = (
+        'How much minutes do you want to keep a copy of federated tracks'
+        'locally? Federated files that were not listened in this interval '
+        'will be erased and refetched from the remote on the next listening.'
+    )

api/funkwhale_api/federation/factories.py

@@ -113,15 +113,6 @@ class FollowFactory(factory.DjangoModelFactory):
         )
 
 
-@registry.register
-class FollowRequestFactory(factory.DjangoModelFactory):
-    target = factory.SubFactory(ActorFactory)
-    actor = factory.SubFactory(ActorFactory)
-
-    class Meta:
-        model = models.FollowRequest
-
-
 @registry.register
 class LibraryFactory(factory.DjangoModelFactory):
     actor = factory.SubFactory(ActorFactory)
@@ -194,6 +185,11 @@ class LibraryTrackFactory(factory.DjangoModelFactory):
     class Meta:
         model = models.LibraryTrack
 
+    class Params:
+        with_audio_file = factory.Trait(
+            audio_file=factory.django.FileField()
+        )
+
 
 @registry.register(name='federation.Note')
 class NoteFactory(factory.Factory):

api/funkwhale_api/federation/filters.py

+import django_filters
+
+from funkwhale_api.common import fields
+
+from . import models
+
+
+class LibraryFilter(django_filters.FilterSet):
+    approved = django_filters.BooleanFilter('following__approved')
+    q = fields.SearchFilter(search_fields=[
+        'actor__domain',
+    ])
+
+    class Meta:
+        model = models.Library
+        fields = {
+            'approved': ['exact'],
+            'federation_enabled': ['exact'],
+            'download_files': ['exact'],
+            'autoimport': ['exact'],
+            'tracks_count': ['exact'],
+        }
+
+
+class LibraryTrackFilter(django_filters.FilterSet):
+    library = django_filters.CharFilter('library__uuid')
+    q = fields.SearchFilter(search_fields=[
+        'artist_name',
+        'title',
+        'album_title',
+        'library__actor__domain',
+    ])
+
+    class Meta:
+        model = models.LibraryTrack
+        fields = {
+            'library': ['exact'],
+            'artist_name': ['exact', 'icontains'],
+            'title': ['exact', 'icontains'],
+            'album_title': ['exact', 'icontains'],
+            'audio_mimetype': ['exact', 'icontains'],
+        }
+
+
+class FollowFilter(django_filters.FilterSet):
+    pending = django_filters.CharFilter(method='filter_pending')
+    ordering = django_filters.OrderingFilter(
+        # tuple-mapping retains order
+        fields=(
+            ('creation_date', 'creation_date'),
+            ('modification_date', 'modification_date'),
+        ),
+    )
+    q = fields.SearchFilter(search_fields=[
+        'actor__domain',
+        'actor__preferred_username',
+    ])
+
+    class Meta:
+        model = models.Follow
+        fields = ['approved', 'pending', 'q']
+
+    def filter_pending(self, queryset, field_name, value):
+        if value.lower() in ['true', '1', 'yes']:
+            queryset = queryset.filter(approved__isnull=True)
+        return queryset

api/funkwhale_api/federation/library.py

+import json
+import requests
+
+from django.conf import settings
+
+from funkwhale_api.common import session
+
+from . import actors
+from . import models
+from . import serializers
+from . import signing
+from . import webfinger
+
+
+def scan_from_account_name(account_name):
+    """
+    Given an account name such as library@test.library, will:
+
+    1. Perform the webfinger lookup
+    2. Perform the actor lookup
+    3. Perform the library's collection lookup
+
+    and return corresponding data in a dictionary.
+    """
+    data = {}
+    try:
+        username, domain = webfinger.clean_acct(
+            account_name, ensure_local=False)
+    except serializers.ValidationError:
+        return {
+            'webfinger': {
+                'errors': ['Invalid account string']
+            }
+        }
+    system_library = actors.SYSTEM_ACTORS['library'].get_actor_instance()
+    library = models.Library.objects.filter(
+        actor__domain=domain,
+        actor__preferred_username=username
+    ).select_related('actor').first()
+    data['local'] = {
+        'following': False,
+        'awaiting_approval': False,
+    }
+    try:
+        follow = models.Follow.objects.get(
+            target__preferred_username=username,
+            target__domain=username,
+            actor=system_library,
+        )
+        data['local']['awaiting_approval'] = not bool(follow.approved)
+        data['local']['following'] = True
+    except models.Follow.DoesNotExist:
+        pass
+
+    try:
+        data['webfinger'] = webfinger.get_resource(
+            'acct:{}'.format(account_name))
+    except requests.ConnectionError:
+        return {
+            'webfinger': {
+                'errors': ['This webfinger resource is not reachable']
+            }
+        }
+    except requests.HTTPError as e:
+        return {
+            'webfinger': {
+                'errors': [
+                    'Error {} during webfinger request'.format(
+                        e.response.status_code)]
+            }
+        }
+    except json.JSONDecodeError as e:
+        return {
+            'webfinger': {
+                'errors': ['Could not process webfinger response']
+            }
+        }
+
+    try:
+        data['actor'] = actors.get_actor_data(data['webfinger']['actor_url'])
+    except requests.ConnectionError:
+        data['actor'] = {
+            'errors': ['This actor is not reachable']
+        }
+        return data
+    except requests.HTTPError as e:
+        data['actor'] = {
+            'errors': [
+                'Error {} during actor request'.format(
+                    e.response.status_code)]
+        }
+        return data
+
+    serializer = serializers.LibraryActorSerializer(data=data['actor'])
+    if not serializer.is_valid():
+        data['actor'] = {
+            'errors': ['Invalid ActivityPub actor']
+        }
+        return data
+    data['library'] = get_library_data(
+        serializer.validated_data['library_url'])
+
+    return data
+
+
+def get_library_data(library_url):
+    actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
+    auth = signing.get_auth(actor.private_key, actor.private_key_id)
+    try:
+        response = session.get_session().get(
+            library_url,
+            auth=auth,
+            timeout=5,
+            verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
+            headers={
+                'Content-Type': 'application/activity+json'
+            }
+        )
+    except requests.ConnectionError:
+        return {
+            'errors': ['This library is not reachable']
+        }
+    scode = response.status_code
+    if scode == 401:
+        return {
+            'errors': ['This library requires authentication']
+        }
+    elif scode == 403:
+        return {
+            'errors': ['Permission denied while scanning library']
+        }
+    elif scode >= 400:
+        return {
+            'errors': ['Error {} while fetching the library'.format(scode)]
+        }
+    serializer = serializers.PaginatedCollectionSerializer(
+        data=response.json(),
+    )
+    if not serializer.is_valid():
+        return {
+            'errors': [
+                'Invalid ActivityPub response from remote library']
+        }
+
+    return serializer.validated_data
+
+
+def get_library_page(library, page_url):
+    actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
+    auth = signing.get_auth(actor.private_key, actor.private_key_id)
+    response = session.get_session().get(
+        page_url,
+        auth=auth,
+        timeout=5,
+        verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
+        headers={
+            'Content-Type': 'application/activity+json'
+        }
+    )
+    serializer = serializers.CollectionPageSerializer(
+        data=response.json(),
+        context={
+            'library': library,
+            'item_serializer': serializers.AudioSerializer})
+    serializer.is_valid(raise_exception=True)
+    return serializer.validated_data

api/funkwhale_api/federation/migrations/0004_auto_20180410_2025.py

+# Generated by Django 2.0.3 on 2018-04-10 20:25
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('federation', '0003_auto_20180407_1010'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='followrequest',
+            name='actor',
+        ),
+        migrations.RemoveField(
+            model_name='followrequest',
+            name='target',
+        ),
+        migrations.AddField(
+            model_name='follow',
+            name='approved',
+            field=models.NullBooleanField(default=None),
+        ),
+        migrations.AddField(
+            model_name='library',
+            name='follow',
+            field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='library', to='federation.Follow'),
+        ),
+        migrations.DeleteModel(
+            name='FollowRequest',
+        ),
+    ]

api/funkwhale_api/federation/migrations/0005_auto_20180413_1723.py

+# Generated by Django 2.0.3 on 2018-04-13 17:23
+
+import django.contrib.postgres.fields.jsonb
+import django.core.serializers.json
+from django.db import migrations, models
+import funkwhale_api.federation.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('federation', '0004_auto_20180410_2025'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='librarytrack',
+            name='audio_file',
+            field=models.FileField(blank=True, null=True, upload_to=funkwhale_api.federation.models.get_file_path),
+        ),
+        migrations.AlterField(
+            model_name='librarytrack',
+            name='metadata',
+            field=django.contrib.postgres.fields.jsonb.JSONField(default={}, encoder=django.core.serializers.json.DjangoJSONEncoder, max_length=10000),
+        ),
+    ]

api/funkwhale_api/federation/models.py

+import os
 import uuid
+import tempfile
 
 from django.conf import settings
 from django.contrib.postgres.fields import JSONField
+from django.core.serializers.json import DjangoJSONEncoder
 from django.db import models
 from django.utils import timezone
 
+from funkwhale_api.common import session
+from funkwhale_api.music import utils as music_utils
+
 TYPE_CHOICES = [
     ('Person', 'Person'),
     ('Application', 'Application'),
@@ -109,6 +115,7 @@ class Follow(models.Model):
     creation_date = models.DateTimeField(default=timezone.now)
     modification_date = models.DateTimeField(
         auto_now=True)
+    approved = models.NullBooleanField(default=None)
 
     class Meta:
         unique_together = ['actor', 'target']
@@ -117,49 +124,6 @@ class Follow(models.Model):
         return '{}#follows/{}'.format(self.actor.url, self.uuid)
 
 
-class FollowRequest(models.Model):
-    uuid = models.UUIDField(default=uuid.uuid4, unique=True)
-    actor = models.ForeignKey(
-        Actor,
-        related_name='emmited_follow_requests',
-        on_delete=models.CASCADE,
-    )
-    target = models.ForeignKey(
-        Actor,
-        related_name='received_follow_requests',
-        on_delete=models.CASCADE,
-    )
-    creation_date = models.DateTimeField(default=timezone.now)
-    modification_date = models.DateTimeField(
-        auto_now=True)
-    approved = models.NullBooleanField(default=None)
-
-    def approve(self):
-        from . import activity
-        from . import serializers
-        self.approved = True
-        self.save(update_fields=['approved'])
-        Follow.objects.get_or_create(
-            target=self.target,
-            actor=self.actor
-        )
-        if self.target.is_local:
-            follow = {
-                '@context': serializers.AP_CONTEXT,
-                'actor': self.actor.url,
-                'id': self.actor.url + '#follows/{}'.format(uuid.uuid4()),
-                'object': self.target.url,
-                'type': 'Follow'
-            }
-            activity.accept_follow(
-                self.target, follow, self.actor
-            )
-
-    def refuse(self):
-        self.approved = False
-        self.save(update_fields=['approved'])
-
-
 class Library(models.Model):
     creation_date = models.DateTimeField(default=timezone.now)
     modification_date = models.DateTimeField(
@@ -179,12 +143,32 @@ class Library(models.Model):
     # should we automatically import new files from this library?
     autoimport = models.BooleanField()
     tracks_count = models.PositiveIntegerField(null=True, blank=True)
+    follow = models.OneToOneField(
+        Follow,
+        related_name='library',
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL,
+    )
+
+
+def get_file_path(instance, filename):
+    uid = str(uuid.uuid4())
+    chunk_size = 2
+    chunks = [uid[i:i+chunk_size] for i in range(0, len(uid), chunk_size)]
+    parts = chunks[:3] + [filename]
+    return os.path.join('federation_cache', *parts)
 
 
 class LibraryTrack(models.Model):
     url = models.URLField(unique=True)
     audio_url = models.URLField()
     audio_mimetype = models.CharField(max_length=200)
+    audio_file = models.FileField(
+        upload_to=get_file_path,
+        null=True,
+        blank=True)
+
     creation_date = models.DateTimeField(default=timezone.now)
     modification_date = models.DateTimeField(
         auto_now=True)
@@ -195,4 +179,35 @@ class LibraryTrack(models.Model):
     artist_name = models.CharField(max_length=500)
     album_title = models.CharField(max_length=500)
     title = models.CharField(max_length=500)
-    metadata = JSONField(default={}, max_length=10000)
+    metadata = JSONField(
+        default={}, max_length=10000, encoder=DjangoJSONEncoder)
+
+    @property
+    def mbid(self):
+        try:
+            return self.metadata['recording']['musicbrainz_id']
+        except KeyError:
+            pass
+
+    def download_audio(self):
+        from . import actors
+        auth = actors.SYSTEM_ACTORS['library'].get_request_auth()
+        remote_response = session.get_session().get(
+            self.audio_url,
+            auth=auth,
+            stream=True,
+            timeout=20,
+            verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
+            headers={
+                'Content-Type': 'application/activity+json'
+            }
+        )
+        with remote_response as r:
+            remote_response.raise_for_status()
+            extension = music_utils.get_ext_from_type(self.audio_mimetype)
+            title = ' - '.join([self.title, self.album_title, self.artist_name])
+            filename = '{}.{}'.format(title, extension)
+            tmp_file = tempfile.TemporaryFile()
+            for chunk in r.iter_content(chunk_size=512):
+                tmp_file.write(chunk)
+            self.audio_file.save(filename, tmp_file)

api/funkwhale_api/federation/tasks.py

+import datetime
+import json
+import logging
+
+from django.conf import settings
+from django.utils import timezone
+
+from requests.exceptions import RequestException
+from dynamic_preferences.registries import global_preferences_registry
+
+from funkwhale_api.common import session
+from funkwhale_api.history.models import Listening
+from funkwhale_api.taskapp import celery
+
+from . import actors
+from . import library as lb
+from . import models
+from . import signing
+
+
+logger = logging.getLogger(__name__)
+
+
+@celery.app.task(
+    name='federation.send',
+    autoretry_for=[RequestException],
+    retry_backoff=30,
+    max_retries=5)
+@celery.require_instance(models.Actor, 'actor')
+def send(activity, actor, to):
+    logger.info('Preparing activity delivery to %s', to)
+    auth = signing.get_auth(
+        actor.private_key, actor.private_key_id)
+    for url in to:
+        recipient_actor = actors.get_actor(url)
+        logger.debug('delivering to %s', recipient_actor.inbox_url)
+        logger.debug('activity content: %s', json.dumps(activity))
+        response = session.get_session().post(
+            auth=auth,
+            json=activity,
+            url=recipient_actor.inbox_url,
+            timeout=5,
+            verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
+            headers={
+                'Content-Type': 'application/activity+json'
+            }
+        )
+        response.raise_for_status()
+        logger.debug('Remote answered with %s', response.status_code)
+
+
+@celery.app.task(
+    name='federation.scan_library',
+    autoretry_for=[RequestException],
+    retry_backoff=30,
+    max_retries=5)
+@celery.require_instance(models.Library, 'library')
+def scan_library(library, until=None):
+    if not library.federation_enabled:
+        return
+
+    data = lb.get_library_data(library.url)
+    scan_library_page.delay(
+        library_id=library.id, page_url=data['first'], until=until)
+    library.fetched_date = timezone.now()
+    library.tracks_count = data['totalItems']
+    library.save(update_fields=['fetched_date', 'tracks_count'])
+
+
+@celery.app.task(
+    name='federation.scan_library_page',
+    autoretry_for=[RequestException],
+    retry_backoff=30,
+    max_retries=5)
+@celery.require_instance(models.Library, 'library')
+def scan_library_page(library, page_url, until=None):
+    if not library.federation_enabled:
+        return
+
+    data = lb.get_library_page(library, page_url)
+    lts = []
+    for item_serializer in data['items']:
+        item_date = item_serializer.validated_data['published']
+        if until and item_date < until:
+            return
+        lts.append(item_serializer.save())
+
+    next_page = data.get('next')
+    if next_page and next_page != page_url:
+        scan_library_page.delay(library_id=library.id, page_url=next_page)
+
+
+@celery.app.task(name='federation.clean_music_cache')
+def clean_music_cache():
+    preferences = global_preferences_registry.manager()
+    delay = preferences['federation__music_cache_duration']
+    if delay < 1:
+        return  # cache clearing disabled
+
+    candidates = models.LibraryTrack.objects.filter(
+        audio_file__isnull=False
+    ).values_list('local_track_file__track', flat=True)
+    listenings = Listening.objects.filter(
+        creation_date__gte=timezone.now() - datetime.timedelta(minutes=delay),
+        track__pk__in=candidates).values_list('track', flat=True)
+    too_old = set(candidates) - set(listenings)
+
+    to_remove = models.LibraryTrack.objects.filter(
+        local_track_file__track__pk__in=too_old).only('audio_file')
+    for lt in to_remove:
+        lt.audio_file.delete()

api/funkwhale_api/federation/views.py

 from django import forms
 from django.conf import settings
 from django.core import paginator
+from django.db import transaction
 from django.http import HttpResponse
 from django.urls import reverse
 
-from rest_framework import viewsets
-from rest_framework import views
+from rest_framework import mixins
+from rest_framework import permissions as rest_permissions
 from rest_framework import response
+from rest_framework import views
+from rest_framework import viewsets
 from rest_framework.decorators import list_route, detail_route
+from rest_framework.serializers import ValidationError
 
+from funkwhale_api.common import utils as funkwhale_utils
+from funkwhale_api.common.permissions import HasModelPermission
 from funkwhale_api.music.models import TrackFile
 
+from . import activity
 from . import actors
 from . import authentication
+from . import filters
+from . import library
+from . import models
 from . import permissions
 from . import renderers
 from . import serializers
+from . import tasks
 from . import utils
 from . import webfinger
 
@@ -58,7 +69,7 @@ class InstanceActorViewSet(FederationMixin, viewsets.GenericViewSet):
             data = handler(request.data, actor=request.actor)
         except NotImplementedError:
             return response.Response(status=405)
-        return response.Response(data, status=200)
+        return response.Response({}, status=200)
 
     @detail_route(methods=['get', 'post'])
     def outbox(self, request, *args, **kwargs):
@@ -70,7 +81,7 @@ class InstanceActorViewSet(FederationMixin, viewsets.GenericViewSet):
             data = handler(request.data, actor=request.actor)
         except NotImplementedError:
             return response.Response(status=405)
-        return response.Response(data, status=200)
+        return response.Response({}, status=200)
 
 
 class WellKnownViewSet(FederationMixin, viewsets.GenericViewSet):
@@ -121,7 +132,7 @@ class MusicFilesViewSet(FederationMixin, viewsets.GenericViewSet):
         qs = TrackFile.objects.order_by('-creation_date').select_related(
             'track__artist',
             'track__album__artist'
-        )
+        ).filter(library_track__isnull=True)
         if page is None:
             conf = {
                 'id': utils.full_url(reverse('federation:music:files-list')),
@@ -154,3 +165,127 @@ class MusicFilesViewSet(FederationMixin, viewsets.GenericViewSet):
                 return response.Response(status=404)
 
         return response.Response(data)
+
+
+class LibraryPermission(HasModelPermission):
+    model = models.Library
+
+
+class LibraryViewSet(
+        mixins.RetrieveModelMixin,
+        mixins.UpdateModelMixin,
+        mixins.ListModelMixin,
+        viewsets.GenericViewSet):
+    permission_classes = [LibraryPermission]
+    queryset = models.Library.objects.all().select_related(
+        'actor',
+        'follow',
+    )
+    lookup_field = 'uuid'
+    filter_class = filters.LibraryFilter
+    serializer_class = serializers.APILibrarySerializer
+    ordering_fields = (
+        'id',
+        'creation_date',
+        'fetched_date',
+        'actor__domain',
+        'tracks_count',
+    )
+
+    @list_route(methods=['get'])
+    def fetch(self, request, *args, **kwargs):
+        account = request.GET.get('account')
+        if not account:
+            return response.Response(
+                {'account': 'This field is mandatory'}, status=400)
+
+        data = library.scan_from_account_name(account)
+        return response.Response(data)
+
+    @detail_route(methods=['post'])
+    def scan(self, request, *args, **kwargs):
+        library = self.get_object()
+        serializer = serializers.APILibraryScanSerializer(
+            data=request.data
+        )
+        serializer.is_valid(raise_exception=True)
+        result = tasks.scan_library.delay(
+            library_id=library.pk,
+            until=serializer.validated_data.get('until')
+        )
+        return response.Response({'task': result.id})
+
+    @list_route(methods=['get'])
+    def following(self, request, *args, **kwargs):
+        library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
+        queryset = models.Follow.objects.filter(
+            actor=library_actor
+        ).select_related(
+            'actor',
+            'target',
+        ).order_by('-creation_date')
+        filterset = filters.FollowFilter(request.GET, queryset=queryset)
+        final_qs = filterset.qs
+        serializer = serializers.APIFollowSerializer(final_qs, many=True)
+        data = {
+            'results': serializer.data,
+            'count': len(final_qs),
+        }
+        return response.Response(data)
+
+    @list_route(methods=['get', 'patch'])
+    def followers(self, request, *args, **kwargs):
+        if request.method.lower() == 'patch':
+            serializer = serializers.APILibraryFollowUpdateSerializer(
+                data=request.data)
+            serializer.is_valid(raise_exception=True)
+            follow = serializer.save()
+            return response.Response(
+                serializers.APIFollowSerializer(follow).data
+            )
+
+        library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
+        queryset = models.Follow.objects.filter(
+            target=library_actor
+        ).select_related(
+            'actor',
+            'target',
+        ).order_by('-creation_date')
+        filterset = filters.FollowFilter(request.GET, queryset=queryset)
+        final_qs = filterset.qs
+        serializer = serializers.APIFollowSerializer(final_qs, many=True)
+        data = {
+            'results': serializer.data,
+            'count': len(final_qs),
+        }
+        return response.Response(data)
+
+    @transaction.atomic
+    def create(self, request, *args, **kwargs):
+        serializer = serializers.APILibraryCreateSerializer(data=request.data)
+        serializer.is_valid(raise_exception=True)
+        library = serializer.save()
+        return response.Response(serializer.data, status=201)
+
+
+class LibraryTrackViewSet(
+        mixins.ListModelMixin,
+        viewsets.GenericViewSet):
+    permission_classes = [LibraryPermission]
+    queryset = models.LibraryTrack.objects.all().select_related(
+        'library__actor',
+        'library__follow',
+        'local_track_file',
+    )
+    filter_class = filters.LibraryTrackFilter
+    serializer_class = serializers.APILibraryTrackSerializer
+    ordering_fields = (
+        'id',
+        'artist_name',
+        'title',
+        'album_title',
+        'creation_date',
+        'modification_date',
+        'fetched_date',
+        'published_date',
+    )

api/funkwhale_api/federation/webfinger.py

@@ -2,8 +2,11 @@ from django import forms
 from django.conf import settings
 from django.urls import reverse
 
+from funkwhale_api.common import session
+
 from . import actors
 from . import utils
+from . import serializers
 
 VALID_RESOURCE_TYPES = ['acct']
 
@@ -23,17 +26,32 @@ def clean_resource(resource_string):
     return resource_type, resource
 
 
-def clean_acct(acct_string):
+def clean_acct(acct_string, ensure_local=True):
     try:
         username, hostname = acct_string.split('@')
     except ValueError:
         raise forms.ValidationError('Invalid format')
 
-    if hostname.lower() != settings.FEDERATION_HOSTNAME:
+    if ensure_local and hostname.lower() != settings.FEDERATION_HOSTNAME:
         raise forms.ValidationError(
             'Invalid hostname {}'.format(hostname))
 
-    if username not in actors.SYSTEM_ACTORS:
+    if ensure_local and username not in actors.SYSTEM_ACTORS:
         raise forms.ValidationError('Invalid username')
 
     return username, hostname
+
+
+def get_resource(resource_string):
+    resource_type, resource = clean_resource(resource_string)
+    username, hostname = clean_acct(resource, ensure_local=False)
+    url = 'https://{}/.well-known/webfinger?resource={}'.format(
+        hostname, resource_string)
+    response = session.get_session().get(
+        url,
+        verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
+        timeout=5)
+    response.raise_for_status()
+    serializer = serializers.ActorWebfingerSerializer(data=response.json())
+    serializer.is_valid(raise_exception=True)
+    return serializer.validated_data