Resolve "Per-user libraries" (use !368 instead)

api/config/api_urls.py

@@ -14,12 +14,11 @@ router.register(r"settings", GlobalPreferencesViewSet, base_name="settings")
 router.register(r"activity", activity_views.ActivityViewSet, "activity")
 router.register(r"tags", views.TagViewSet, "tags")
 router.register(r"tracks", views.TrackViewSet, "tracks")
-router.register(r"trackfiles", views.TrackFileViewSet, "trackfiles")
+router.register(r"track-files", views.TrackFileViewSet, "trackfiles")
+router.register(r"libraries", views.LibraryViewSet, "libraries")
+router.register(r"listen", views.ListenViewSet, "listen")
 router.register(r"artists", views.ArtistViewSet, "artists")
 router.register(r"albums", views.AlbumViewSet, "albums")
-router.register(r"import-batches", views.ImportBatchViewSet, "import-batches")
-router.register(r"import-jobs", views.ImportJobViewSet, "import-jobs")
-router.register(r"submit", views.SubmitViewSet, "submit")
 router.register(r"playlists", playlists_views.PlaylistViewSet, "playlists")
 router.register(
     r"playlist-tracks", playlists_views.PlaylistTrackViewSet, "playlist-tracks"

api/config/routing.py

@@ -8,9 +8,7 @@ application = ProtocolTypeRouter(
     {
         # Empty for now (http->django views is added by default)
         "websocket": TokenAuthMiddleware(
-            URLRouter(
-                [url("^api/v1/instance/activity$", consumers.InstanceActivityConsumer)]
-            )
+            URLRouter([url("^api/v1/activity$", consumers.InstanceActivityConsumer)])
         )
     }
 )

api/config/settings/common.py

@@ -126,7 +126,6 @@ LOCAL_APPS = (
     "funkwhale_api.history",
     "funkwhale_api.playlists",
     "funkwhale_api.providers.audiofile",
-    "funkwhale_api.providers.youtube",
     "funkwhale_api.providers.acoustid",
     "funkwhale_api.subsonic",
 )
@@ -280,7 +279,7 @@ MEDIA_ROOT = env("MEDIA_ROOT", default=str(APPS_DIR("media")))
 
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#media-url
 MEDIA_URL = env("MEDIA_URL", default="/media/")
-
+FILE_UPLOAD_PERMISSIONS = 0o644
 # URL Configuration
 # ------------------------------------------------------------------------------
 ROOT_URLCONF = "config.urls"
@@ -446,7 +445,7 @@ REST_FRAMEWORK = {
     "DEFAULT_AUTHENTICATION_CLASSES": (
         "funkwhale_api.common.authentication.JSONWebTokenAuthenticationQS",
         "funkwhale_api.common.authentication.BearerTokenHeaderAuth",
-        "rest_framework_jwt.authentication.JSONWebTokenAuthentication",
+        "funkwhale_api.common.authentication.JSONWebTokenAuthentication",
         "rest_framework.authentication.SessionAuthentication",
         "rest_framework.authentication.BasicAuthentication",
     ),

api/config/urls.py

@@ -2,11 +2,13 @@
 from __future__ import unicode_literals
 
 from django.conf import settings
-from django.conf.urls import include, url
+from django.conf.urls import url
+from django.urls import include, path
 from django.conf.urls.static import static
 from django.contrib import admin
 from django.views import defaults as default_views
 
+
 urlpatterns = [
     # Django Admin, use {% url 'admin:index' %}
     url(settings.ADMIN_URL, admin.site.urls),
@@ -36,4 +38,6 @@ if settings.DEBUG:
     if "debug_toolbar" in settings.INSTALLED_APPS:
         import debug_toolbar
 
-        urlpatterns += [url(r"^__debug__/", include(debug_toolbar.urls))]
+        urlpatterns = [
+            path("api/__debug__/", include(debug_toolbar.urls))
+        ] + urlpatterns

api/funkwhale_api/common/authentication.py

@@ -56,3 +56,20 @@ class BearerTokenHeaderAuth(authentication.BaseJSONWebTokenAuthentication):
 
     def authenticate_header(self, request):
         return '{0} realm="{1}"'.format("Bearer", self.www_authenticate_realm)
+
+    def authenticate(self, request):
+        auth = super().authenticate(request)
+        if auth:
+            if not auth[0].actor:
+                auth[0].create_actor()
+        return auth
+
+
+class JSONWebTokenAuthentication(authentication.JSONWebTokenAuthentication):
+    def authenticate(self, request):
+        auth = super().authenticate(request)
+
+        if auth:
+            if not auth[0].actor:
+                auth[0].create_actor()
+        return auth

api/funkwhale_api/common/channels.py

+import json
+import logging
+
 from asgiref.sync import async_to_sync
 from channels.layers import get_channel_layer
+from django.core.serializers.json import DjangoJSONEncoder
 
+logger = logging.getLogger(__file__)
 channel_layer = get_channel_layer()
-group_send = async_to_sync(channel_layer.group_send)
 group_add = async_to_sync(channel_layer.group_add)
+
+
+def group_send(group, event):
+    # we serialize the payload ourselves and deserialize it to ensure it
+    # works with msgpack. This is dirty, but we'll find a better solution
+    # later
+    s = json.dumps(event, cls=DjangoJSONEncoder)
+    event = json.loads(s)
+    logger.debug(
+        "[channels] Dispatching %s to group %s: %s",
+        event["type"],
+        group,
+        {"type": event["data"]["type"]},
+    )
+    async_to_sync(channel_layer.group_send)(group, event)

api/funkwhale_api/common/consumers.py

@@ -16,3 +16,5 @@ class JsonAuthConsumer(JsonWebsocketConsumer):
         super().accept()
         for group in self.groups:
             channels.group_add(group, self.channel_name)
+        for group in self.scope["user"].get_channels_groups():
+            channels.group_add(group, self.channel_name)

api/funkwhale_api/common/scripts/__init__.py

 from . import create_actors
 from . import create_image_variations
 from . import django_permissions_to_user_permissions
+from . import migrate_to_user_libraries
 from . import test
 
 
@@ -8,5 +9,6 @@ __all__ = [
     "create_actors",
     "create_image_variations",
     "django_permissions_to_user_permissions",
+    "migrate_to_user_libraries",
     "test",
 ]

api/funkwhale_api/common/scripts/migrate_to_user_libraries.py

+"""
+Mirate instance files to a library #463. For each user that imported music on an
+instance, we will create a "default" library with related files and an instance-level
+visibility.
+
+Files without any import job will be bounded to a "default" library on the first
+superuser account found. This should now happen though.
+"""
+
+from funkwhale_api.music import models
+from funkwhale_api.users.models import User
+
+
+def main(command, **kwargs):
+    importer_ids = set(
+        models.ImportBatch.objects.values_list("submitted_by", flat=True)
+    )
+    importers = User.objects.filter(pk__in=importer_ids).order_by("id").select_related()
+    command.stdout.write(
+        "* {} users imported music on this instance".format(len(importers))
+    )
+    files = models.TrackFile.objects.filter(
+        library__isnull=True, jobs__isnull=False
+    ).distinct()
+    command.stdout.write(
+        "* Reassigning {} files to importers libraries...".format(files.count())
+    )
+    for user in importers:
+        command.stdout.write(
+            "  * Setting up @{}'s 'default' library".format(user.username)
+        )
+        library = user.actor.libraries.get_or_create(actor=user.actor, name="default")[
+            0
+        ]
+        user_files = files.filter(jobs__batch__submitted_by=user)
+        total = user_files.count()
+        command.stdout.write(
+            "    * Reassigning {} files to the user library...".format(total)
+        )
+        user_files.update(library=library)
+
+    files = models.TrackFile.objects.filter(
+        library__isnull=True, jobs__isnull=True
+    ).distinct()
+    command.stdout.write(
+        "* Handling {} files with no import jobs...".format(files.count())
+    )
+
+    user = User.objects.order_by("id").filter(is_superuser=True).first()
+
+    command.stdout.write("  * Setting up @{}'s 'default' library".format(user.username))
+    library = user.actor.libraries.get_or_create(actor=user.actor, name="default")[0]
+    total = files.count()
+    command.stdout.write(
+        "    * Reassigning {} files to the user library...".format(total)
+    )
+    files.update(library=library)
+    command.stdout.write(" * Done!")

api/funkwhale_api/common/serializers.py

+import collections
+
 from rest_framework import serializers
 
+from django.core.exceptions import ObjectDoesNotExist
+from django.utils.encoding import smart_text
+from django.utils.translation import ugettext_lazy as _
+
+
+class RelatedField(serializers.RelatedField):
+    default_error_messages = {
+        "does_not_exist": _("Object with {related_field_name}={value} does not exist."),
+        "invalid": _("Invalid value."),
+    }
+
+    def __init__(self, related_field_name, serializer, **kwargs):
+        self.related_field_name = related_field_name
+        self.serializer = serializer
+        self.filters = kwargs.pop("filters", None)
+        kwargs["queryset"] = kwargs.pop(
+            "queryset", self.serializer.Meta.model.objects.all()
+        )
+        super().__init__(**kwargs)
+
+    def get_filters(self, data):
+        filters = {self.related_field_name: data}
+        if self.filters:
+            filters.update(self.filters(self.context))
+        return filters
+
+    def to_internal_value(self, data):
+        try:
+            queryset = self.get_queryset()
+            filters = self.get_filters(data)
+            return queryset.get(**filters)
+        except ObjectDoesNotExist:
+            self.fail(
+                "does_not_exist",
+                related_field_name=self.related_field_name,
+                value=smart_text(data),
+            )
+        except (TypeError, ValueError):
+            self.fail("invalid")
+
+    def to_representation(self, obj):
+        return self.serializer.to_representation(obj)
+
+    def get_choices(self, cutoff=None):
+        queryset = self.get_queryset()
+        if queryset is None:
+            # Ensure that field.choices returns something sensible
+            # even when accessed with a read-only field.
+            return {}
+
+        if cutoff is not None:
+            queryset = queryset[:cutoff]
+
+        return collections.OrderedDict(
+            [
+                (
+                    self.to_representation(item)[self.related_field_name],
+                    self.display_value(item),
+                )
+                for item in queryset
+            ]
+        )
+
 
 class Action(object):
     def __init__(self, name, allow_all=False, qs_filter=None):
@@ -21,6 +86,7 @@ class ActionSerializer(serializers.Serializer):
     objects = serializers.JSONField(required=True)
     filters = serializers.DictField(required=False)
     actions = None
+    pk_field = "pk"
 
     def __init__(self, *args, **kwargs):
         self.actions_by_name = {a.name: a for a in self.actions}
@@ -51,7 +117,9 @@ class ActionSerializer(serializers.Serializer):
         if value == "all":
             return self.queryset.all().order_by("id")
         if type(value) in [list, tuple]:
-            return self.queryset.filter(pk__in=value).order_by("id")
+            return self.queryset.filter(
+                **{"{}__in".format(self.pk_field): value}
+            ).order_by("id")
 
         raise serializers.ValidationError(
             "{} is not a valid value for objects. You must provide either a "

api/funkwhale_api/favorites/views.py

@@ -3,9 +3,12 @@ from rest_framework.decorators import list_route
 from rest_framework.permissions import IsAuthenticatedOrReadOnly
 from rest_framework.response import Response
 
+from django.db.models import Prefetch
+
 from funkwhale_api.activity import record
 from funkwhale_api.common import fields, permissions
 from funkwhale_api.music.models import Track
+from funkwhale_api.music import utils as music_utils
 
 from . import filters, models, serializers
 
@@ -19,11 +22,7 @@ class TrackFavoriteViewSet(
 
     filter_class = filters.TrackFavoriteFilter
     serializer_class = serializers.UserTrackFavoriteSerializer
-    queryset = (
-        models.TrackFavorite.objects.all()
-        .select_related("track__artist", "track__album__artist", "user")
-        .prefetch_related("track__files")
-    )
+    queryset = models.TrackFavorite.objects.all().select_related("user")
     permission_classes = [
         permissions.ConditionalAuthentication,
         permissions.OwnerPermission,
@@ -49,9 +48,14 @@ class TrackFavoriteViewSet(
 
     def get_queryset(self):
         queryset = super().get_queryset()
-        return queryset.filter(
+        queryset = queryset.filter(
             fields.privacy_level_query(self.request.user, "user__privacy_level")
         )
+        tracks = Track.objects.annotate_playable_by_actor(
+            music_utils.get_actor_from_request(self.request)
+        ).select_related("artist", "album__artist")
+        queryset = queryset.prefetch_related(Prefetch("track", queryset=tracks))
+        return queryset
 
     def perform_create(self, serializer):
         track = Track.objects.get(pk=serializer.data["track"])

api/funkwhale_api/federation/activity.py

+import uuid
+
+from funkwhale_api.common import utils as funkwhale_utils
+
+PUBLIC_ADDRESS = "https://www.w3.org/ns/activitystreams#Public"
+
 ACTIVITY_TYPES = [
     "Accept",
     "Add",
@@ -58,4 +64,145 @@ def accept_follow(follow):
     from . import serializers
 
     serializer = serializers.AcceptFollowSerializer(follow)
-    return deliver(serializer.data, to=[follow.actor.url], on_behalf_of=follow.target)
+    return deliver(serializer.data, to=[follow.actor.fid], on_behalf_of=follow.target)
+
+
+def receive(activity, on_behalf_of):
+    from . import models
+    from . import serializers
+    from . import tasks
+
+    # we ensure the activity has the bare minimum structure before storing
+    # it in our database
+    serializer = serializers.BaseActivitySerializer(
+        data=activity, context={"actor": on_behalf_of, "local_recipients": True}
+    )
+    serializer.is_valid(raise_exception=True)
+    copy = serializer.save()
+    # we create inbox items for further delivery
+    items = [
+        models.InboxItem(activity=copy, actor=r, type="to")
+        for r in serializer.validated_data["recipients"]["to"]
+        if hasattr(r, "fid")
+    ]
+    items += [
+        models.InboxItem(activity=copy, actor=r, type="cc")
+        for r in serializer.validated_data["recipients"]["cc"]
+        if hasattr(r, "fid")
+    ]
+    models.InboxItem.objects.bulk_create(items)
+    # at this point, we have the activity in database. Even if we crash, it's
+    # okay, as we can retry later
+    tasks.dispatch_inbox.delay(activity_id=copy.pk)
+    return copy
+
+
+class Router:
+    def __init__(self):
+        self.routes = []
+
+    def connect(self, route, handler):
+        self.routes.append((route, handler))
+
+    def register(self, route):
+        def decorator(handler):
+            self.connect(route, handler)
+            return handler
+
+        return decorator
+
+
+class InboxRouter(Router):
+    def dispatch(self, payload, context):
+        """
+        Receives an Activity payload and some context and trigger our
+        business logic
+        """
+        for route, handler in self.routes:
+            if match_route(route, payload):
+                return handler(payload, context=context)
+
+
+class OutboxRouter(Router):
+    def dispatch(self, routing, context):
+        """
+        Receives a routing payload and some business objects in the context
+        and may yield data that should be persisted in the Activity model
+        for further delivery.
+        """
+        from . import models
+        from . import tasks
+
+        for route, handler in self.routes:
+            if match_route(route, routing):
+                activities_data = []
+                for e in handler(context):
+                    # a route can yield zero, one or more activity payloads
+                    if e:
+                        activities_data.append(e)
+
+                inbox_items_by_activity_uuid = {}
+                prepared_activities = []
+                for activity_data in activities_data:
+                    to = activity_data.pop("to", [])
+                    cc = activity_data.pop("cc", [])
+                    a = models.Activity(**activity_data)
+                    a.uuid = uuid.uuid4()
+                    to_items, new_to = prepare_inbox_items(to, "to")
+                    cc_items, new_cc = prepare_inbox_items(cc, "cc")
+                    if not to_items and not cc_items:
+                        continue
+                    inbox_items_by_activity_uuid[str(a.uuid)] = to_items + cc_items
+                    if new_to:
+                        a.payload["to"] = new_to
+                    if new_cc:
+                        a.payload["cc"] = new_cc
+                    prepared_activities.append(a)
+
+                activities = models.Activity.objects.bulk_create(prepared_activities)
+                activities = [a for a in activities if a]
+
+                final_inbox_items = []
+                for a in activities:
+                    try:
+                        prepared_inbox_items = inbox_items_by_activity_uuid[str(a.uuid)]
+                    except KeyError:
+                        continue
+
+                    for ii in prepared_inbox_items:
+                        ii.activity = a
+                        final_inbox_items.append(ii)
+
+                # create all inbox items, in bulk
+                models.InboxItem.objects.bulk_create(final_inbox_items)
+
+                for a in activities:
+                    funkwhale_utils.on_commit(
+                        tasks.dispatch_outbox.delay, activity_id=a.pk
+                    )
+                return activities
+
+
+def match_route(route, payload):
+    for key, value in route.items():
+        if payload.get(key) != value:
+            return False
+
+    return True
+
+
+def prepare_inbox_items(recipient_list, type):
+    from . import models
+
+    items = []
+    new_list = []  # we return a list of actors url instead
+
+    for r in recipient_list:
+        if r != PUBLIC_ADDRESS:
+            item = models.InboxItem(actor=r, type=type)
+            items.append(item)
+            new_list.append(r.fid)
+        else:
+            new_list.append(r)
+
+    return items, new_list

api/funkwhale_api/federation/actors.py

@@ -3,15 +3,11 @@ import logging
 import xml
 
 from django.conf import settings
-from django.db import transaction
 from django.urls import reverse
 from django.utils import timezone
 from rest_framework.exceptions import PermissionDenied
 
 from funkwhale_api.common import preferences, session
-from funkwhale_api.common import utils as funkwhale_utils
-from funkwhale_api.music import models as music_models
-from funkwhale_api.music import tasks as music_tasks
 
 from . import activity, keys, models, serializers, signing, utils
 
@@ -39,9 +35,9 @@ def get_actor_data(actor_url):
         raise ValueError("Invalid actor payload: {}".format(response.text))
 
 
-def get_actor(actor_url):
+def get_actor(fid):
     try:
-        actor = models.Actor.objects.get(url=actor_url)
+        actor = models.Actor.objects.get(fid=fid)
     except models.Actor.DoesNotExist:
         actor = None
     fetch_delta = datetime.timedelta(
@@ -50,7 +46,7 @@ def get_actor(actor_url):
     if actor and actor.last_fetch_date > timezone.now() - fetch_delta:
         # cache is hot, we can return as is
         return actor
-    data = get_actor_data(actor_url)
+    data = get_actor_data(fid)
     serializer = serializers.ActorSerializer(data=data)
     serializer.is_valid(raise_exception=True)
 
@@ -72,7 +68,7 @@ class SystemActor(object):
 
     def get_actor_instance(self):
         try:
-            return models.Actor.objects.get(url=self.get_actor_url())
+            return models.Actor.objects.get(fid=self.get_actor_id())
         except models.Actor.DoesNotExist:
             pass
         private, public = keys.get_key_pair()
@@ -83,7 +79,7 @@ class SystemActor(object):
         args["public_key"] = public.decode("utf-8")
         return models.Actor.objects.create(**args)
 
-    def get_actor_url(self):
+    def get_actor_id(self):
         return utils.full_url(
             reverse("federation:instance-actors-detail", kwargs={"actor": self.id})
         )
@@ -95,7 +91,7 @@ class SystemActor(object):
             "type": "Person",
             "name": name.format(host=settings.FEDERATION_HOSTNAME),
             "manually_approves_followers": True,
-            "url": self.get_actor_url(),
+            "fid": self.get_actor_id(),
             "shared_inbox_url": utils.full_url(
                 reverse("federation:instance-actors-inbox", kwargs={"actor": id})
             ),
@@ -178,91 +174,13 @@ class SystemActor(object):
         if ac["object"]["type"] != "Follow":
             return
 
-        if ac["object"]["actor"] != sender.url:
+        if ac["object"]["actor"] != sender.fid:
             # not the same actor, permission issue
             return
 
         self.handle_undo_follow(ac, sender)
 
 
-class LibraryActor(SystemActor):
-    id = "library"
-    name = "{host}'s library"
-    summary = "Bot account to federate with {host}'s library"
-    additional_attributes = {"manually_approves_followers": True}
-
-    def serialize(self):
-        data = super().serialize()
-        urls = data.setdefault("url", [])
-        urls.append(
-            {
-                "type": "Link",
-                "mediaType": "application/activity+json",
-                "name": "library",
-                "href": utils.full_url(reverse("federation:music:files-list")),
-            }
-        )
-        return data
-
-    @property
-    def manually_approves_followers(self):
-        return preferences.get("federation__music_needs_approval")
-
-    @transaction.atomic
-    def handle_create(self, ac, sender):
-        try:
-            remote_library = models.Library.objects.get(
-                actor=sender, federation_enabled=True
-            )
-        except models.Library.DoesNotExist:
-            logger.info("Skipping import, we're not following %s", sender.url)
-            return
-
-        if ac["object"]["type"] != "Collection":
-            return
-
-        if ac["object"]["totalItems"] <= 0:
-            return
-
-        try:
-            items = ac["object"]["items"]
-        except KeyError:
-            logger.warning("No items in collection!")
-            return
-
-        item_serializers = [
-            serializers.AudioSerializer(data=i, context={"library": remote_library})
-            for i in items
-        ]
-        now = timezone.now()
-        valid_serializers = []
-        for s in item_serializers:
-            if s.is_valid():
-                valid_serializers.append(s)
-            else:
-                logger.debug("Skipping invalid item %s, %s", s.initial_data, s.errors)
-
-        lts = []
-        for s in valid_serializers:
-            lts.append(s.save())
-
-        if remote_library.autoimport:
-            batch = music_models.ImportBatch.objects.create(source="federation")
-            for lt in lts:
-                if lt.creation_date < now:
-                    # track was already in the library, we do not trigger
-                    # an import
-                    continue
-                job = music_models.ImportJob.objects.create(
-                    batch=batch, library_track=lt, mbid=lt.mbid, source=lt.url
-                )
-                funkwhale_utils.on_commit(
-                    music_tasks.import_job_run.delay,
-                    import_job_id=job.pk,
-                    use_acoustid=False,
-                )
-
-
 class TestActor(SystemActor):
     id = "test"
     name = "{host}'s test account"
@@ -321,7 +239,7 @@ class TestActor(SystemActor):
                 {},
             ],
             "type": "Create",
-            "actor": test_actor.url,
+            "actor": test_actor.fid,
             "id": "{}/activity".format(reply_url),
             "published": now.isoformat(),
             "to": ac["actor"],
@@ -336,14 +254,14 @@ class TestActor(SystemActor):
                 "sensitive": False,
                 "url": reply_url,
                 "to": [ac["actor"]],
-                "attributedTo": test_actor.url,
+                "attributedTo": test_actor.fid,
                 "cc": [],
                 "attachment": [],
                 "tag": [
                     {
                         "type": "Mention",
                         "href": ac["actor"],
-                        "name": sender.mention_username,
+                        "name": sender.full_username,
                     }
                 ],
             },
@@ -359,7 +277,7 @@ class TestActor(SystemActor):
         )[0]
         activity.deliver(
             serializers.FollowSerializer(follow_back).data,
-            to=[follow_back.target.url],
+            to=[follow_back.target.fid],
             on_behalf_of=follow_back.actor,
         )
 
@@ -373,7 +291,7 @@ class TestActor(SystemActor):
             return
         undo = serializers.UndoFollowSerializer(follow).data
         follow.delete()
-        activity.deliver(undo, to=[sender.url], on_behalf_of=actor)
+        activity.deliver(undo, to=[sender.fid], on_behalf_of=actor)
 
 
-SYSTEM_ACTORS = {"library": LibraryActor(), "test": TestActor()}
+SYSTEM_ACTORS = {"test": TestActor()}

api/funkwhale_api/federation/admin.py

@@ -6,14 +6,14 @@ from . import models
 @admin.register(models.Actor)
 class ActorAdmin(admin.ModelAdmin):
     list_display = [
-        "url",
+        "fid",
         "domain",
         "preferred_username",
         "type",
         "creation_date",
         "last_fetch_date",
     ]
-    search_fields = ["url", "domain", "preferred_username"]
+    search_fields = ["fid", "domain", "preferred_username"]
     list_filter = ["type"]
 
 
@@ -21,14 +21,14 @@ class ActorAdmin(admin.ModelAdmin):
 class FollowAdmin(admin.ModelAdmin):
     list_display = ["actor", "target", "approved", "creation_date"]
     list_filter = ["approved"]
-    search_fields = ["actor__url", "target__url"]
+    search_fields = ["actor__fid", "target__fid"]
     list_select_related = True
 
 
 @admin.register(models.Library)
 class LibraryAdmin(admin.ModelAdmin):
     list_display = ["actor", "url", "creation_date", "fetched_date", "tracks_count"]
-    search_fields = ["actor__url", "url"]
+    search_fields = ["actor__fid", "url"]
     list_filter = ["federation_enabled", "download_files", "autoimport"]
     list_select_related = True
 

api/funkwhale_api/federation/api_serializers.py

+from rest_framework import serializers
+
+from funkwhale_api.common import serializers as common_serializers
+from funkwhale_api.music import models as music_models
+
+from . import serializers as federation_serializers
+from . import models
+
+
+class NestedLibraryFollowSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = models.LibraryFollow
+        fields = ["creation_date", "uuid", "fid", "approved", "modification_date"]
+
+
+class LibrarySerializer(serializers.ModelSerializer):
+    actor = federation_serializers.APIActorSerializer()
+    files_count = serializers.SerializerMethodField()
+    follow = serializers.SerializerMethodField()
+
+    class Meta:
+        model = music_models.Library
+        fields = [
+            "fid",
+            "uuid",
+            "actor",
+            "name",
+            "description",
+            "creation_date",
+            "files_count",
+            "privacy_level",
+            "follow",
+        ]
+
+    def get_files_count(self, o):
+        return max(getattr(o, "_files_count", 0), o.files_count)
+
+    def get_follow(self, o):
+        try:
+            return NestedLibraryFollowSerializer(o._follows[0]).data
+        except (AttributeError, IndexError):
+            return None
+
+
+class LibraryFollowSerializer(serializers.ModelSerializer):
+    target = common_serializers.RelatedField("uuid", LibrarySerializer(), required=True)
+
+    class Meta:
+        model = models.LibraryFollow
+        fields = ["creation_date", "uuid", "target", "approved"]
+        read_only_fields = ["uuid", "approved", "creation_date"]
+
+    def validate_target(self, v):
+        actor = self.context["actor"]
+        if v.received_follows.filter(actor=actor).exists():
+            raise serializers.ValidationError("You are already following this library")
+        return v

api/funkwhale_api/federation/api_urls.py

 from rest_framework import routers
 
-from . import views
+from . import api_views
 
 router = routers.SimpleRouter()
-router.register(r"libraries", views.LibraryViewSet, "libraries")
-router.register(r"library-tracks", views.LibraryTrackViewSet, "library-tracks")
+router.register(r"follows/library", api_views.LibraryFollowViewSet, "library-follows")
+router.register(r"libraries", api_views.LibraryViewSet, "libraries")
 
 urlpatterns = router.urls

api/funkwhale_api/federation/api_views.py

+import requests.exceptions
+
+from django.db.models import Count
+
+from rest_framework import decorators
+from rest_framework import mixins
+from rest_framework import permissions
+from rest_framework import response
+from rest_framework import viewsets
+
+from funkwhale_api.music import models as music_models
+
+from . import api_serializers
+from . import filters
+from . import models
+from . import routes
+from . import serializers
+from . import utils
+
+
+class LibraryFollowViewSet(
+    mixins.CreateModelMixin,
+    mixins.ListModelMixin,
+    mixins.RetrieveModelMixin,
+    viewsets.GenericViewSet,
+):
+    lookup_field = "uuid"
+    queryset = (
+        models.LibraryFollow.objects.all()
+        .order_by("-creation_date")
+        .select_related("target__actor", "actor")
+    )
+    serializer_class = api_serializers.LibraryFollowSerializer
+    permission_classes = [permissions.IsAuthenticated]
+    filter_class = filters.LibraryFollowFilter
+    ordering_fields = ("creation_date",)
+
+    def get_queryset(self):
+        qs = super().get_queryset()
+        return qs.filter(actor=self.request.user.actor)
+
+    def perform_create(self, serializer):
+        follow = serializer.save(actor=self.request.user.actor)
+        routes.outbox.dispatch({"type": "Follow"}, context={"follow": follow})
+
+    def get_serializer_context(self):
+        context = super().get_serializer_context()
+        context["actor"] = self.request.user.actor
+        return context
+
+
+class LibraryViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet):
+    lookup_field = "uuid"
+    queryset = (
+        music_models.Library.objects.all()
+        .order_by("-creation_date")
+        .select_related("actor")
+        .annotate(_files_count=Count("files"))
+    )
+    serializer_class = api_serializers.LibrarySerializer
+    permission_classes = [permissions.IsAuthenticated]
+    filter_class = filters.LibraryFollowFilter
+    ordering_fields = ("creation_date",)
+
+    def get_queryset(self):
+        qs = super().get_queryset()
+        return qs.viewable_by(actor=self.request.user.actor)
+
+    @decorators.list_route(methods=["post"])
+    def scan(self, request, *args, **kwargs):
+        try:
+            fid = request.data["fid"]
+        except KeyError:
+            return response.Response({"fid": ["This field is required"]})
+        try:
+            library = utils.retrieve(
+                fid,
+                queryset=self.queryset,
+                serializer_class=serializers.LibrarySerializer,
+            )
+        except requests.exceptions.RequestException as e:
+            return response.Response(
+                {"detail": "Error while scanning the library: {}".format(str(e))},
+                status=400,
+            )
+        except serializers.serializers.ValidationError as e:
+            return response.Response(
+                {"detail": "Invalid data in remote library: {}".format(str(e))},
+                status=400,
+            )
+        serializer = self.serializer_class(library)
+        return response.Response({"count": 1, "results": [serializer.data]})

api/funkwhale_api/federation/factories.py

@@ -8,6 +8,7 @@ from django.utils import timezone
 from django.utils.http import http_date
 
 from funkwhale_api.factories import registry
+from funkwhale_api.users import factories as user_factories
 
 from . import keys, models
 
@@ -61,6 +62,10 @@ class LinkFactory(factory.Factory):
         audio = factory.Trait(mediaType=factory.Iterator(["audio/mp3", "audio/ogg"]))
 
 
+def create_user(actor):
+    return user_factories.UserFactory(actor=actor)
+
+
 @registry.register
 class ActorFactory(factory.DjangoModelFactory):
     public_key = None
@@ -68,7 +73,7 @@ class ActorFactory(factory.DjangoModelFactory):
     preferred_username = factory.Faker("user_name")
     summary = factory.Faker("paragraph")
     domain = factory.Faker("domain_name")
-    url = factory.LazyAttribute(
+    fid = factory.LazyAttribute(
         lambda o: "https://{}/users/{}".format(o.domain, o.preferred_username)
     )
     inbox_url = factory.LazyAttribute(
@@ -81,20 +86,34 @@ class ActorFactory(factory.DjangoModelFactory):
     class Meta:
         model = models.Actor
 
-    class Params:
-        local = factory.Trait(
-            domain=factory.LazyAttribute(lambda o: settings.FEDERATION_HOSTNAME)
-        )
-
-    @classmethod
-    def _generate(cls, create, attrs):
-        has_public = attrs.get("public_key") is not None
-        has_private = attrs.get("private_key") is not None
-        if not has_public and not has_private:
+    @factory.post_generation
+    def local(self, create, extracted, **kwargs):
+        if not extracted and not kwargs:
+            return
+        from funkwhale_api.users.factories import UserFactory
+
+        self.domain = settings.FEDERATION_HOSTNAME
+        self.save(update_fields=["domain"])
+        if not create:
+            if extracted and hasattr(extracted, "pk"):
+                extracted.actor = self
+            else:
+                UserFactory.build(actor=self, **kwargs)
+        if extracted and hasattr(extracted, "pk"):
+            extracted.actor = self
+            extracted.save(update_fields=["user"])
+        else:
+            self.user = UserFactory(actor=self, **kwargs)
+
+    @factory.post_generation
+    def keys(self, create, extracted, **kwargs):
+        if not create:
+            # Simple build, do nothing.
+            return
+        if not extracted:
             private, public = keys.get_key_pair()
-            attrs["private_key"] = private.decode("utf-8")
-            attrs["public_key"] = public.decode("utf-8")
-        return super()._generate(create, attrs)
+            self.private_key = private.decode("utf-8")
+            self.public_key = public.decode("utf-8")
 
 
 @registry.register
@@ -110,15 +129,70 @@ class FollowFactory(factory.DjangoModelFactory):
 
 
 @registry.register
-class LibraryFactory(factory.DjangoModelFactory):
+class MusicLibraryFactory(factory.django.DjangoModelFactory):
+    actor = factory.SubFactory(ActorFactory)
+    privacy_level = "me"
+    name = factory.Faker("sentence")
+    description = factory.Faker("sentence")
+    files_count = 0
+
+    class Meta:
+        model = "music.Library"
+
+    @factory.post_generation
+    def fid(self, create, extracted, **kwargs):
+        if not create:
+            # Simple build, do nothing.
+            return
+
+        self.fid = extracted or self.get_federation_id()
+
+    @factory.post_generation
+    def followers_url(self, create, extracted, **kwargs):
+        if not create:
+            # Simple build, do nothing.
+            return
+
+        self.followers_url = extracted or self.fid + "/followers"
+
+
+@registry.register
+class LibraryScan(factory.django.DjangoModelFactory):
+    library = factory.SubFactory(MusicLibraryFactory)
+    actor = factory.SubFactory(ActorFactory)
+    total_files = factory.LazyAttribute(lambda o: o.library.files_count)
+
+    class Meta:
+        model = "music.LibraryScan"
+
+
+@registry.register
+class ActivityFactory(factory.django.DjangoModelFactory):
     actor = factory.SubFactory(ActorFactory)
     url = factory.Faker("url")
-    federation_enabled = True
-    download_files = False
-    autoimport = False
+    payload = factory.LazyFunction(lambda: {"type": "Create"})
 
     class Meta:
-        model = models.Library
+        model = "federation.Activity"
+
+
+@registry.register
+class InboxItemFactory(factory.django.DjangoModelFactory):
+    actor = factory.SubFactory(ActorFactory)
+    activity = factory.SubFactory(ActivityFactory)
+    type = "to"
+
+    class Meta:
+        model = "federation.InboxItem"
+
+
+@registry.register
+class LibraryFollowFactory(factory.DjangoModelFactory):
+    target = factory.SubFactory(MusicLibraryFactory)
+    actor = factory.SubFactory(ActorFactory)
+
+    class Meta:
+        model = "federation.LibraryFollow"
 
 
 class ArtistMetadataFactory(factory.Factory):
@@ -161,25 +235,6 @@ class LibraryTrackMetadataFactory(factory.Factory):
         model = dict
 
 
-@registry.register
-class LibraryTrackFactory(factory.DjangoModelFactory):
-    library = factory.SubFactory(LibraryFactory)
-    url = factory.Faker("url")
-    title = factory.Faker("sentence")
-    artist_name = factory.Faker("sentence")
-    album_title = factory.Faker("sentence")
-    audio_url = factory.Faker("url")
-    audio_mimetype = "audio/ogg"
-    metadata = factory.SubFactory(LibraryTrackMetadataFactory)
-    published_date = factory.LazyFunction(timezone.now)
-
-    class Meta:
-        model = models.LibraryTrack
-
-    class Params:
-        with_audio_file = factory.Trait(audio_file=factory.django.FileField())
-
-
 @registry.register(name="federation.Note")
 class NoteFactory(factory.Factory):
     type = "Note"
@@ -192,22 +247,6 @@ class NoteFactory(factory.Factory):
         model = dict
 
 
-@registry.register(name="federation.Activity")
-class ActivityFactory(factory.Factory):
-    type = "Create"
-    id = factory.Faker("url")
-    published = factory.LazyFunction(lambda: timezone.now().isoformat())
-    actor = factory.Faker("url")
-    object = factory.SubFactory(
-        NoteFactory,
-        actor=factory.SelfAttribute("..actor"),
-        published=factory.SelfAttribute("..published"),
-    )
-
-    class Meta:
-        model = dict
-
-
 @registry.register(name="federation.AudioMetadata")
 class AudioMetadataFactory(factory.Factory):
     recording = factory.LazyAttribute(

api/funkwhale_api/federation/filters.py

 import django_filters
 
 from funkwhale_api.common import fields
-from funkwhale_api.common import search
 
 from . import models
 
 
-class LibraryFilter(django_filters.FilterSet):
-    approved = django_filters.BooleanFilter("following__approved")
-    q = fields.SearchFilter(search_fields=["actor__domain"])
-
-    class Meta:
-        model = models.Library
-        fields = {
-            "approved": ["exact"],
-            "federation_enabled": ["exact"],
-            "download_files": ["exact"],
-            "autoimport": ["exact"],
-            "tracks_count": ["exact"],
-        }
-
-
-class LibraryTrackFilter(django_filters.FilterSet):
-    library = django_filters.CharFilter("library__uuid")
-    status = django_filters.CharFilter(method="filter_status")
-    q = fields.SmartSearchFilter(
-        config=search.SearchConfig(
-            search_fields={
-                "domain": {"to": "library__actor__domain"},
-                "artist": {"to": "artist_name"},
-                "album": {"to": "album_title"},
-                "title": {"to": "title"},
-            },
-            filter_fields={
-                "domain": {"to": "library__actor__domain"},
-                "artist": {"to": "artist_name__iexact"},
-                "album": {"to": "album_title__iexact"},
-                "title": {"to": "title__iexact"},
-            },
-        )
-    )
-
-    def filter_status(self, queryset, field_name, value):
-        if value == "imported":
-            return queryset.filter(local_track_file__isnull=False)
-        elif value == "not_imported":
-            return queryset.filter(local_track_file__isnull=True).exclude(
-                import_jobs__status="pending"
-            )
-        elif value == "import_pending":
-            return queryset.filter(import_jobs__status="pending")
-        return queryset
-
-    class Meta:
-        model = models.LibraryTrack
-        fields = {
-            "library": ["exact"],
-            "artist_name": ["exact", "icontains"],
-            "title": ["exact", "icontains"],
-            "album_title": ["exact", "icontains"],
-            "audio_mimetype": ["exact", "icontains"],
-        }
-
-
 class FollowFilter(django_filters.FilterSet):
     pending = django_filters.CharFilter(method="filter_pending")
     ordering = django_filters.OrderingFilter(
@@ -84,3 +26,9 @@ class FollowFilter(django_filters.FilterSet):
         if value.lower() in ["true", "1", "yes"]:
             queryset = queryset.filter(approved__isnull=True)
         return queryset
+
+
+class LibraryFollowFilter(django_filters.FilterSet):
+    class Meta:
+        model = models.LibraryFollow
+        fields = ["approved"]

api/funkwhale_api/federation/library.py

@@ -71,8 +71,7 @@ def scan_from_account_name(account_name):
     return data
 
 
-def get_library_data(library_url):
-    actor = actors.SYSTEM_ACTORS["library"].get_actor_instance()
+def get_library_data(library_url, actor):
     auth = signing.get_auth(actor.private_key, actor.private_key_id)
     try:
         response = session.get_session().get(
@@ -98,8 +97,7 @@ def get_library_data(library_url):
     return serializer.validated_data
 
 
-def get_library_page(library, page_url):
-    actor = actors.SYSTEM_ACTORS["library"].get_actor_instance()
+def get_library_page(library, page_url, actor):
     auth = signing.get_auth(actor.private_key, actor.private_key_id)
     response = session.get_session().get(
         page_url,