Merge branch 'release/0.17'

api/funkwhale_api/favorites/admin.py

-from django.contrib import admin
+from funkwhale_api.common import admin
 
 from . import models
 

api/funkwhale_api/favorites/serializers.py

-
 from rest_framework import serializers
 
 from funkwhale_api.activity import serializers as activity_serializers

api/funkwhale_api/favorites/views.py

@@ -3,9 +3,12 @@ from rest_framework.decorators import list_route
 from rest_framework.permissions import IsAuthenticatedOrReadOnly
 from rest_framework.response import Response
 
+from django.db.models import Prefetch
+
 from funkwhale_api.activity import record
 from funkwhale_api.common import fields, permissions
 from funkwhale_api.music.models import Track
+from funkwhale_api.music import utils as music_utils
 
 from . import filters, models, serializers
 
@@ -19,11 +22,7 @@ class TrackFavoriteViewSet(
 
     filter_class = filters.TrackFavoriteFilter
     serializer_class = serializers.UserTrackFavoriteSerializer
-    queryset = (
-        models.TrackFavorite.objects.all()
-        .select_related("track__artist", "track__album__artist", "user")
-        .prefetch_related("track__files")
-    )
+    queryset = models.TrackFavorite.objects.all().select_related("user")
     permission_classes = [
         permissions.ConditionalAuthentication,
         permissions.OwnerPermission,
@@ -49,9 +48,14 @@ class TrackFavoriteViewSet(
 
     def get_queryset(self):
         queryset = super().get_queryset()
-        return queryset.filter(
+        queryset = queryset.filter(
             fields.privacy_level_query(self.request.user, "user__privacy_level")
         )
+        tracks = Track.objects.annotate_playable_by_actor(
+            music_utils.get_actor_from_request(self.request)
+        ).select_related("artist", "album__artist")
+        queryset = queryset.prefetch_related(Prefetch("track", queryset=tracks))
+        return queryset
 
     def perform_create(self, serializer):
         track = Track.objects.get(pk=serializer.data["track"])

api/funkwhale_api/federation/activity.py

+import uuid
+import logging
+
+from django.db import transaction, IntegrityError
+from django.db.models import Q
+
+from funkwhale_api.common import channels
+from funkwhale_api.common import utils as funkwhale_utils
+
+
+logger = logging.getLogger(__name__)
+PUBLIC_ADDRESS = "https://www.w3.org/ns/activitystreams#Public"
+
 ACTIVITY_TYPES = [
     "Accept",
     "Add",
@@ -48,14 +61,340 @@ OBJECT_TYPES = [
 ] + ACTIVITY_TYPES
 
 
-def deliver(activity, on_behalf_of, to=[]):
+BROADCAST_TO_USER_ACTIVITIES = ["Follow", "Accept"]
+
+
+@transaction.atomic
+def receive(activity, on_behalf_of):
+    from . import models
+    from . import serializers
     from . import tasks
 
-    return tasks.send.delay(activity=activity, actor_id=on_behalf_of.pk, to=to)
+    # we ensure the activity has the bare minimum structure before storing
+    # it in our database
+    serializer = serializers.BaseActivitySerializer(
+        data=activity, context={"actor": on_behalf_of, "local_recipients": True}
+    )
+    serializer.is_valid(raise_exception=True)
+    try:
+        copy = serializer.save()
+    except IntegrityError:
+        logger.warning(
+            "[federation] Discarding already elivered activity %s",
+            serializer.validated_data.get("id"),
+        )
+        return
 
+    local_to_recipients = get_actors_from_audience(activity.get("to", []))
+    local_to_recipients = local_to_recipients.exclude(user=None)
 
-def accept_follow(follow):
-    from . import serializers
+    local_cc_recipients = get_actors_from_audience(activity.get("cc", []))
+    local_cc_recipients = local_cc_recipients.exclude(user=None)
+
+    inbox_items = []
+    for recipients, type in [(local_to_recipients, "to"), (local_cc_recipients, "cc")]:
+
+        for r in recipients.values_list("pk", flat=True):
+            inbox_items.append(models.InboxItem(actor_id=r, type=type, activity=copy))
+
+    models.InboxItem.objects.bulk_create(inbox_items)
+
+    # at this point, we have the activity in database. Even if we crash, it's
+    # okay, as we can retry later
+    funkwhale_utils.on_commit(tasks.dispatch_inbox.delay, activity_id=copy.pk)
+    return copy
+
+
+class Router:
+    def __init__(self):
+        self.routes = []
+
+    def connect(self, route, handler):
+        self.routes.append((route, handler))
+
+    def register(self, route):
+        def decorator(handler):
+            self.connect(route, handler)
+            return handler
+
+        return decorator
+
+
+class InboxRouter(Router):
+    @transaction.atomic
+    def dispatch(self, payload, context):
+        """
+        Receives an Activity payload and some context and trigger our
+        business logic
+        """
+        from . import api_serializers
+        from . import models
+
+        for route, handler in self.routes:
+            if match_route(route, payload):
+                r = handler(payload, context=context)
+                activity_obj = context.get("activity")
+                if activity_obj and r:
+                    # handler returned additional data we can use
+                    # to update the activity target
+                    for key, value in r.items():
+                        setattr(activity_obj, key, value)
+
+                    update_fields = []
+                    for k in r.keys():
+                        if k in ["object", "target", "related_object"]:
+                            update_fields += [
+                                "{}_id".format(k),
+                                "{}_content_type".format(k),
+                            ]
+                        else:
+                            update_fields.append(k)
+                    activity_obj.save(update_fields=update_fields)
+
+                if payload["type"] not in BROADCAST_TO_USER_ACTIVITIES:
+                    return
+
+                inbox_items = context.get(
+                    "inbox_items", models.InboxItem.objects.none()
+                )
+                inbox_items = (
+                    inbox_items.select_related()
+                    .select_related("actor__user")
+                    .prefetch_related(
+                        "activity__object",
+                        "activity__target",
+                        "activity__related_object",
+                    )
+                )
+
+                for ii in inbox_items:
+                    user = ii.actor.get_user()
+                    if not user:
+                        continue
+                    group = "user.{}.inbox".format(user.pk)
+                    channels.group_send(
+                        group,
+                        {
+                            "type": "event.send",
+                            "text": "",
+                            "data": {
+                                "type": "inbox.item_added",
+                                "item": api_serializers.InboxItemSerializer(ii).data,
+                            },
+                        },
+                    )
+                return
+
+
+class OutboxRouter(Router):
+    @transaction.atomic
+    def dispatch(self, routing, context):
+        """
+        Receives a routing payload and some business objects in the context
+        and may yield data that should be persisted in the Activity model
+        for further delivery.
+        """
+        from . import models
+        from . import tasks
+
+        for route, handler in self.routes:
+            if not match_route(route, routing):
+                continue
+
+            activities_data = []
+            for e in handler(context):
+                # a route can yield zero, one or more activity payloads
+                if e:
+                    activities_data.append(e)
+            inbox_items_by_activity_uuid = {}
+            deliveries_by_activity_uuid = {}
+            prepared_activities = []
+            for activity_data in activities_data:
+                activity_data["payload"]["actor"] = activity_data["actor"].fid
+                to = activity_data["payload"].pop("to", [])
+                cc = activity_data["payload"].pop("cc", [])
+                a = models.Activity(**activity_data)
+                a.uuid = uuid.uuid4()
+                to_inbox_items, to_deliveries, new_to = prepare_deliveries_and_inbox_items(
+                    to, "to"
+                )
+                cc_inbox_items, cc_deliveries, new_cc = prepare_deliveries_and_inbox_items(
+                    cc, "cc"
+                )
+                if not any(
+                    [to_inbox_items, to_deliveries, cc_inbox_items, cc_deliveries]
+                ):
+                    continue
+                deliveries_by_activity_uuid[str(a.uuid)] = to_deliveries + cc_deliveries
+                inbox_items_by_activity_uuid[str(a.uuid)] = (
+                    to_inbox_items + cc_inbox_items
+                )
+                if new_to:
+                    a.payload["to"] = new_to
+                if new_cc:
+                    a.payload["cc"] = new_cc
+                prepared_activities.append(a)
+
+            activities = models.Activity.objects.bulk_create(prepared_activities)
+
+            for activity in activities:
+                if str(activity.uuid) in deliveries_by_activity_uuid:
+                    for obj in deliveries_by_activity_uuid[str(a.uuid)]:
+                        obj.activity = activity
+
+                if str(activity.uuid) in inbox_items_by_activity_uuid:
+                    for obj in inbox_items_by_activity_uuid[str(a.uuid)]:
+                        obj.activity = activity
+
+            # create all deliveries and items, in bulk
+            models.Delivery.objects.bulk_create(
+                [
+                    obj
+                    for collection in deliveries_by_activity_uuid.values()
+                    for obj in collection
+                ]
+            )
+            models.InboxItem.objects.bulk_create(
+                [
+                    obj
+                    for collection in inbox_items_by_activity_uuid.values()
+                    for obj in collection
+                ]
+            )
+
+            for a in activities:
+                funkwhale_utils.on_commit(tasks.dispatch_outbox.delay, activity_id=a.pk)
+            return activities
+
+
+def recursive_gettattr(obj, key):
+    """
+    Given a dictionary such as {'user': {'name': 'Bob'}} and
+    a dotted string such as user.name, returns 'Bob'.
+
+    If the value is not present, returns None
+    """
+    v = obj
+    for k in key.split("."):
+        v = v.get(k)
+        if v is None:
+            return
+
+    return v
+
+
+def match_route(route, payload):
+    for key, value in route.items():
+        payload_value = recursive_gettattr(payload, key)
+        if payload_value != value:
+            return False
+
+    return True
+
+
+def prepare_deliveries_and_inbox_items(recipient_list, type):
+    """
+    Given a list of recipients (
+        either actor instances, public adresses, a dictionnary with a "type" and "target"
+        keys for followers collections)
+    returns a list of deliveries, alist of inbox_items and a list
+    of urls to persist in the activity in place of the initial recipient list.
+    """
+    from . import models
+
+    local_recipients = set()
+    remote_inbox_urls = set()
+    urls = []
+
+    for r in recipient_list:
+        if isinstance(r, models.Actor):
+            if r.is_local:
+                local_recipients.add(r)
+            else:
+                remote_inbox_urls.add(r.shared_inbox_url or r.inbox_url)
+            urls.append(r.fid)
+        elif r == PUBLIC_ADDRESS:
+            urls.append(r)
+        elif isinstance(r, dict) and r["type"] == "followers":
+            received_follows = (
+                r["target"]
+                .received_follows.filter(approved=True)
+                .select_related("actor__user")
+            )
+            for follow in received_follows:
+                actor = follow.actor
+                if actor.is_local:
+                    local_recipients.add(actor)
+                else:
+                    remote_inbox_urls.add(actor.shared_inbox_url or actor.inbox_url)
+            urls.append(r["target"].followers_url)
+
+    deliveries = [models.Delivery(inbox_url=url) for url in remote_inbox_urls]
+    inbox_items = [
+        models.InboxItem(actor=actor, type=type) for actor in local_recipients
+    ]
+
+    return inbox_items, deliveries, urls
+
+
+def join_queries_or(left, right):
+    if left:
+        return left | right
+    else:
+        return right
+
+
+def get_actors_from_audience(urls):
+    """
+    Given a list of urls such as [
+        "https://hello.world/@bob/followers",
+        "https://eldritch.cafe/@alice/followers",
+        "https://funkwhale.demo/libraries/uuid/followers",
+    ]
+    Returns a queryset of actors that are member of the collections
+    listed in the given urls. The urls may contain urls referring
+    to an actor, an actor followers collection or an library followers
+    collection.
+
+    Urls that don't match anything are simply discarded
+    """
+    from . import models
+
+    queries = {"followed": None, "actors": []}
+    for url in urls:
+        if url == PUBLIC_ADDRESS:
+            continue
+        queries["actors"].append(url)
+        queries["followed"] = join_queries_or(
+            queries["followed"], Q(target__followers_url=url)
+        )
+    final_query = None
+    if queries["actors"]:
+        final_query = join_queries_or(final_query, Q(fid__in=queries["actors"]))
+    if queries["followed"]:
+        actor_follows = models.Follow.objects.filter(queries["followed"], approved=True)
+        final_query = join_queries_or(
+            final_query, Q(pk__in=actor_follows.values_list("actor", flat=True))
+        )
+
+        library_follows = models.LibraryFollow.objects.filter(
+            queries["followed"], approved=True
+        )
+        final_query = join_queries_or(
+            final_query, Q(pk__in=library_follows.values_list("actor", flat=True))
+        )
+    if not final_query:
+        return models.Actor.objects.none()
+    return models.Actor.objects.filter(final_query)
+
+
+def get_inbox_urls(actor_queryset):
+    """
+    Given an actor queryset, returns a deduplicated set containing
+    all inbox or shared inbox urls where we should deliver our payloads for
+    those actors
+    """
+    values = actor_queryset.values("inbox_url", "shared_inbox_url")
 
-    serializer = serializers.AcceptFollowSerializer(follow)
-    return deliver(serializer.data, to=[follow.actor.url], on_behalf_of=follow.target)
+    urls = set([actor["shared_inbox_url"] or actor["inbox_url"] for actor in values])
+    return sorted(urls)

api/funkwhale_api/federation/actors.py

 import datetime
 import logging
-import xml
 
 from django.conf import settings
-from django.db import transaction
-from django.urls import reverse
 from django.utils import timezone
-from rest_framework.exceptions import PermissionDenied
 
 from funkwhale_api.common import preferences, session
-from funkwhale_api.common import utils as funkwhale_utils
-from funkwhale_api.music import models as music_models
-from funkwhale_api.music import tasks as music_tasks
 
-from . import activity, keys, models, serializers, signing, utils
+from . import models, serializers
 
 logger = logging.getLogger(__name__)
 
 
-def remove_tags(text):
-    logger.debug("Removing tags from %s", text)
-    return "".join(
-        xml.etree.ElementTree.fromstring("<div>{}</div>".format(text)).itertext()
-    )
-
-
 def get_actor_data(actor_url):
     response = session.get_session().get(
         actor_url,
@@ -39,9 +25,9 @@ def get_actor_data(actor_url):
         raise ValueError("Invalid actor payload: {}".format(response.text))
 
 
-def get_actor(actor_url):
+def get_actor(fid):
     try:
-        actor = models.Actor.objects.get(url=actor_url)
+        actor = models.Actor.objects.get(fid=fid)
     except models.Actor.DoesNotExist:
         actor = None
     fetch_delta = datetime.timedelta(
@@ -50,330 +36,8 @@ def get_actor(actor_url):
     if actor and actor.last_fetch_date > timezone.now() - fetch_delta:
         # cache is hot, we can return as is
         return actor
-    data = get_actor_data(actor_url)
+    data = get_actor_data(fid)
     serializer = serializers.ActorSerializer(data=data)
     serializer.is_valid(raise_exception=True)
 
     return serializer.save(last_fetch_date=timezone.now())
-
-
-class SystemActor(object):
-    additional_attributes = {}
-    manually_approves_followers = False
-
-    def get_request_auth(self):
-        actor = self.get_actor_instance()
-        return signing.get_auth(actor.private_key, actor.private_key_id)
-
-    def serialize(self):
-        actor = self.get_actor_instance()
-        serializer = serializers.ActorSerializer(actor)
-        return serializer.data
-
-    def get_actor_instance(self):
-        try:
-            return models.Actor.objects.get(url=self.get_actor_url())
-        except models.Actor.DoesNotExist:
-            pass
-        private, public = keys.get_key_pair()
-        args = self.get_instance_argument(
-            self.id, name=self.name, summary=self.summary, **self.additional_attributes
-        )
-        args["private_key"] = private.decode("utf-8")
-        args["public_key"] = public.decode("utf-8")
-        return models.Actor.objects.create(**args)
-
-    def get_actor_url(self):
-        return utils.full_url(
-            reverse("federation:instance-actors-detail", kwargs={"actor": self.id})
-        )
-
-    def get_instance_argument(self, id, name, summary, **kwargs):
-        p = {
-            "preferred_username": id,
-            "domain": settings.FEDERATION_HOSTNAME,
-            "type": "Person",
-            "name": name.format(host=settings.FEDERATION_HOSTNAME),
-            "manually_approves_followers": True,
-            "url": self.get_actor_url(),
-            "shared_inbox_url": utils.full_url(
-                reverse("federation:instance-actors-inbox", kwargs={"actor": id})
-            ),
-            "inbox_url": utils.full_url(
-                reverse("federation:instance-actors-inbox", kwargs={"actor": id})
-            ),
-            "outbox_url": utils.full_url(
-                reverse("federation:instance-actors-outbox", kwargs={"actor": id})
-            ),
-            "summary": summary.format(host=settings.FEDERATION_HOSTNAME),
-        }
-        p.update(kwargs)
-        return p
-
-    def get_inbox(self, data, actor=None):
-        raise NotImplementedError
-
-    def post_inbox(self, data, actor=None):
-        return self.handle(data, actor=actor)
-
-    def get_outbox(self, data, actor=None):
-        raise NotImplementedError
-
-    def post_outbox(self, data, actor=None):
-        raise NotImplementedError
-
-    def handle(self, data, actor=None):
-        """
-        Main entrypoint for handling activities posted to the
-        actor's inbox
-        """
-        logger.info("Received activity on %s inbox", self.id)
-
-        if actor is None:
-            raise PermissionDenied("Actor not authenticated")
-
-        serializer = serializers.ActivitySerializer(data=data, context={"actor": actor})
-        serializer.is_valid(raise_exception=True)
-
-        ac = serializer.data
-        try:
-            handler = getattr(self, "handle_{}".format(ac["type"].lower()))
-        except (KeyError, AttributeError):
-            logger.debug("No handler for activity %s", ac["type"])
-            return
-
-        return handler(data, actor)
-
-    def handle_follow(self, ac, sender):
-        serializer = serializers.FollowSerializer(
-            data=ac, context={"follow_actor": sender}
-        )
-        if not serializer.is_valid():
-            return logger.info("Invalid follow payload")
-        approved = True if not self.manually_approves_followers else None
-        follow = serializer.save(approved=approved)
-        if follow.approved:
-            return activity.accept_follow(follow)
-
-    def handle_accept(self, ac, sender):
-        system_actor = self.get_actor_instance()
-        serializer = serializers.AcceptFollowSerializer(
-            data=ac, context={"follow_target": sender, "follow_actor": system_actor}
-        )
-        if not serializer.is_valid(raise_exception=True):
-            return logger.info("Received invalid payload")
-
-        return serializer.save()
-
-    def handle_undo_follow(self, ac, sender):
-        system_actor = self.get_actor_instance()
-        serializer = serializers.UndoFollowSerializer(
-            data=ac, context={"actor": sender, "target": system_actor}
-        )
-        if not serializer.is_valid():
-            return logger.info("Received invalid payload")
-        serializer.save()
-
-    def handle_undo(self, ac, sender):
-        if ac["object"]["type"] != "Follow":
-            return
-
-        if ac["object"]["actor"] != sender.url:
-            # not the same actor, permission issue
-            return
-
-        self.handle_undo_follow(ac, sender)
-
-
-class LibraryActor(SystemActor):
-    id = "library"
-    name = "{host}'s library"
-    summary = "Bot account to federate with {host}'s library"
-    additional_attributes = {"manually_approves_followers": True}
-
-    def serialize(self):
-        data = super().serialize()
-        urls = data.setdefault("url", [])
-        urls.append(
-            {
-                "type": "Link",
-                "mediaType": "application/activity+json",
-                "name": "library",
-                "href": utils.full_url(reverse("federation:music:files-list")),
-            }
-        )
-        return data
-
-    @property
-    def manually_approves_followers(self):
-        return preferences.get("federation__music_needs_approval")
-
-    @transaction.atomic
-    def handle_create(self, ac, sender):
-        try:
-            remote_library = models.Library.objects.get(
-                actor=sender, federation_enabled=True
-            )
-        except models.Library.DoesNotExist:
-            logger.info("Skipping import, we're not following %s", sender.url)
-            return
-
-        if ac["object"]["type"] != "Collection":
-            return
-
-        if ac["object"]["totalItems"] <= 0:
-            return
-
-        try:
-            items = ac["object"]["items"]
-        except KeyError:
-            logger.warning("No items in collection!")
-            return
-
-        item_serializers = [
-            serializers.AudioSerializer(data=i, context={"library": remote_library})
-            for i in items
-        ]
-        now = timezone.now()
-        valid_serializers = []
-        for s in item_serializers:
-            if s.is_valid():
-                valid_serializers.append(s)
-            else:
-                logger.debug("Skipping invalid item %s, %s", s.initial_data, s.errors)
-
-        lts = []
-        for s in valid_serializers:
-            lts.append(s.save())
-
-        if remote_library.autoimport:
-            batch = music_models.ImportBatch.objects.create(source="federation")
-            for lt in lts:
-                if lt.creation_date < now:
-                    # track was already in the library, we do not trigger
-                    # an import
-                    continue
-                job = music_models.ImportJob.objects.create(
-                    batch=batch, library_track=lt, mbid=lt.mbid, source=lt.url
-                )
-                funkwhale_utils.on_commit(
-                    music_tasks.import_job_run.delay,
-                    import_job_id=job.pk,
-                    use_acoustid=False,
-                )
-
-
-class TestActor(SystemActor):
-    id = "test"
-    name = "{host}'s test account"
-    summary = (
-        "Bot account to test federation with {host}. "
-        "Send me /ping and I'll answer you."
-    )
-    additional_attributes = {"manually_approves_followers": False}
-    manually_approves_followers = False
-
-    def get_outbox(self, data, actor=None):
-        return {
-            "@context": [
-                "https://www.w3.org/ns/activitystreams",
-                "https://w3id.org/security/v1",
-                {},
-            ],
-            "id": utils.full_url(
-                reverse("federation:instance-actors-outbox", kwargs={"actor": self.id})
-            ),
-            "type": "OrderedCollection",
-            "totalItems": 0,
-            "orderedItems": [],
-        }
-
-    def parse_command(self, message):
-        """
-        Remove any links or fancy markup to extract /command from
-        a note message.
-        """
-        raw = remove_tags(message)
-        try:
-            return raw.split("/")[1]
-        except IndexError:
-            return
-
-    def handle_create(self, ac, sender):
-        if ac["object"]["type"] != "Note":
-            return
-
-        # we received a toot \o/
-        command = self.parse_command(ac["object"]["content"])
-        logger.debug("Parsed command: %s", command)
-        if command != "ping":
-            return
-
-        now = timezone.now()
-        test_actor = self.get_actor_instance()
-        reply_url = "https://{}/activities/note/{}".format(
-            settings.FEDERATION_HOSTNAME, now.timestamp()
-        )
-        reply_activity = {
-            "@context": [
-                "https://www.w3.org/ns/activitystreams",
-                "https://w3id.org/security/v1",
-                {},
-            ],
-            "type": "Create",
-            "actor": test_actor.url,
-            "id": "{}/activity".format(reply_url),
-            "published": now.isoformat(),
-            "to": ac["actor"],
-            "cc": [],
-            "object": {
-                "type": "Note",
-                "content": "Pong!",
-                "summary": None,
-                "published": now.isoformat(),
-                "id": reply_url,
-                "inReplyTo": ac["object"]["id"],
-                "sensitive": False,
-                "url": reply_url,
-                "to": [ac["actor"]],
-                "attributedTo": test_actor.url,
-                "cc": [],
-                "attachment": [],
-                "tag": [
-                    {
-                        "type": "Mention",
-                        "href": ac["actor"],
-                        "name": sender.mention_username,
-                    }
-                ],
-            },
-        }
-        activity.deliver(reply_activity, to=[ac["actor"]], on_behalf_of=test_actor)
-
-    def handle_follow(self, ac, sender):
-        super().handle_follow(ac, sender)
-        # also, we follow back
-        test_actor = self.get_actor_instance()
-        follow_back = models.Follow.objects.get_or_create(
-            actor=test_actor, target=sender, approved=None
-        )[0]
-        activity.deliver(
-            serializers.FollowSerializer(follow_back).data,
-            to=[follow_back.target.url],
-            on_behalf_of=follow_back.actor,
-        )
-
-    def handle_undo_follow(self, ac, sender):
-        super().handle_undo_follow(ac, sender)
-        actor = self.get_actor_instance()
-        # we also unfollow the sender, if possible
-        try:
-            follow = models.Follow.objects.get(target=sender, actor=actor)
-        except models.Follow.DoesNotExist:
-            return
-        undo = serializers.UndoFollowSerializer(follow).data
-        follow.delete()
-        activity.deliver(undo, to=[sender.url], on_behalf_of=actor)
-
-
-SYSTEM_ACTORS = {"library": LibraryActor(), "test": TestActor()}

api/funkwhale_api/federation/admin.py

-from django.contrib import admin
+from funkwhale_api.common import admin
 
 from . import models
+from . import tasks
+
+
+def redeliver_deliveries(modeladmin, request, queryset):
+    queryset.update(is_delivered=False)
+    for delivery in queryset:
+        tasks.deliver_to_remote.delay(delivery_id=delivery.pk)
+
+
+redeliver_deliveries.short_description = "Redeliver"
+
+
+def redeliver_activities(modeladmin, request, queryset):
+    for activity in queryset.select_related("actor__user"):
+        if activity.actor.get_user():
+            tasks.dispatch_outbox.delay(activity_id=activity.pk)
+        else:
+            tasks.dispatch_inbox.delay(activity_id=activity.pk)
+
+
+redeliver_activities.short_description = "Redeliver"
+
+
+@admin.register(models.Activity)
+class ActivityAdmin(admin.ModelAdmin):
+    list_display = ["type", "fid", "url", "actor", "creation_date"]
+    search_fields = ["payload", "fid", "url", "actor__domain"]
+    list_filter = ["type", "actor__domain"]
+    actions = [redeliver_activities]
+    list_select_related = True
 
 
 @admin.register(models.Actor)
 class ActorAdmin(admin.ModelAdmin):
     list_display = [
-        "url",
+        "fid",
         "domain",
         "preferred_username",
         "type",
         "creation_date",
         "last_fetch_date",
     ]
-    search_fields = ["url", "domain", "preferred_username"]
+    search_fields = ["fid", "domain", "preferred_username"]
     list_filter = ["type"]
 
 
@@ -21,28 +51,36 @@ class ActorAdmin(admin.ModelAdmin):
 class FollowAdmin(admin.ModelAdmin):
     list_display = ["actor", "target", "approved", "creation_date"]
     list_filter = ["approved"]
-    search_fields = ["actor__url", "target__url"]
+    search_fields = ["actor__fid", "target__fid"]
+    list_select_related = True
+
+
+@admin.register(models.LibraryFollow)
+class LibraryFollowAdmin(admin.ModelAdmin):
+    list_display = ["actor", "target", "approved", "creation_date"]
+    list_filter = ["approved"]
+    search_fields = ["actor__fid", "target__fid"]
     list_select_related = True
 
 
-@admin.register(models.Library)
-class LibraryAdmin(admin.ModelAdmin):
-    list_display = ["actor", "url", "creation_date", "fetched_date", "tracks_count"]
-    search_fields = ["actor__url", "url"]
-    list_filter = ["federation_enabled", "download_files", "autoimport"]
+@admin.register(models.InboxItem)
+class InboxItemAdmin(admin.ModelAdmin):
+    list_display = ["actor", "activity", "type", "is_read"]
+    list_filter = ["type", "activity__type", "is_read"]
+    search_fields = ["actor__fid", "activity__fid"]
     list_select_related = True
 
 
-@admin.register(models.LibraryTrack)
-class LibraryTrackAdmin(admin.ModelAdmin):
+@admin.register(models.Delivery)
+class DeliveryAdmin(admin.ModelAdmin):
     list_display = [
-        "title",
-        "artist_name",
-        "album_title",
-        "url",
-        "library",
-        "creation_date",
-        "published_date",
+        "inbox_url",
+        "activity",
+        "last_attempt_date",
+        "attempts",
+        "is_delivered",
     ]
-    search_fields = ["library__url", "url", "artist_name", "title", "album_title"]
+    list_filter = ["activity__type", "is_delivered"]
+    search_fields = ["inbox_url"]
     list_select_related = True
+    actions = [redeliver_deliveries]

api/funkwhale_api/federation/api_serializers.py

+from rest_framework import serializers
+
+from funkwhale_api.common import serializers as common_serializers
+from funkwhale_api.music import models as music_models
+
+from . import filters
+from . import models
+from . import serializers as federation_serializers
+
+
+class NestedLibraryFollowSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = models.LibraryFollow
+        fields = ["creation_date", "uuid", "fid", "approved", "modification_date"]
+
+
+class LibraryScanSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = music_models.LibraryScan
+        fields = [
+            "total_files",
+            "processed_files",
+            "errored_files",
+            "status",
+            "creation_date",
+            "modification_date",
+        ]
+
+
+class LibrarySerializer(serializers.ModelSerializer):
+    actor = federation_serializers.APIActorSerializer()
+    uploads_count = serializers.SerializerMethodField()
+    latest_scan = serializers.SerializerMethodField()
+    follow = serializers.SerializerMethodField()
+
+    class Meta:
+        model = music_models.Library
+        fields = [
+            "fid",
+            "uuid",
+            "actor",
+            "name",
+            "description",
+            "creation_date",
+            "uploads_count",
+            "privacy_level",
+            "follow",
+            "latest_scan",
+        ]
+
+    def get_uploads_count(self, o):
+        return max(getattr(o, "_uploads_count", 0), o.uploads_count)
+
+    def get_follow(self, o):
+        try:
+            return NestedLibraryFollowSerializer(o._follows[0]).data
+        except (AttributeError, IndexError):
+            return None
+
+    def get_latest_scan(self, o):
+        scan = o.scans.order_by("-creation_date").first()
+        if scan:
+            return LibraryScanSerializer(scan).data
+
+
+class LibraryFollowSerializer(serializers.ModelSerializer):
+    target = common_serializers.RelatedField("uuid", LibrarySerializer(), required=True)
+    actor = serializers.SerializerMethodField()
+
+    class Meta:
+        model = models.LibraryFollow
+        fields = ["creation_date", "actor", "uuid", "target", "approved"]
+        read_only_fields = ["uuid", "actor", "approved", "creation_date"]
+
+    def validate_target(self, v):
+        actor = self.context["actor"]
+        if v.actor == actor:
+            raise serializers.ValidationError("You cannot follow your own library")
+
+        if v.received_follows.filter(actor=actor).exists():
+            raise serializers.ValidationError("You are already following this library")
+        return v
+
+    def get_actor(self, o):
+        return federation_serializers.APIActorSerializer(o.actor).data
+
+
+def serialize_generic_relation(activity, obj):
+    data = {"uuid": obj.uuid, "type": obj._meta.label}
+    if data["type"] == "music.Library":
+        data["name"] = obj.name
+    if data["type"] == "federation.LibraryFollow":
+        data["approved"] = obj.approved
+
+    return data
+
+
+class ActivitySerializer(serializers.ModelSerializer):
+    actor = federation_serializers.APIActorSerializer()
+    object = serializers.SerializerMethodField()
+    target = serializers.SerializerMethodField()
+    related_object = serializers.SerializerMethodField()
+
+    class Meta:
+        model = models.Activity
+        fields = [
+            "uuid",
+            "fid",
+            "actor",
+            "payload",
+            "object",
+            "target",
+            "related_object",
+            "actor",
+            "creation_date",
+            "type",
+        ]
+
+    def get_object(self, o):
+        if o.object:
+            return serialize_generic_relation(o, o.object)
+
+    def get_related_object(self, o):
+        if o.related_object:
+            return serialize_generic_relation(o, o.related_object)
+
+    def get_target(self, o):
+        if o.target:
+            return serialize_generic_relation(o, o.target)
+
+
+class InboxItemSerializer(serializers.ModelSerializer):
+    activity = ActivitySerializer()
+
+    class Meta:
+        model = models.InboxItem
+        fields = ["id", "type", "activity", "is_read"]
+        read_only_fields = ["id", "type", "activity"]
+
+
+class InboxItemActionSerializer(common_serializers.ActionSerializer):
+    actions = [common_serializers.Action("read", allow_all=True)]
+    filterset_class = filters.InboxItemFilter
+
+    def handle_read(self, objects):
+        return objects.update(is_read=True)

api/funkwhale_api/federation/api_urls.py

 from rest_framework import routers
 
-from . import views
+from . import api_views
 
 router = routers.SimpleRouter()
-router.register(r"libraries", views.LibraryViewSet, "libraries")
-router.register(r"library-tracks", views.LibraryTrackViewSet, "library-tracks")
+router.register(r"follows/library", api_views.LibraryFollowViewSet, "library-follows")
+router.register(r"inbox", api_views.InboxItemViewSet, "inbox")
+router.register(r"libraries", api_views.LibraryViewSet, "libraries")
 
 urlpatterns = router.urls

api/funkwhale_api/federation/api_views.py

+import requests.exceptions
+
+from django.db import transaction
+from django.db.models import Count
+
+from rest_framework import decorators
+from rest_framework import mixins
+from rest_framework import permissions
+from rest_framework import response
+from rest_framework import viewsets
+
+from funkwhale_api.music import models as music_models
+
+from . import activity
+from . import api_serializers
+from . import filters
+from . import models
+from . import routes
+from . import serializers
+from . import utils
+
+
+@transaction.atomic
+def update_follow(follow, approved):
+    follow.approved = approved
+    follow.save(update_fields=["approved"])
+    routes.outbox.dispatch({"type": "Accept"}, context={"follow": follow})
+
+
+class LibraryFollowViewSet(
+    mixins.CreateModelMixin,
+    mixins.ListModelMixin,
+    mixins.RetrieveModelMixin,
+    mixins.DestroyModelMixin,
+    viewsets.GenericViewSet,
+):
+    lookup_field = "uuid"
+    queryset = (
+        models.LibraryFollow.objects.all()
+        .order_by("-creation_date")
+        .select_related("actor", "target__actor")
+    )
+    serializer_class = api_serializers.LibraryFollowSerializer
+    permission_classes = [permissions.IsAuthenticated]
+    filter_class = filters.LibraryFollowFilter
+    ordering_fields = ("creation_date",)
+
+    def get_queryset(self):
+        qs = super().get_queryset()
+        return qs.filter(actor=self.request.user.actor)
+
+    def perform_create(self, serializer):
+        follow = serializer.save(actor=self.request.user.actor)
+        routes.outbox.dispatch({"type": "Follow"}, context={"follow": follow})
+
+    @transaction.atomic
+    def perform_destroy(self, instance):
+        routes.outbox.dispatch(
+            {"type": "Undo", "object": {"type": "Follow"}}, context={"follow": instance}
+        )
+        instance.delete()
+
+    def get_serializer_context(self):
+        context = super().get_serializer_context()
+        context["actor"] = self.request.user.actor
+        return context
+
+    @decorators.detail_route(methods=["post"])
+    def accept(self, request, *args, **kwargs):
+        try:
+            follow = self.queryset.get(
+                target__actor=self.request.user.actor, uuid=kwargs["uuid"]
+            )
+        except models.LibraryFollow.DoesNotExist:
+            return response.Response({}, status=404)
+        update_follow(follow, approved=True)
+        return response.Response(status=204)
+
+    @decorators.detail_route(methods=["post"])
+    def reject(self, request, *args, **kwargs):
+        try:
+            follow = self.queryset.get(
+                target__actor=self.request.user.actor, uuid=kwargs["uuid"]
+            )
+        except models.LibraryFollow.DoesNotExist:
+            return response.Response({}, status=404)
+
+        update_follow(follow, approved=False)
+        return response.Response(status=204)
+
+
+class LibraryViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet):
+    lookup_field = "uuid"
+    queryset = (
+        music_models.Library.objects.all()
+        .order_by("-creation_date")
+        .select_related("actor__user")
+        .annotate(_uploads_count=Count("uploads"))
+    )
+    serializer_class = api_serializers.LibrarySerializer
+    permission_classes = [permissions.IsAuthenticated]
+
+    def get_queryset(self):
+        qs = super().get_queryset()
+        return qs.viewable_by(actor=self.request.user.actor)
+
+    @decorators.detail_route(methods=["post"])
+    def scan(self, request, *args, **kwargs):
+        library = self.get_object()
+        if library.actor.get_user():
+            return response.Response({"status": "skipped"}, 200)
+
+        scan = library.schedule_scan(actor=request.user.actor)
+        if scan:
+            return response.Response(
+                {
+                    "status": "scheduled",
+                    "scan": api_serializers.LibraryScanSerializer(scan).data,
+                },
+                200,
+            )
+        return response.Response({"status": "skipped"}, 200)
+
+    @decorators.list_route(methods=["post"])
+    def fetch(self, request, *args, **kwargs):
+        try:
+            fid = request.data["fid"]
+        except KeyError:
+            return response.Response({"fid": ["This field is required"]})
+        try:
+            library = utils.retrieve(
+                fid,
+                queryset=self.queryset,
+                serializer_class=serializers.LibrarySerializer,
+            )
+        except requests.exceptions.RequestException as e:
+            return response.Response(
+                {"detail": "Error while fetching the library: {}".format(str(e))},
+                status=400,
+            )
+        except serializers.serializers.ValidationError as e:
+            return response.Response(
+                {"detail": "Invalid data in remote library: {}".format(str(e))},
+                status=400,
+            )
+        serializer = self.serializer_class(library)
+        return response.Response({"count": 1, "results": [serializer.data]})
+
+
+class InboxItemViewSet(
+    mixins.UpdateModelMixin,
+    mixins.ListModelMixin,
+    mixins.RetrieveModelMixin,
+    viewsets.GenericViewSet,
+):
+
+    queryset = (
+        models.InboxItem.objects.select_related("activity__actor")
+        .prefetch_related("activity__object", "activity__target")
+        .filter(activity__type__in=activity.BROADCAST_TO_USER_ACTIVITIES, type="to")
+        .order_by("-activity__creation_date")
+    )
+    serializer_class = api_serializers.InboxItemSerializer
+    permission_classes = [permissions.IsAuthenticated]
+    filter_class = filters.InboxItemFilter
+    ordering_fields = ("activity__creation_date",)
+
+    def get_queryset(self):
+        qs = super().get_queryset()
+        return qs.filter(actor=self.request.user.actor)
+
+    @decorators.list_route(methods=["post"])
+    def action(self, request, *args, **kwargs):
+        queryset = self.get_queryset()
+        serializer = api_serializers.InboxItemActionSerializer(
+            request.data, queryset=queryset
+        )
+        serializer.is_valid(raise_exception=True)
+        result = serializer.save()
+        return response.Response(result, status=200)

api/funkwhale_api/federation/dynamic_preferences_registry.py

-
 from dynamic_preferences import types
 from dynamic_preferences.registries import global_preferences_registry
 

api/funkwhale_api/federation/factories.py

@@ -8,6 +8,7 @@ from django.utils import timezone
 from django.utils.http import http_date
 
 from funkwhale_api.factories import registry
+from funkwhale_api.users import factories as user_factories
 
 from . import keys, models
 
@@ -61,6 +62,10 @@ class LinkFactory(factory.Factory):
         audio = factory.Trait(mediaType=factory.Iterator(["audio/mp3", "audio/ogg"]))
 
 
+def create_user(actor):
+    return user_factories.UserFactory(actor=actor)
+
+
 @registry.register
 class ActorFactory(factory.DjangoModelFactory):
     public_key = None
@@ -68,9 +73,12 @@ class ActorFactory(factory.DjangoModelFactory):
     preferred_username = factory.Faker("user_name")
     summary = factory.Faker("paragraph")
     domain = factory.Faker("domain_name")
-    url = factory.LazyAttribute(
+    fid = factory.LazyAttribute(
         lambda o: "https://{}/users/{}".format(o.domain, o.preferred_username)
     )
+    followers_url = factory.LazyAttribute(
+        lambda o: "https://{}/users/{}followers".format(o.domain, o.preferred_username)
+    )
     inbox_url = factory.LazyAttribute(
         lambda o: "https://{}/users/{}/inbox".format(o.domain, o.preferred_username)
     )
@@ -81,20 +89,34 @@ class ActorFactory(factory.DjangoModelFactory):
     class Meta:
         model = models.Actor
 
-    class Params:
-        local = factory.Trait(
-            domain=factory.LazyAttribute(lambda o: settings.FEDERATION_HOSTNAME)
-        )
+    @factory.post_generation
+    def local(self, create, extracted, **kwargs):
+        if not extracted and not kwargs:
+            return
+        from funkwhale_api.users.factories import UserFactory
+
+        self.domain = settings.FEDERATION_HOSTNAME
+        self.save(update_fields=["domain"])
+        if not create:
+            if extracted and hasattr(extracted, "pk"):
+                extracted.actor = self
+            else:
+                UserFactory.build(actor=self, **kwargs)
+        if extracted and hasattr(extracted, "pk"):
+            extracted.actor = self
+            extracted.save(update_fields=["user"])
+        else:
+            self.user = UserFactory(actor=self, **kwargs)
 
-    @classmethod
-    def _generate(cls, create, attrs):
-        has_public = attrs.get("public_key") is not None
-        has_private = attrs.get("private_key") is not None
-        if not has_public and not has_private:
+    @factory.post_generation
+    def keys(self, create, extracted, **kwargs):
+        if not create:
+            # Simple build, do nothing.
+            return
+        if not extracted:
             private, public = keys.get_key_pair()
-            attrs["private_key"] = private.decode("utf-8")
-            attrs["public_key"] = public.decode("utf-8")
-        return super()._generate(create, attrs)
+            self.private_key = private.decode("utf-8")
+            self.public_key = public.decode("utf-8")
 
 
 @registry.register
@@ -110,15 +132,72 @@ class FollowFactory(factory.DjangoModelFactory):
 
 
 @registry.register
-class LibraryFactory(factory.DjangoModelFactory):
+class MusicLibraryFactory(factory.django.DjangoModelFactory):
     actor = factory.SubFactory(ActorFactory)
-    url = factory.Faker("url")
-    federation_enabled = True
-    download_files = False
-    autoimport = False
+    privacy_level = "me"
+    name = factory.Faker("sentence")
+    description = factory.Faker("sentence")
+    uploads_count = 0
+    fid = factory.Faker("federation_url")
+
+    class Meta:
+        model = "music.Library"
+
+    @factory.post_generation
+    def followers_url(self, create, extracted, **kwargs):
+        if not create:
+            # Simple build, do nothing.
+            return
+
+        self.followers_url = extracted or self.fid + "/followers"
+
+
+@registry.register
+class LibraryScan(factory.django.DjangoModelFactory):
+    library = factory.SubFactory(MusicLibraryFactory)
+    actor = factory.SubFactory(ActorFactory)
+    total_files = factory.LazyAttribute(lambda o: o.library.uploads_count)
+
+    class Meta:
+        model = "music.LibraryScan"
+
+
+@registry.register
+class ActivityFactory(factory.django.DjangoModelFactory):
+    actor = factory.SubFactory(ActorFactory)
+    url = factory.Faker("federation_url")
+    payload = factory.LazyFunction(lambda: {"type": "Create"})
+
+    class Meta:
+        model = "federation.Activity"
+
+
+@registry.register
+class InboxItemFactory(factory.django.DjangoModelFactory):
+    actor = factory.SubFactory(ActorFactory, local=True)
+    activity = factory.SubFactory(ActivityFactory)
+    type = "to"
+
+    class Meta:
+        model = "federation.InboxItem"
+
+
+@registry.register
+class DeliveryFactory(factory.django.DjangoModelFactory):
+    activity = factory.SubFactory(ActivityFactory)
+    inbox_url = factory.Faker("url")
 
     class Meta:
-        model = models.Library
+        model = "federation.Delivery"
+
+
+@registry.register
+class LibraryFollowFactory(factory.DjangoModelFactory):
+    target = factory.SubFactory(MusicLibraryFactory)
+    actor = factory.SubFactory(ActorFactory)
+
+    class Meta:
+        model = "federation.LibraryFollow"
 
 
 class ArtistMetadataFactory(factory.Factory):
@@ -161,25 +240,6 @@ class LibraryTrackMetadataFactory(factory.Factory):
         model = dict
 
 
-@registry.register
-class LibraryTrackFactory(factory.DjangoModelFactory):
-    library = factory.SubFactory(LibraryFactory)
-    url = factory.Faker("url")
-    title = factory.Faker("sentence")
-    artist_name = factory.Faker("sentence")
-    album_title = factory.Faker("sentence")
-    audio_url = factory.Faker("url")
-    audio_mimetype = "audio/ogg"
-    metadata = factory.SubFactory(LibraryTrackMetadataFactory)
-    published_date = factory.LazyFunction(timezone.now)
-
-    class Meta:
-        model = models.LibraryTrack
-
-    class Params:
-        with_audio_file = factory.Trait(audio_file=factory.django.FileField())
-
-
 @registry.register(name="federation.Note")
 class NoteFactory(factory.Factory):
     type = "Note"
@@ -192,22 +252,6 @@ class NoteFactory(factory.Factory):
         model = dict
 
 
-@registry.register(name="federation.Activity")
-class ActivityFactory(factory.Factory):
-    type = "Create"
-    id = factory.Faker("url")
-    published = factory.LazyFunction(lambda: timezone.now().isoformat())
-    actor = factory.Faker("url")
-    object = factory.SubFactory(
-        NoteFactory,
-        actor=factory.SelfAttribute("..actor"),
-        published=factory.SelfAttribute("..published"),
-    )
-
-    class Meta:
-        model = dict
-
-
 @registry.register(name="federation.AudioMetadata")
 class AudioMetadataFactory(factory.Factory):
     recording = factory.LazyAttribute(
@@ -230,9 +274,9 @@ class AudioMetadataFactory(factory.Factory):
 @registry.register(name="federation.Audio")
 class AudioFactory(factory.Factory):
     type = "Audio"
-    id = factory.Faker("url")
+    id = factory.Faker("federation_url")
     published = factory.LazyFunction(lambda: timezone.now().isoformat())
-    actor = factory.Faker("url")
+    actor = factory.Faker("federation_url")
     url = factory.SubFactory(LinkFactory, audio=True)
     metadata = factory.SubFactory(LibraryTrackMetadataFactory)
 

api/funkwhale_api/federation/filters.py

-import django_filters
+import django_filters.widgets
 
 from funkwhale_api.common import fields
-from funkwhale_api.common import search
 
 from . import models
 
 
-class LibraryFilter(django_filters.FilterSet):
-    approved = django_filters.BooleanFilter("following__approved")
-    q = fields.SearchFilter(search_fields=["actor__domain"])
-
-    class Meta:
-        model = models.Library
-        fields = {
-            "approved": ["exact"],
-            "federation_enabled": ["exact"],
-            "download_files": ["exact"],
-            "autoimport": ["exact"],
-            "tracks_count": ["exact"],
-        }
-
-
-class LibraryTrackFilter(django_filters.FilterSet):
-    library = django_filters.CharFilter("library__uuid")
-    status = django_filters.CharFilter(method="filter_status")
-    q = fields.SmartSearchFilter(
-        config=search.SearchConfig(
-            search_fields={
-                "domain": {"to": "library__actor__domain"},
-                "artist": {"to": "artist_name"},
-                "album": {"to": "album_title"},
-                "title": {"to": "title"},
-            },
-            filter_fields={
-                "domain": {"to": "library__actor__domain"},
-                "artist": {"to": "artist_name__iexact"},
-                "album": {"to": "album_title__iexact"},
-                "title": {"to": "title__iexact"},
-            },
-        )
-    )
-
-    def filter_status(self, queryset, field_name, value):
-        if value == "imported":
-            return queryset.filter(local_track_file__isnull=False)
-        elif value == "not_imported":
-            return queryset.filter(local_track_file__isnull=True).exclude(
-                import_jobs__status="pending"
-            )
-        elif value == "import_pending":
-            return queryset.filter(import_jobs__status="pending")
-        return queryset
-
-    class Meta:
-        model = models.LibraryTrack
-        fields = {
-            "library": ["exact"],
-            "artist_name": ["exact", "icontains"],
-            "title": ["exact", "icontains"],
-            "album_title": ["exact", "icontains"],
-            "audio_mimetype": ["exact", "icontains"],
-        }
-
-
 class FollowFilter(django_filters.FilterSet):
     pending = django_filters.CharFilter(method="filter_pending")
     ordering = django_filters.OrderingFilter(
@@ -84,3 +26,23 @@ class FollowFilter(django_filters.FilterSet):
         if value.lower() in ["true", "1", "yes"]:
             queryset = queryset.filter(approved__isnull=True)
         return queryset
+
+
+class LibraryFollowFilter(django_filters.FilterSet):
+    class Meta:
+        model = models.LibraryFollow
+        fields = ["approved"]
+
+
+class InboxItemFilter(django_filters.FilterSet):
+    is_read = django_filters.BooleanFilter(
+        "is_read", widget=django_filters.widgets.BooleanWidget()
+    )
+    before = django_filters.NumberFilter(method="filter_before")
+
+    class Meta:
+        model = models.InboxItem
+        fields = ["is_read", "activity__type", "activity__actor"]
+
+    def filter_before(self, queryset, field_name, value):
+        return queryset.filter(pk__lte=value)

api/funkwhale_api/federation/library.py

-import json
-
 import requests
 from django.conf import settings
 
 from funkwhale_api.common import session
 
-from . import actors, models, serializers, signing, webfinger
-
-
-def scan_from_account_name(account_name):
-    """
-    Given an account name such as library@test.library, will:
-
-    1. Perform the webfinger lookup
-    2. Perform the actor lookup
-    3. Perform the library's collection lookup
-
-    and return corresponding data in a dictionary.
-    """
-    data = {}
-    try:
-        username, domain = webfinger.clean_acct(account_name, ensure_local=False)
-    except serializers.ValidationError:
-        return {"webfinger": {"errors": ["Invalid account string"]}}
-    system_library = actors.SYSTEM_ACTORS["library"].get_actor_instance()
-    data["local"] = {"following": False, "awaiting_approval": False}
-    try:
-        follow = models.Follow.objects.get(
-            target__preferred_username=username,
-            target__domain=username,
-            actor=system_library,
-        )
-        data["local"]["awaiting_approval"] = not bool(follow.approved)
-        data["local"]["following"] = True
-    except models.Follow.DoesNotExist:
-        pass
-
-    try:
-        data["webfinger"] = webfinger.get_resource("acct:{}".format(account_name))
-    except requests.ConnectionError:
-        return {"webfinger": {"errors": ["This webfinger resource is not reachable"]}}
-    except requests.HTTPError as e:
-        return {
-            "webfinger": {
-                "errors": [
-                    "Error {} during webfinger request".format(e.response.status_code)
-                ]
-            }
-        }
-    except json.JSONDecodeError as e:
-        return {"webfinger": {"errors": ["Could not process webfinger response"]}}
-
-    try:
-        data["actor"] = actors.get_actor_data(data["webfinger"]["actor_url"])
-    except requests.ConnectionError:
-        data["actor"] = {"errors": ["This actor is not reachable"]}
-        return data
-    except requests.HTTPError as e:
-        data["actor"] = {
-            "errors": ["Error {} during actor request".format(e.response.status_code)]
-        }
-        return data
-
-    serializer = serializers.LibraryActorSerializer(data=data["actor"])
-    if not serializer.is_valid():
-        data["actor"] = {"errors": ["Invalid ActivityPub actor"]}
-        return data
-    data["library"] = get_library_data(serializer.validated_data["library_url"])
-
-    return data
+from . import serializers, signing
 
 
-def get_library_data(library_url):
-    actor = actors.SYSTEM_ACTORS["library"].get_actor_instance()
+def get_library_data(library_url, actor):
     auth = signing.get_auth(actor.private_key, actor.private_key_id)
     try:
         response = session.get_session().get(
@@ -91,15 +25,14 @@ def get_library_data(library_url):
         return {"errors": ["Permission denied while scanning library"]}
     elif scode >= 400:
         return {"errors": ["Error {} while fetching the library".format(scode)]}
-    serializer = serializers.PaginatedCollectionSerializer(data=response.json())
+    serializer = serializers.LibrarySerializer(data=response.json())
     if not serializer.is_valid():
         return {"errors": ["Invalid ActivityPub response from remote library"]}
 
     return serializer.validated_data
 
 
-def get_library_page(library, page_url):
-    actor = actors.SYSTEM_ACTORS["library"].get_actor_instance()
+def get_library_page(library, page_url, actor):
     auth = signing.get_auth(actor.private_key, actor.private_key_id)
     response = session.get_session().get(
         page_url,
@@ -110,7 +43,7 @@ def get_library_page(library, page_url):
     )
     serializer = serializers.CollectionPageSerializer(
         data=response.json(),
-        context={"library": library, "item_serializer": serializers.AudioSerializer},
+        context={"library": library, "item_serializer": serializers.UploadSerializer},
     )
     serializer.is_valid(raise_exception=True)
     return serializer.validated_data

api/funkwhale_api/federation/migrations/0007_auto_20180807_1748.py

+# Generated by Django 2.0.7 on 2018-08-07 17:48
+
+import django.contrib.postgres.fields.jsonb
+import django.core.serializers.json
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("federation", "0006_auto_20180521_1702")]
+
+    operations = [
+        migrations.CreateModel(
+            name="Activity",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("uuid", models.UUIDField(default=uuid.uuid4, unique=True)),
+                (
+                    "fid",
+                    models.URLField(blank=True, max_length=500, null=True, unique=True),
+                ),
+                ("url", models.URLField(blank=True, max_length=500, null=True)),
+                (
+                    "payload",
+                    django.contrib.postgres.fields.jsonb.JSONField(
+                        default={},
+                        encoder=django.core.serializers.json.DjangoJSONEncoder,
+                        max_length=50000,
+                    ),
+                ),
+                (
+                    "creation_date",
+                    models.DateTimeField(default=django.utils.timezone.now),
+                ),
+                ("delivered", models.NullBooleanField(default=None)),
+                ("delivered_date", models.DateTimeField(blank=True, null=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name="LibraryFollow",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "fid",
+                    models.URLField(blank=True, max_length=500, null=True, unique=True),
+                ),
+                ("uuid", models.UUIDField(default=uuid.uuid4, unique=True)),
+                (
+                    "creation_date",
+                    models.DateTimeField(default=django.utils.timezone.now),
+                ),
+                ("modification_date", models.DateTimeField(auto_now=True)),
+                ("approved", models.NullBooleanField(default=None)),
+            ],
+        ),
+        migrations.RenameField("actor", "url", "fid"),
+        migrations.AddField(
+            model_name="actor",
+            name="url",
+            field=models.URLField(blank=True, max_length=500, null=True),
+        ),
+        migrations.AddField(
+            model_name="follow",
+            name="fid",
+            field=models.URLField(blank=True, max_length=500, null=True, unique=True),
+        ),
+        migrations.AddField(
+            model_name="libraryfollow",
+            name="actor",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="library_follows",
+                to="federation.Actor",
+            ),
+        ),
+    ]

api/funkwhale_api/federation/migrations/0008_auto_20180807_1748.py

+# Generated by Django 2.0.7 on 2018-08-07 17:48
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("music", "0029_auto_20180807_1748"),
+        ("federation", "0007_auto_20180807_1748"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="libraryfollow",
+            name="target",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="received_follows",
+                to="music.Library",
+            ),
+        ),
+        migrations.AddField(
+            model_name="activity",
+            name="actor",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="activities",
+                to="federation.Actor",
+            ),
+        ),
+        migrations.AlterUniqueTogether(
+            name="libraryfollow", unique_together={("actor", "target")}
+        ),
+    ]

api/funkwhale_api/federation/migrations/0009_auto_20180822_1956.py

+# Generated by Django 2.0.8 on 2018-08-22 19:56
+
+import django.contrib.postgres.fields.jsonb
+import django.core.serializers.json
+from django.db import migrations, models
+import django.db.models.deletion
+import funkwhale_api.federation.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("federation", "0008_auto_20180807_1748")]
+
+    operations = [
+        migrations.AddField(
+            model_name="activity",
+            name="recipient",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                related_name="inbox_activities",
+                to="federation.Actor",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="activity",
+            name="payload",
+            field=django.contrib.postgres.fields.jsonb.JSONField(
+                default=funkwhale_api.federation.models.empty_dict,
+                encoder=django.core.serializers.json.DjangoJSONEncoder,
+                max_length=50000,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="librarytrack",
+            name="metadata",
+            field=django.contrib.postgres.fields.jsonb.JSONField(
+                default=funkwhale_api.federation.models.empty_dict,
+                encoder=django.core.serializers.json.DjangoJSONEncoder,
+                max_length=10000,
+            ),
+        ),
+    ]

api/funkwhale_api/federation/migrations/0010_auto_20180904_2011.py

+# Generated by Django 2.0.8 on 2018-09-04 20:11
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("federation", "0009_auto_20180822_1956")]
+
+    operations = [
+        migrations.CreateModel(
+            name="InboxItem",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("is_delivered", models.BooleanField(default=False)),
+                (
+                    "type",
+                    models.CharField(
+                        choices=[("to", "to"), ("cc", "cc")], max_length=10
+                    ),
+                ),
+                ("last_delivery_date", models.DateTimeField(blank=True, null=True)),
+                ("delivery_attempts", models.PositiveIntegerField(default=0)),
+            ],
+        ),
+        migrations.RemoveField(model_name="activity", name="delivered"),
+        migrations.RemoveField(model_name="activity", name="delivered_date"),
+        migrations.RemoveField(model_name="activity", name="recipient"),
+        migrations.AlterField(
+            model_name="activity",
+            name="actor",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="outbox_activities",
+                to="federation.Actor",
+            ),
+        ),
+        migrations.AddField(
+            model_name="inboxitem",
+            name="activity",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="inbox_items",
+                to="federation.Activity",
+            ),
+        ),
+        migrations.AddField(
+            model_name="inboxitem",
+            name="actor",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="inbox_items",
+                to="federation.Actor",
+            ),
+        ),
+        migrations.AddField(
+            model_name="activity",
+            name="recipients",
+            field=models.ManyToManyField(
+                related_name="inbox_activities",
+                through="federation.InboxItem",
+                to="federation.Actor",
+            ),
+        ),
+    ]

api/funkwhale_api/federation/migrations/0011_auto_20180910_1902.py

+# Generated by Django 2.0.8 on 2018-09-10 19:02
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('contenttypes', '0002_remove_content_type_name'),
+        ('federation', '0010_auto_20180904_2011'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='activity',
+            name='object_content_type',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='objecting_activities', to='contenttypes.ContentType'),
+        ),
+        migrations.AddField(
+            model_name='activity',
+            name='object_id',
+            field=models.IntegerField(null=True),
+        ),
+        migrations.AddField(
+            model_name='activity',
+            name='related_object_content_type',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='related_objecting_activities', to='contenttypes.ContentType'),
+        ),
+        migrations.AddField(
+            model_name='activity',
+            name='related_object_id',
+            field=models.IntegerField(null=True),
+        ),
+        migrations.AddField(
+            model_name='activity',
+            name='target_content_type',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='targeting_activities', to='contenttypes.ContentType'),
+        ),
+        migrations.AddField(
+            model_name='activity',
+            name='target_id',
+            field=models.IntegerField(null=True),
+        ),
+        migrations.AddField(
+            model_name='activity',
+            name='type',
+            field=models.CharField(db_index=True, max_length=100, null=True),
+        ),
+        migrations.AddField(
+            model_name='inboxitem',
+            name='is_read',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AlterField(
+            model_name='activity',
+            name='creation_date',
+            field=models.DateTimeField(db_index=True, default=django.utils.timezone.now),
+        ),
+    ]

api/funkwhale_api/federation/migrations/0012_auto_20180920_1803.py

+# Generated by Django 2.0.8 on 2018-09-20 18:03
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('federation', '0011_auto_20180910_1902'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Delivery',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('is_delivered', models.BooleanField(default=False)),
+                ('last_attempt_date', models.DateTimeField(blank=True, null=True)),
+                ('attempts', models.PositiveIntegerField(default=0)),
+                ('inbox_url', models.URLField(max_length=500)),
+                ('activity', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='deliveries', to='federation.Activity')),
+            ],
+        ),
+        migrations.RemoveField(
+            model_name='inboxitem',
+            name='delivery_attempts',
+        ),
+        migrations.RemoveField(
+            model_name='inboxitem',
+            name='is_delivered',
+        ),
+        migrations.RemoveField(
+            model_name='inboxitem',
+            name='last_delivery_date',
+        ),
+    ]

api/funkwhale_api/federation/models.py

@@ -3,14 +3,20 @@ import uuid
 
 from django.conf import settings
 from django.contrib.postgres.fields import JSONField
+from django.contrib.contenttypes.fields import GenericForeignKey
+from django.contrib.contenttypes.models import ContentType
+from django.core.exceptions import ObjectDoesNotExist
 from django.core.serializers.json import DjangoJSONEncoder
 from django.db import models
 from django.utils import timezone
+from django.urls import reverse
 
 from funkwhale_api.common import session
 from funkwhale_api.common import utils as common_utils
 from funkwhale_api.music import utils as music_utils
 
+from . import utils as federation_utils
+
 TYPE_CHOICES = [
     ("Person", "Person"),
     ("Application", "Application"),
@@ -20,15 +26,47 @@ TYPE_CHOICES = [
 ]
 
 
+def empty_dict():
+    return {}
+
+
+def get_shared_inbox_url():
+    return federation_utils.full_url(reverse("federation:shared-inbox"))
+
+
+class FederationMixin(models.Model):
+    # federation id/url
+    fid = models.URLField(unique=True, max_length=500, db_index=True)
+    url = models.URLField(max_length=500, null=True, blank=True)
+
+    class Meta:
+        abstract = True
+
+
 class ActorQuerySet(models.QuerySet):
     def local(self, include=True):
         return self.exclude(user__isnull=include)
 
+    def with_current_usage(self):
+        qs = self
+        for s in ["pending", "skipped", "errored", "finished"]:
+            qs = qs.annotate(
+                **{
+                    "_usage_{}".format(s): models.Sum(
+                        "libraries__uploads__size",
+                        filter=models.Q(libraries__uploads__import_status=s),
+                    )
+                }
+            )
+
+        return qs
+
 
 class Actor(models.Model):
     ap_type = "Actor"
 
-    url = models.URLField(unique=True, max_length=500, db_index=True)
+    fid = models.URLField(unique=True, max_length=500, db_index=True)
+    url = models.URLField(max_length=500, null=True, blank=True)
     outbox_url = models.URLField(max_length=500)
     inbox_url = models.URLField(max_length=500)
     following_url = models.URLField(max_length=500, null=True, blank=True)
@@ -39,8 +77,8 @@ class Actor(models.Model):
     domain = models.CharField(max_length=1000)
     summary = models.CharField(max_length=500, null=True, blank=True)
     preferred_username = models.CharField(max_length=200, null=True, blank=True)
-    public_key = models.CharField(max_length=5000, null=True, blank=True)
-    private_key = models.CharField(max_length=5000, null=True, blank=True)
+    public_key = models.TextField(max_length=5000, null=True, blank=True)
+    private_key = models.TextField(max_length=5000, null=True, blank=True)
     creation_date = models.DateTimeField(default=timezone.now)
     last_fetch_date = models.DateTimeField(default=timezone.now)
     manually_approves_followers = models.NullBooleanField(default=None)
@@ -63,11 +101,14 @@ class Actor(models.Model):
 
     @property
     def private_key_id(self):
-        return "{}#main-key".format(self.url)
+        return "{}#main-key".format(self.fid)
 
     @property
-    def mention_username(self):
-        return "@{}@{}".format(self.preferred_username, self.domain)
+    def full_username(self):
+        return "{}@{}".format(self.preferred_username, self.domain)
+
+    def __str__(self):
+        return "{}@{}".format(self.preferred_username, self.domain)
 
     def save(self, **kwargs):
         lowercase_fields = ["domain"]
@@ -104,26 +145,137 @@ class Actor(models.Model):
         follows = self.received_follows.filter(approved=True)
         return self.followers.filter(pk__in=follows.values_list("actor", flat=True))
 
+    def should_autoapprove_follow(self, actor):
+        return False
 
-class Follow(models.Model):
-    ap_type = "Follow"
+    def get_user(self):
+        try:
+            return self.user
+        except ObjectDoesNotExist:
+            return None
 
+    def get_current_usage(self):
+        actor = self.__class__.objects.filter(pk=self.pk).with_current_usage().get()
+        data = {}
+        for s in ["pending", "skipped", "errored", "finished"]:
+            data[s] = getattr(actor, "_usage_{}".format(s)) or 0
+
+        data["total"] = sum(data.values())
+        return data
+
+
+class InboxItem(models.Model):
+    """
+    Store activities binding to local actors, with read/unread status.
+    """
+
+    actor = models.ForeignKey(
+        Actor, related_name="inbox_items", on_delete=models.CASCADE
+    )
+    activity = models.ForeignKey(
+        "Activity", related_name="inbox_items", on_delete=models.CASCADE
+    )
+    type = models.CharField(max_length=10, choices=[("to", "to"), ("cc", "cc")])
+    is_read = models.BooleanField(default=False)
+
+
+class Delivery(models.Model):
+    """
+    Store deliveries attempt to remote inboxes
+    """
+
+    is_delivered = models.BooleanField(default=False)
+    last_attempt_date = models.DateTimeField(null=True, blank=True)
+    attempts = models.PositiveIntegerField(default=0)
+    inbox_url = models.URLField(max_length=500)
+
+    activity = models.ForeignKey(
+        "Activity", related_name="deliveries", on_delete=models.CASCADE
+    )
+
+
+class Activity(models.Model):
+    actor = models.ForeignKey(
+        Actor, related_name="outbox_activities", on_delete=models.CASCADE
+    )
+    recipients = models.ManyToManyField(
+        Actor, related_name="inbox_activities", through=InboxItem
+    )
+    uuid = models.UUIDField(default=uuid.uuid4, unique=True)
+    fid = models.URLField(unique=True, max_length=500, null=True, blank=True)
+    url = models.URLField(max_length=500, null=True, blank=True)
+    payload = JSONField(default=empty_dict, max_length=50000, encoder=DjangoJSONEncoder)
+    creation_date = models.DateTimeField(default=timezone.now, db_index=True)
+    type = models.CharField(db_index=True, null=True, max_length=100)
+
+    # generic relations
+    object_id = models.IntegerField(null=True)
+    object_content_type = models.ForeignKey(
+        ContentType,
+        null=True,
+        on_delete=models.SET_NULL,
+        related_name="objecting_activities",
+    )
+    object = GenericForeignKey("object_content_type", "object_id")
+    target_id = models.IntegerField(null=True)
+    target_content_type = models.ForeignKey(
+        ContentType,
+        null=True,
+        on_delete=models.SET_NULL,
+        related_name="targeting_activities",
+    )
+    target = GenericForeignKey("target_content_type", "target_id")
+    related_object_id = models.IntegerField(null=True)
+    related_object_content_type = models.ForeignKey(
+        ContentType,
+        null=True,
+        on_delete=models.SET_NULL,
+        related_name="related_objecting_activities",
+    )
+    related_object = GenericForeignKey(
+        "related_object_content_type", "related_object_id"
+    )
+
+
+class AbstractFollow(models.Model):
+    ap_type = "Follow"
+    fid = models.URLField(unique=True, max_length=500, null=True, blank=True)
     uuid = models.UUIDField(default=uuid.uuid4, unique=True)
+    creation_date = models.DateTimeField(default=timezone.now)
+    modification_date = models.DateTimeField(auto_now=True)
+    approved = models.NullBooleanField(default=None)
+
+    class Meta:
+        abstract = True
+
+    def get_federation_id(self):
+        return federation_utils.full_url(
+            "{}#follows/{}".format(self.actor.fid, self.uuid)
+        )
+
+
+class Follow(AbstractFollow):
     actor = models.ForeignKey(
         Actor, related_name="emitted_follows", on_delete=models.CASCADE
     )
     target = models.ForeignKey(
         Actor, related_name="received_follows", on_delete=models.CASCADE
     )
-    creation_date = models.DateTimeField(default=timezone.now)
-    modification_date = models.DateTimeField(auto_now=True)
-    approved = models.NullBooleanField(default=None)
 
     class Meta:
         unique_together = ["actor", "target"]
 
-    def get_federation_url(self):
-        return "{}#follows/{}".format(self.actor.url, self.uuid)
+
+class LibraryFollow(AbstractFollow):
+    actor = models.ForeignKey(
+        Actor, related_name="library_follows", on_delete=models.CASCADE
+    )
+    target = models.ForeignKey(
+        "music.Library", related_name="received_follows", on_delete=models.CASCADE
+    )
+
+    class Meta:
+        unique_together = ["actor", "target"]
 
 
 class Library(models.Model):
@@ -167,7 +319,9 @@ class LibraryTrack(models.Model):
     artist_name = models.CharField(max_length=500)
     album_title = models.CharField(max_length=500)
     title = models.CharField(max_length=500)
-    metadata = JSONField(default={}, max_length=10000, encoder=DjangoJSONEncoder)
+    metadata = JSONField(
+        default=empty_dict, max_length=10000, encoder=DjangoJSONEncoder
+    )
 
     @property
     def mbid(self):