Audio federation

CONTRIBUTING.rst

@@ -249,6 +249,7 @@ Then, in separate terminals, you can setup as many different instances as you
 need::
 
     export COMPOSE_PROJECT_NAME=node2
+    export VUE_PORT=1234  # this has to be unique for each instance
     docker-compose -f dev.yml run --rm api python manage.py migrate
     docker-compose -f dev.yml run --rm api python manage.py createsuperuser
     docker-compose -f dev.yml up nginx api front nginx api celeryworker

api/config/api_urls.py

@@ -14,7 +14,7 @@ router.register(r"settings", GlobalPreferencesViewSet, base_name="settings")
 router.register(r"activity", activity_views.ActivityViewSet, "activity")
 router.register(r"tags", views.TagViewSet, "tags")
 router.register(r"tracks", views.TrackViewSet, "tracks")
-router.register(r"track-files", views.TrackFileViewSet, "trackfiles")
+router.register(r"uploads", views.UploadViewSet, "uploads")
 router.register(r"libraries", views.LibraryViewSet, "libraries")
 router.register(r"listen", views.ListenViewSet, "listen")
 router.register(r"artists", views.ArtistViewSet, "artists")

api/config/settings/common.py

@@ -514,8 +514,14 @@ ACCOUNT_USERNAME_BLACKLIST = [
     "me",
     "ghost",
     "_",
+    "-",
     "hello",
     "contact",
+    "inbox",
+    "outbox",
+    "shared-inbox",
+    "shared_inbox",
+    "actor",
 ] + env.list("ACCOUNT_USERNAME_BLACKLIST", default=[])
 
 EXTERNAL_REQUESTS_VERIFY_SSL = env.bool("EXTERNAL_REQUESTS_VERIFY_SSL", default=True)

api/funkwhale_api/common/permissions.py

@@ -9,7 +9,9 @@ from funkwhale_api.common import preferences
 class ConditionalAuthentication(BasePermission):
     def has_permission(self, request, view):
         if preferences.get("common__api_authentication_required"):
-            return request.user and request.user.is_authenticated
+            return (request.user and request.user.is_authenticated) or (
+                hasattr(request, "actor") and request.actor
+            )
         return True
 
 

api/funkwhale_api/common/scripts/migrate_to_user_libraries.py

@@ -5,6 +5,12 @@ visibility.
 
 Files without any import job will be bounded to a "default" library on the first
 superuser account found. This should now happen though.
+
+XXX TODO:
+
+- add followers url on actor
+- shared inbox url on actor
+- compute hash from files
 """
 
 from funkwhale_api.music import models
@@ -19,7 +25,7 @@ def main(command, **kwargs):
     command.stdout.write(
         "* {} users imported music on this instance".format(len(importers))
     )
-    files = models.TrackFile.objects.filter(
+    files = models.Upload.objects.filter(
         library__isnull=True, jobs__isnull=False
     ).distinct()
     command.stdout.write(
@@ -39,7 +45,7 @@ def main(command, **kwargs):
         )
         user_files.update(library=library)
 
-    files = models.TrackFile.objects.filter(
+    files = models.Upload.objects.filter(
         library__isnull=True, jobs__isnull=True
     ).distinct()
     command.stdout.write(

api/funkwhale_api/common/utils.py

@@ -64,3 +64,46 @@ class ChunkedPath(object):
             new_filename = "".join(chunks[3:]) + ".{}".format(ext)
             parts = chunks[:3] + [new_filename]
         return os.path.join(self.root, *parts)
+
+
+def chunk_queryset(source_qs, chunk_size):
+    """
+    From https://github.com/peopledoc/django-chunkator/blob/master/chunkator/__init__.py
+    """
+    pk = None
+    # In django 1.9, _fields is always present and `None` if 'values()' is used
+    # In Django 1.8 and below, _fields will only be present if using `values()`
+    has_fields = hasattr(source_qs, "_fields") and source_qs._fields
+    if has_fields:
+        if "pk" not in source_qs._fields:
+            raise ValueError("The values() call must include the `pk` field")
+
+    field = source_qs.model._meta.pk
+    # set the correct field name:
+    # for ForeignKeys, we want to use `model_id` field, and not `model`,
+    # to bypass default ordering on related model
+    order_by_field = field.attname
+
+    source_qs = source_qs.order_by(order_by_field)
+    queryset = source_qs
+    while True:
+        if pk:
+            queryset = source_qs.filter(pk__gt=pk)
+        page = queryset[:chunk_size]
+        page = list(page)
+        nb_items = len(page)
+
+        if nb_items == 0:
+            return
+
+        last_item = page[-1]
+        # source_qs._fields exists *and* is not none when using "values()"
+        if has_fields:
+            pk = last_item["pk"]
+        else:
+            pk = last_item.pk
+
+        yield page
+
+        if nb_items < chunk_size:
+            return

api/funkwhale_api/federation/activity.py

@@ -2,11 +2,12 @@ import uuid
 import logging
 
 from django.db import transaction, IntegrityError
-from django.utils import timezone
+from django.db.models import Q
 
 from funkwhale_api.common import channels
 from funkwhale_api.common import utils as funkwhale_utils
 
+
 logger = logging.getLogger(__name__)
 PUBLIC_ADDRESS = "https://www.w3.org/ns/activitystreams#Public"
 
@@ -83,18 +84,21 @@ def receive(activity, on_behalf_of):
             serializer.validated_data.get("id"),
         )
         return
-    # we create inbox items for further delivery
-    items = [
-        models.InboxItem(activity=copy, actor=r, type="to")
-        for r in serializer.validated_data["recipients"]["to"]
-        if hasattr(r, "fid")
-    ]
-    items += [
-        models.InboxItem(activity=copy, actor=r, type="cc")
-        for r in serializer.validated_data["recipients"]["cc"]
-        if hasattr(r, "fid")
-    ]
-    models.InboxItem.objects.bulk_create(items)
+
+    local_to_recipients = get_actors_from_audience(activity.get("to", []))
+    local_to_recipients = local_to_recipients.exclude(user=None)
+
+    local_cc_recipients = get_actors_from_audience(activity.get("cc", []))
+    local_cc_recipients = local_cc_recipients.exclude(user=None)
+
+    inbox_items = []
+    for recipients, type in [(local_to_recipients, "to"), (local_cc_recipients, "cc")]:
+
+        for r in recipients.values_list("pk", flat=True):
+            inbox_items.append(models.InboxItem(actor_id=r, type=type, activity=copy))
+
+    models.InboxItem.objects.bulk_create(inbox_items)
+
     # at this point, we have the activity in database. Even if we crash, it's
     # okay, as we can retry later
     funkwhale_utils.on_commit(tasks.dispatch_inbox.delay, activity_id=copy.pk)
@@ -153,6 +157,16 @@ class InboxRouter(Router):
                 inbox_items = context.get(
                     "inbox_items", models.InboxItem.objects.none()
                 )
+                inbox_items = (
+                    inbox_items.select_related()
+                    .select_related("actor__user")
+                    .prefetch_related(
+                        "activity__object",
+                        "activity__target",
+                        "activity__related_object",
+                    )
+                )
+
                 for ii in inbox_items:
                     user = ii.actor.get_user()
                     if not user:
@@ -169,7 +183,6 @@ class InboxRouter(Router):
                             },
                         },
                     )
-                inbox_items.update(is_delivered=True, last_delivery_date=timezone.now())
                 return
 
 
@@ -185,73 +198,203 @@ class OutboxRouter(Router):
         from . import tasks
 
         for route, handler in self.routes:
-            if match_route(route, routing):
-                activities_data = []
-                for e in handler(context):
-                    # a route can yield zero, one or more activity payloads
-                    if e:
-                        activities_data.append(e)
-                inbox_items_by_activity_uuid = {}
-                prepared_activities = []
-                for activity_data in activities_data:
-                    to = activity_data["payload"].pop("to", [])
-                    cc = activity_data["payload"].pop("cc", [])
-                    a = models.Activity(**activity_data)
-                    a.uuid = uuid.uuid4()
-                    to_items, new_to = prepare_inbox_items(to, "to")
-                    cc_items, new_cc = prepare_inbox_items(cc, "cc")
-                    if not to_items and not cc_items:
-                        continue
-                    inbox_items_by_activity_uuid[str(a.uuid)] = to_items + cc_items
-                    if new_to:
-                        a.payload["to"] = new_to
-                    if new_cc:
-                        a.payload["cc"] = new_cc
-                    prepared_activities.append(a)
-
-                activities = models.Activity.objects.bulk_create(prepared_activities)
-
-                final_inbox_items = []
-                for a in activities:
-                    try:
-                        prepared_inbox_items = inbox_items_by_activity_uuid[str(a.uuid)]
-                    except KeyError:
-                        continue
-
-                    for ii in prepared_inbox_items:
-                        ii.activity = a
-                        final_inbox_items.append(ii)
-
-                # create all inbox items, in bulk
-                models.InboxItem.objects.bulk_create(final_inbox_items)
-
-                for a in activities:
-                    funkwhale_utils.on_commit(
-                        tasks.dispatch_outbox.delay, activity_id=a.pk
-                    )
-                return activities
+            if not match_route(route, routing):
+                continue
+
+            activities_data = []
+            for e in handler(context):
+                # a route can yield zero, one or more activity payloads
+                if e:
+                    activities_data.append(e)
+            inbox_items_by_activity_uuid = {}
+            deliveries_by_activity_uuid = {}
+            prepared_activities = []
+            for activity_data in activities_data:
+                activity_data["payload"]["actor"] = activity_data["actor"].fid
+                to = activity_data["payload"].pop("to", [])
+                cc = activity_data["payload"].pop("cc", [])
+                a = models.Activity(**activity_data)
+                a.uuid = uuid.uuid4()
+                to_inbox_items, to_deliveries, new_to = prepare_deliveries_and_inbox_items(
+                    to, "to"
+                )
+                cc_inbox_items, cc_deliveries, new_cc = prepare_deliveries_and_inbox_items(
+                    cc, "cc"
+                )
+                if not any(
+                    [to_inbox_items, to_deliveries, cc_inbox_items, cc_deliveries]
+                ):
+                    continue
+                deliveries_by_activity_uuid[str(a.uuid)] = to_deliveries + cc_deliveries
+                inbox_items_by_activity_uuid[str(a.uuid)] = (
+                    to_inbox_items + cc_inbox_items
+                )
+                if new_to:
+                    a.payload["to"] = new_to
+                if new_cc:
+                    a.payload["cc"] = new_cc
+                prepared_activities.append(a)
+
+            activities = models.Activity.objects.bulk_create(prepared_activities)
+
+            for activity in activities:
+                if str(activity.uuid) in deliveries_by_activity_uuid:
+                    for obj in deliveries_by_activity_uuid[str(a.uuid)]:
+                        obj.activity = activity
+
+                if str(activity.uuid) in inbox_items_by_activity_uuid:
+                    for obj in inbox_items_by_activity_uuid[str(a.uuid)]:
+                        obj.activity = activity
+
+            # create all deliveries and items, in bulk
+            models.Delivery.objects.bulk_create(
+                [
+                    obj
+                    for collection in deliveries_by_activity_uuid.values()
+                    for obj in collection
+                ]
+            )
+            models.InboxItem.objects.bulk_create(
+                [
+                    obj
+                    for collection in inbox_items_by_activity_uuid.values()
+                    for obj in collection
+                ]
+            )
+
+            for a in activities:
+                funkwhale_utils.on_commit(tasks.dispatch_outbox.delay, activity_id=a.pk)
+            return activities
+
+
+def recursive_gettattr(obj, key):
+    """
+    Given a dictionary such as {'user': {'name': 'Bob'}} and
+    a dotted string such as user.name, returns 'Bob'.
+
+    If the value is not present, returns None
+    """
+    v = obj
+    for k in key.split("."):
+        v = v.get(k)
+        if v is None:
+            return
+
+    return v
 
 
 def match_route(route, payload):
     for key, value in route.items():
-        if payload.get(key) != value:
+        payload_value = recursive_gettattr(payload, key)
+        if payload_value != value:
             return False
 
     return True
 
 
-def prepare_inbox_items(recipient_list, type):
+def prepare_deliveries_and_inbox_items(recipient_list, type):
+    """
+    Given a list of recipients (
+        either actor instances, public adresses, a dictionnary with a "type" and "target"
+        keys for followers collections)
+    returns a list of deliveries, alist of inbox_items and a list
+    of urls to persist in the activity in place of the initial recipient list.
+    """
     from . import models
 
-    items = []
-    new_list = []  # we return a list of actors url instead
+    local_recipients = set()
+    remote_inbox_urls = set()
+    urls = []
 
     for r in recipient_list:
-        if r != PUBLIC_ADDRESS:
-            item = models.InboxItem(actor=r, type=type)
-            items.append(item)
-            new_list.append(r.fid)
-        else:
-            new_list.append(r)
-
-    return items, new_list
+        if isinstance(r, models.Actor):
+            if r.is_local:
+                local_recipients.add(r)
+            else:
+                remote_inbox_urls.add(r.shared_inbox_url or r.inbox_url)
+            urls.append(r.fid)
+        elif r == PUBLIC_ADDRESS:
+            urls.append(r)
+        elif isinstance(r, dict) and r["type"] == "followers":
+            received_follows = (
+                r["target"]
+                .received_follows.filter(approved=True)
+                .select_related("actor__user")
+            )
+            for follow in received_follows:
+                actor = follow.actor
+                if actor.is_local:
+                    local_recipients.add(actor)
+                else:
+                    remote_inbox_urls.add(actor.shared_inbox_url or actor.inbox_url)
+            urls.append(r["target"].followers_url)
+
+    deliveries = [models.Delivery(inbox_url=url) for url in remote_inbox_urls]
+    inbox_items = [
+        models.InboxItem(actor=actor, type=type) for actor in local_recipients
+    ]
+
+    return inbox_items, deliveries, urls
+
+
+def join_queries_or(left, right):
+    if left:
+        return left | right
+    else:
+        return right
+
+
+def get_actors_from_audience(urls):
+    """
+    Given a list of urls such as [
+        "https://hello.world/@bob/followers",
+        "https://eldritch.cafe/@alice/followers",
+        "https://funkwhale.demo/libraries/uuid/followers",
+    ]
+    Returns a queryset of actors that are member of the collections
+    listed in the given urls. The urls may contain urls referring
+    to an actor, an actor followers collection or an library followers
+    collection.
+
+    Urls that don't match anything are simply discarded
+    """
+    from . import models
+
+    queries = {"followed": None, "actors": []}
+    for url in urls:
+        if url == PUBLIC_ADDRESS:
+            continue
+        queries["actors"].append(url)
+        queries["followed"] = join_queries_or(
+            queries["followed"], Q(target__followers_url=url)
+        )
+    final_query = None
+    if queries["actors"]:
+        final_query = join_queries_or(final_query, Q(fid__in=queries["actors"]))
+    if queries["followed"]:
+        actor_follows = models.Follow.objects.filter(queries["followed"], approved=True)
+        final_query = join_queries_or(
+            final_query, Q(pk__in=actor_follows.values_list("actor", flat=True))
+        )
+
+        library_follows = models.LibraryFollow.objects.filter(
+            queries["followed"], approved=True
+        )
+        final_query = join_queries_or(
+            final_query, Q(pk__in=library_follows.values_list("actor", flat=True))
+        )
+    if not final_query:
+        return models.Actor.objects.none()
+    return models.Actor.objects.filter(final_query)
+
+
+def get_inbox_urls(actor_queryset):
+    """
+    Given an actor queryset, returns a deduplicated set containing
+    all inbox or shared inbox urls where we should deliver our payloads for
+    those actors
+    """
+    values = actor_queryset.values("inbox_url", "shared_inbox_url")
+
+    urls = set([actor["shared_inbox_url"] or actor["inbox_url"] for actor in values])
+    return sorted(urls)

api/funkwhale_api/federation/admin.py

@@ -4,23 +4,21 @@ from . import models
 from . import tasks
 
 
-def redeliver_inbox_items(modeladmin, request, queryset):
-    for id in set(
-        queryset.filter(activity__actor__user__isnull=False).values_list(
-            "activity", flat=True
-        )
-    ):
-        tasks.dispatch_outbox.delay(activity_id=id)
+def redeliver_deliveries(modeladmin, request, queryset):
+    queryset.update(is_delivered=False)
+    for delivery in queryset:
+        tasks.deliver_to_remote.delay(delivery_id=delivery.pk)
 
 
-redeliver_inbox_items.short_description = "Redeliver"
+redeliver_deliveries.short_description = "Redeliver"
 
 
 def redeliver_activities(modeladmin, request, queryset):
-    for id in set(
-        queryset.filter(actor__user__isnull=False).values_list("id", flat=True)
-    ):
-        tasks.dispatch_outbox.delay(activity_id=id)
+    for activity in queryset.select_related("actor__user"):
+        if activity.actor.is_local:
+            tasks.dispatch_outbox.delay(activity_id=activity.pk)
+        else:
+            tasks.dispatch_inbox.delay(activity_id=activity.pk)
 
 
 redeliver_activities.short_description = "Redeliver"
@@ -67,14 +65,22 @@ class LibraryFollowAdmin(admin.ModelAdmin):
 
 @admin.register(models.InboxItem)
 class InboxItemAdmin(admin.ModelAdmin):
+    list_display = ["actor", "activity", "type", "is_read"]
+    list_filter = ["type", "activity__type", "is_read"]
+    search_fields = ["actor__fid", "activity__fid"]
+    list_select_related = True
+
+
+@admin.register(models.Delivery)
+class DeliveryAdmin(admin.ModelAdmin):
     list_display = [
-        "actor",
+        "inbox_url",
         "activity",
-        "type",
-        "last_delivery_date",
-        "delivery_attempts",
+        "last_attempt_date",
+        "attempts",
+        "is_delivered",
     ]
-    list_filter = ["type"]
-    search_fields = ["actor__fid", "activity__fid"]
+    list_filter = ["activity__type", "is_delivered"]
+    search_fields = ["inbox_url"]
     list_select_related = True
-    actions = [redeliver_inbox_items]
+    actions = [redeliver_deliveries]

api/funkwhale_api/federation/api_serializers.py

@@ -16,7 +16,7 @@ class NestedLibraryFollowSerializer(serializers.ModelSerializer):
 
 class LibrarySerializer(serializers.ModelSerializer):
     actor = federation_serializers.APIActorSerializer()
-    files_count = serializers.SerializerMethodField()
+    uploads_count = serializers.SerializerMethodField()
     follow = serializers.SerializerMethodField()
 
     class Meta:
@@ -28,13 +28,13 @@ class LibrarySerializer(serializers.ModelSerializer):
             "name",
             "description",
             "creation_date",
-            "files_count",
+            "uploads_count",
             "privacy_level",
             "follow",
         ]
 
-    def get_files_count(self, o):
-        return max(getattr(o, "_files_count", 0), o.files_count)
+    def get_uploads_count(self, o):
+        return max(getattr(o, "_uploads_count", 0), o.uploads_count)
 
     def get_follow(self, o):
         try:

api/funkwhale_api/federation/api_views.py

@@ -87,7 +87,7 @@ class LibraryViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet):
         music_models.Library.objects.all()
         .order_by("-creation_date")
         .select_related("actor")
-        .annotate(_files_count=Count("files"))
+        .annotate(_uploads_count=Count("uploads"))
     )
     serializer_class = api_serializers.LibrarySerializer
     permission_classes = [permissions.IsAuthenticated]

api/funkwhale_api/federation/factories.py

@@ -76,6 +76,9 @@ class ActorFactory(factory.DjangoModelFactory):
     fid = factory.LazyAttribute(
         lambda o: "https://{}/users/{}".format(o.domain, o.preferred_username)
     )
+    followers_url = factory.LazyAttribute(
+        lambda o: "https://{}/users/{}followers".format(o.domain, o.preferred_username)
+    )
     inbox_url = factory.LazyAttribute(
         lambda o: "https://{}/users/{}/inbox".format(o.domain, o.preferred_username)
     )
@@ -134,19 +137,12 @@ class MusicLibraryFactory(factory.django.DjangoModelFactory):
     privacy_level = "me"
     name = factory.Faker("sentence")
     description = factory.Faker("sentence")
-    files_count = 0
+    uploads_count = 0
+    fid = factory.Faker("federation_url")
 
     class Meta:
         model = "music.Library"
 
-    @factory.post_generation
-    def fid(self, create, extracted, **kwargs):
-        if not create:
-            # Simple build, do nothing.
-            return
-
-        self.fid = extracted or self.get_federation_id()
-
     @factory.post_generation
     def followers_url(self, create, extracted, **kwargs):
         if not create:
@@ -160,7 +156,7 @@ class MusicLibraryFactory(factory.django.DjangoModelFactory):
 class LibraryScan(factory.django.DjangoModelFactory):
     library = factory.SubFactory(MusicLibraryFactory)
     actor = factory.SubFactory(ActorFactory)
-    total_files = factory.LazyAttribute(lambda o: o.library.files_count)
+    total_files = factory.LazyAttribute(lambda o: o.library.uploads_count)
 
     class Meta:
         model = "music.LibraryScan"
@@ -169,7 +165,7 @@ class LibraryScan(factory.django.DjangoModelFactory):
 @registry.register
 class ActivityFactory(factory.django.DjangoModelFactory):
     actor = factory.SubFactory(ActorFactory)
-    url = factory.Faker("url")
+    url = factory.Faker("federation_url")
     payload = factory.LazyFunction(lambda: {"type": "Create"})
 
     class Meta:
@@ -178,7 +174,7 @@ class ActivityFactory(factory.django.DjangoModelFactory):
 
 @registry.register
 class InboxItemFactory(factory.django.DjangoModelFactory):
-    actor = factory.SubFactory(ActorFactory)
+    actor = factory.SubFactory(ActorFactory, local=True)
     activity = factory.SubFactory(ActivityFactory)
     type = "to"
 
@@ -186,6 +182,15 @@ class InboxItemFactory(factory.django.DjangoModelFactory):
         model = "federation.InboxItem"
 
 
+@registry.register
+class DeliveryFactory(factory.django.DjangoModelFactory):
+    activity = factory.SubFactory(ActivityFactory)
+    inbox_url = factory.Faker("url")
+
+    class Meta:
+        model = "federation.Delivery"
+
+
 @registry.register
 class LibraryFollowFactory(factory.DjangoModelFactory):
     target = factory.SubFactory(MusicLibraryFactory)
@@ -269,9 +274,9 @@ class AudioMetadataFactory(factory.Factory):
 @registry.register(name="federation.Audio")
 class AudioFactory(factory.Factory):
     type = "Audio"
-    id = factory.Faker("url")
+    id = factory.Faker("federation_url")
     published = factory.LazyFunction(lambda: timezone.now().isoformat())
-    actor = factory.Faker("url")
+    actor = factory.Faker("federation_url")
     url = factory.SubFactory(LinkFactory, audio=True)
     metadata = factory.SubFactory(LibraryTrackMetadataFactory)
 

api/funkwhale_api/federation/library.py

@@ -108,7 +108,7 @@ def get_library_page(library, page_url, actor):
     )
     serializer = serializers.CollectionPageSerializer(
         data=response.json(),
-        context={"library": library, "item_serializer": serializers.AudioSerializer},
+        context={"library": library, "item_serializer": serializers.UploadSerializer},
     )
     serializer.is_valid(raise_exception=True)
     return serializer.validated_data

api/funkwhale_api/federation/migrations/0012_auto_20180920_1803.py

+# Generated by Django 2.0.8 on 2018-09-20 18:03
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('federation', '0011_auto_20180910_1902'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Delivery',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('is_delivered', models.BooleanField(default=False)),
+                ('last_attempt_date', models.DateTimeField(blank=True, null=True)),
+                ('attempts', models.PositiveIntegerField(default=0)),
+                ('inbox_url', models.URLField(max_length=500)),
+                ('activity', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='deliveries', to='federation.Activity')),
+            ],
+        ),
+        migrations.RemoveField(
+            model_name='inboxitem',
+            name='delivery_attempts',
+        ),
+        migrations.RemoveField(
+            model_name='inboxitem',
+            name='is_delivered',
+        ),
+        migrations.RemoveField(
+            model_name='inboxitem',
+            name='last_delivery_date',
+        ),
+    ]

api/funkwhale_api/federation/models.py

@@ -48,8 +48,8 @@ class ActorQuerySet(models.QuerySet):
             qs = qs.annotate(
                 **{
                     "_usage_{}".format(s): models.Sum(
-                        "libraries__files__size",
-                        filter=models.Q(libraries__files__import_status=s),
+                        "libraries__uploads__size",
+                        filter=models.Q(libraries__uploads__import_status=s),
                     )
                 }
             )
@@ -72,8 +72,8 @@ class Actor(models.Model):
     domain = models.CharField(max_length=1000)
     summary = models.CharField(max_length=500, null=True, blank=True)
     preferred_username = models.CharField(max_length=200, null=True, blank=True)
-    public_key = models.CharField(max_length=5000, null=True, blank=True)
-    private_key = models.CharField(max_length=5000, null=True, blank=True)
+    public_key = models.TextField(max_length=5000, null=True, blank=True)
+    private_key = models.TextField(max_length=5000, null=True, blank=True)
     creation_date = models.DateTimeField(default=timezone.now)
     last_fetch_date = models.DateTimeField(default=timezone.now)
     manually_approves_followers = models.NullBooleanField(default=None)
@@ -159,25 +159,34 @@ class Actor(models.Model):
         return data
 
 
-class InboxItemQuerySet(models.QuerySet):
-    def local(self, include=True):
-        return self.exclude(actor__user__isnull=include)
-
-
 class InboxItem(models.Model):
+    """
+    Store activities binding to local actors, with read/unread status.
+    """
+
     actor = models.ForeignKey(
         Actor, related_name="inbox_items", on_delete=models.CASCADE
     )
     activity = models.ForeignKey(
         "Activity", related_name="inbox_items", on_delete=models.CASCADE
     )
-    is_delivered = models.BooleanField(default=False)
     type = models.CharField(max_length=10, choices=[("to", "to"), ("cc", "cc")])
-    last_delivery_date = models.DateTimeField(null=True, blank=True)
-    delivery_attempts = models.PositiveIntegerField(default=0)
     is_read = models.BooleanField(default=False)
 
-    objects = InboxItemQuerySet.as_manager()
+
+class Delivery(models.Model):
+    """
+    Store deliveries attempt to remote inboxes
+    """
+
+    is_delivered = models.BooleanField(default=False)
+    last_attempt_date = models.DateTimeField(null=True, blank=True)
+    attempts = models.PositiveIntegerField(default=0)
+    inbox_url = models.URLField(max_length=500)
+
+    activity = models.ForeignKey(
+        "Activity", related_name="deliveries", on_delete=models.CASCADE
+    )
 
 
 class Activity(models.Model):

api/funkwhale_api/federation/routes.py

 import logging
 
+from funkwhale_api.music import models as music_models
+
 from . import activity
 from . import serializers
 
@@ -90,3 +92,109 @@ def outbox_follow(context):
         "object": follow.target,
         "related_object": follow,
     }
+
+
+@outbox.register({"type": "Create", "object.type": "Audio"})
+def outbox_create_audio(context):
+    upload = context["upload"]
+    serializer = serializers.ActivitySerializer(
+        {
+            "type": "Create",
+            "actor": upload.library.actor.fid,
+            "object": serializers.UploadSerializer(upload).data,
+        }
+    )
+    yield {
+        "type": "Create",
+        "actor": upload.library.actor,
+        "payload": with_recipients(
+            serializer.data, to=[{"type": "followers", "target": upload.library}]
+        ),
+        "object": upload,
+        "target": upload.library,
+    }
+
+
+@inbox.register({"type": "Create", "object.type": "Audio"})
+def inbox_create_audio(payload, context):
+    serializer = serializers.UploadSerializer(
+        data=payload["object"],
+        context={"activity": context.get("activity"), "actor": context["actor"]},
+    )
+
+    if not serializer.is_valid(raise_exception=context.get("raise_exception", False)):
+        logger.warn("Discarding invalid audio create")
+        return
+
+    upload = serializer.save()
+
+    return {"object": upload, "target": upload.library}
+
+
+@inbox.register({"type": "Delete", "object.type": "Library"})
+def inbox_delete_library(payload, context):
+    actor = context["actor"]
+    library_id = payload["object"].get("id")
+    if not library_id:
+        logger.debug("Discarding deletion of empty library")
+        return
+
+    try:
+        library = actor.libraries.get(fid=library_id)
+    except music_models.Library.DoesNotExist:
+        logger.debug("Discarding deletion of unkwnown library %s", library_id)
+        return
+
+    library.delete()
+
+
+@outbox.register({"type": "Delete", "object.type": "Library"})
+def outbox_delete_library(context):
+    library = context["library"]
+    serializer = serializers.ActivitySerializer(
+        {"type": "Delete", "object": {"type": "Library", "id": library.fid}}
+    )
+    yield {
+        "type": "Delete",
+        "actor": library.actor,
+        "payload": with_recipients(
+            serializer.data, to=[{"type": "followers", "target": library}]
+        ),
+    }
+
+
+@inbox.register({"type": "Delete", "object.type": "Audio"})
+def inbox_delete_audio(payload, context):
+    actor = context["actor"]
+    try:
+        upload_fids = [i for i in payload["object"]["id"]]
+    except TypeError:
+        # we did not receive a list of Ids, so we can probably use the value directly
+        upload_fids = [payload["object"]["id"]]
+
+    candidates = music_models.Upload.objects.filter(
+        library__actor=actor, fid__in=upload_fids
+    )
+
+    total = candidates.count()
+    logger.info("Deleting %s uploads with ids %s", total, upload_fids)
+    candidates.delete()
+
+
+@outbox.register({"type": "Delete", "object.type": "Audio"})
+def outbox_delete_audio(context):
+    uploads = context["uploads"]
+    library = uploads[0].library
+    serializer = serializers.ActivitySerializer(
+        {
+            "type": "Delete",
+            "object": {"type": "Audio", "id": [u.get_federation_id() for u in uploads]},
+        }
+    )
+    yield {
+        "type": "Delete",
+        "actor": library.actor,
+        "payload": with_recipients(
+            serializer.data, to=[{"type": "followers", "target": library}]
+        ),
+    }

api/funkwhale_api/federation/serializers.py

@@ -4,6 +4,7 @@ import urllib.parse
 
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.paginator import Paginator
+from django.db.models import F, Q
 from rest_framework import serializers
 
 from funkwhale_api.common import utils as funkwhale_utils
@@ -29,7 +30,7 @@ class ActorSerializer(serializers.Serializer):
     manuallyApprovesFollowers = serializers.NullBooleanField(required=False)
     name = serializers.CharField(required=False, max_length=200)
     summary = serializers.CharField(max_length=None, required=False)
-    followers = serializers.URLField(max_length=500, required=False, allow_null=True)
+    followers = serializers.URLField(max_length=500)
     following = serializers.URLField(max_length=500, required=False, allow_null=True)
     publicKey = serializers.JSONField(required=False)
 
@@ -174,30 +175,6 @@ class BaseActivitySerializer(serializers.Serializer):
                 "We cannot handle an activity with no recipient"
             )
 
-        matching = models.Actor.objects.filter(fid__in=to + cc)
-        if self.context.get("local_recipients", False):
-            matching = matching.local()
-
-        if not len(matching):
-            raise serializers.ValidationError("No matching recipients found")
-
-        actors_by_fid = {a.fid: a for a in matching}
-
-        def match(recipients, actors):
-            for r in recipients:
-                if r == activity.PUBLIC_ADDRESS:
-                    yield r
-                else:
-                    try:
-                        yield actors[r]
-                    except KeyError:
-                        pass
-
-        return {
-            "to": list(match(to, actors_by_fid)),
-            "cc": list(match(cc, actors_by_fid)),
-        }
-
 
 class FollowSerializer(serializers.Serializer):
     id = serializers.URLField(max_length=500)
@@ -422,7 +399,8 @@ class ActivitySerializer(serializers.Serializer):
     actor = serializers.URLField(max_length=500)
     id = serializers.URLField(max_length=500, required=False)
     type = serializers.ChoiceField(choices=[(c, c) for c in activity.ACTIVITY_TYPES])
-    object = serializers.JSONField()
+    object = serializers.JSONField(required=False)
+    target = serializers.JSONField(required=False)
 
     def validate_object(self, value):
         try:
@@ -528,6 +506,7 @@ class LibrarySerializer(PaginatedCollectionSerializer):
     type = serializers.ChoiceField(choices=["Library"])
     name = serializers.CharField()
     summary = serializers.CharField(allow_blank=True, allow_null=True, required=False)
+    followers = serializers.URLField(max_length=500)
     audience = serializers.ChoiceField(
         choices=["", None, "https://www.w3.org/ns/activitystreams#Public"],
         required=False,
@@ -542,7 +521,7 @@ class LibrarySerializer(PaginatedCollectionSerializer):
             "summary": library.description,
             "page_size": 100,
             "actor": library.actor,
-            "items": library.files.filter(import_status="finished"),
+            "items": library.uploads.filter(import_status="finished"),
             "type": "Library",
         }
         r = super().to_representation(conf)
@@ -551,6 +530,7 @@ class LibrarySerializer(PaginatedCollectionSerializer):
             if library.privacy_level == "public"
             else ""
         )
+        r["followers"] = library.followers_url
         return r
 
     def create(self, validated_data):
@@ -563,9 +543,10 @@ class LibrarySerializer(PaginatedCollectionSerializer):
             fid=validated_data["id"],
             actor=actor,
             defaults={
-                "files_count": validated_data["totalItems"],
+                "uploads_count": validated_data["totalItems"],
                 "name": validated_data["name"],
                 "description": validated_data["summary"],
+                "followers_url": validated_data["followers"],
                 "privacy_level": "everyone"
                 if validated_data["audience"]
                 == "https://www.w3.org/ns/activitystreams#Public"
@@ -639,43 +620,157 @@ class CollectionPageSerializer(serializers.Serializer):
         return d
 
 
-class ArtistMetadataSerializer(serializers.Serializer):
-    musicbrainz_id = serializers.UUIDField(required=False, allow_null=True)
-    name = serializers.CharField()
+class MusicEntitySerializer(serializers.Serializer):
+    id = serializers.URLField(max_length=500)
+    published = serializers.DateTimeField()
+    musicbrainzId = serializers.UUIDField(allow_null=True, required=False)
+    name = serializers.CharField(max_length=1000)
+
+    def create(self, validated_data):
+        mbid = validated_data.get("musicbrainzId")
+        candidates = self.model.objects.filter(
+            Q(mbid=mbid) | Q(fid=validated_data["id"])
+        ).order_by(F("fid").desc(nulls_last=True))
+
+        existing = candidates.first()
+        if existing:
+            return existing
+
+        # nothing matching in our database, let's create a new object
+        return self.model.objects.create(**self.get_create_data(validated_data))
+
+    def get_create_data(self, validated_data):
+        return {
+            "mbid": validated_data.get("musicbrainzId"),
+            "fid": validated_data["id"],
+            "name": validated_data["name"],
+            "creation_date": validated_data["published"],
+            "from_activity": self.context.get("activity"),
+        }
+
+
+class ArtistSerializer(MusicEntitySerializer):
+    model = music_models.Artist
+
+    def to_representation(self, instance):
+        d = {
+            "type": "Artist",
+            "id": instance.fid,
+            "name": instance.name,
+            "published": instance.creation_date.isoformat(),
+            "musicbrainzId": str(instance.mbid) if instance.mbid else None,
+        }
+
+        if self.context.get("include_ap_context", self.parent is None):
+            d["@context"] = AP_CONTEXT
+        return d
+
+
+class AlbumSerializer(MusicEntitySerializer):
+    model = music_models.Album
+    released = serializers.DateField(allow_null=True, required=False)
+    artists = serializers.ListField(child=ArtistSerializer(), min_length=1)
+
+    def to_representation(self, instance):
+        d = {
+            "type": "Album",
+            "id": instance.fid,
+            "name": instance.title,
+            "published": instance.creation_date.isoformat(),
+            "musicbrainzId": str(instance.mbid) if instance.mbid else None,
+            "released": instance.release_date.isoformat()
+            if instance.release_date
+            else None,
+            "artists": [
+                ArtistSerializer(
+                    instance.artist, context={"include_ap_context": False}
+                ).data
+            ],
+        }
+        if instance.cover:
+            d["cover"] = {"type": "Image", "url": utils.full_url(instance.cover.url)}
+        if self.context.get("include_ap_context", self.parent is None):
+            d["@context"] = AP_CONTEXT
+        return d
 
+    def get_create_data(self, validated_data):
+        artist_data = validated_data["artists"][0]
+        artist = ArtistSerializer(
+            context={"activity": self.context.get("activity")}
+        ).create(artist_data)
+
+        return {
+            "mbid": validated_data.get("musicbrainzId"),
+            "fid": validated_data["id"],
+            "title": validated_data["name"],
+            "creation_date": validated_data["published"],
+            "artist": artist,
+            "release_date": validated_data.get("released"),
+            "from_activity": self.context.get("activity"),
+        }
 
-class ReleaseMetadataSerializer(serializers.Serializer):
-    musicbrainz_id = serializers.UUIDField(required=False, allow_null=True)
-    title = serializers.CharField()
 
+class TrackSerializer(MusicEntitySerializer):
+    model = music_models.Track
+    position = serializers.IntegerField(min_value=0, allow_null=True, required=False)
+    artists = serializers.ListField(child=ArtistSerializer(), min_length=1)
+    album = AlbumSerializer()
+
+    def to_representation(self, instance):
+        d = {
+            "type": "Track",
+            "id": instance.fid,
+            "name": instance.title,
+            "published": instance.creation_date.isoformat(),
+            "musicbrainzId": str(instance.mbid) if instance.mbid else None,
+            "position": instance.position,
+            "artists": [
+                ArtistSerializer(
+                    instance.artist, context={"include_ap_context": False}
+                ).data
+            ],
+            "album": AlbumSerializer(
+                instance.album, context={"include_ap_context": False}
+            ).data,
+        }
 
-class RecordingMetadataSerializer(serializers.Serializer):
-    musicbrainz_id = serializers.UUIDField(required=False, allow_null=True)
-    title = serializers.CharField()
+        if self.context.get("include_ap_context", self.parent is None):
+            d["@context"] = AP_CONTEXT
+        return d
 
+    def get_create_data(self, validated_data):
+        artist_data = validated_data["artists"][0]
+        artist = ArtistSerializer(
+            context={"activity": self.context.get("activity")}
+        ).create(artist_data)
+        album = AlbumSerializer(
+            context={"activity": self.context.get("activity")}
+        ).create(validated_data["album"])
 
-class AudioMetadataSerializer(serializers.Serializer):
-    artist = ArtistMetadataSerializer()
-    release = ReleaseMetadataSerializer()
-    recording = RecordingMetadataSerializer()
-    bitrate = serializers.IntegerField(required=False, allow_null=True, min_value=0)
-    size = serializers.IntegerField(required=False, allow_null=True, min_value=0)
-    length = serializers.IntegerField(required=False, allow_null=True, min_value=0)
+        return {
+            "mbid": validated_data.get("musicbrainzId"),
+            "fid": validated_data["id"],
+            "title": validated_data["name"],
+            "position": validated_data.get("position"),
+            "creation_date": validated_data["published"],
+            "artist": artist,
+            "album": album,
+            "from_activity": self.context.get("activity"),
+        }
 
 
-class AudioSerializer(serializers.Serializer):
-    type = serializers.CharField()
+class UploadSerializer(serializers.Serializer):
+    type = serializers.ChoiceField(choices=["Audio"])
     id = serializers.URLField(max_length=500)
     library = serializers.URLField(max_length=500)
     url = serializers.JSONField()
     published = serializers.DateTimeField()
-    updated = serializers.DateTimeField(required=False)
-    metadata = AudioMetadataSerializer()
+    updated = serializers.DateTimeField(required=False, allow_null=True)
+    bitrate = serializers.IntegerField(min_value=0)
+    size = serializers.IntegerField(min_value=0)
+    duration = serializers.IntegerField(min_value=0)
 
-    def validate_type(self, v):
-        if v != "Audio":
-            raise serializers.ValidationError("Invalid type for audio")
-        return v
+    track = TrackSerializer(required=True)
 
     def validate_url(self, v):
         try:
@@ -699,61 +794,64 @@ class AudioSerializer(serializers.Serializer):
             if lb.fid != v:
                 raise serializers.ValidationError("Invalid library")
             return lb
+
+        actor = self.context.get("actor")
+        kwargs = {}
+        if actor:
+            kwargs["actor"] = actor
         try:
-            return music_models.Library.objects.get(fid=v)
+            return music_models.Library.objects.get(fid=v, **kwargs)
         except music_models.Library.DoesNotExist:
             raise serializers.ValidationError("Invalid library")
 
     def create(self, validated_data):
-        defaults = {
+        try:
+            return music_models.Upload.objects.get(fid=validated_data["id"])
+        except music_models.Upload.DoesNotExist:
+            pass
+
+        track = TrackSerializer(
+            context={"activity": self.context.get("activity")}
+        ).create(validated_data["track"])
+
+        data = {
+            "fid": validated_data["id"],
             "mimetype": validated_data["url"]["mediaType"],
             "source": validated_data["url"]["href"],
             "creation_date": validated_data["published"],
             "modification_date": validated_data.get("updated"),
-            "metadata": self.initial_data,
+            "track": track,
+            "duration": validated_data["duration"],
+            "size": validated_data["size"],
+            "bitrate": validated_data["bitrate"],
+            "library": validated_data["library"],
+            "from_activity": self.context.get("activity"),
+            "import_status": "finished",
         }
-        tf, created = validated_data["library"].files.update_or_create(
-            fid=validated_data["id"], defaults=defaults
-        )
-        return tf
+        return music_models.Upload.objects.create(**data)
 
     def to_representation(self, instance):
         track = instance.track
-        album = instance.track.album
-        artist = instance.track.artist
         d = {
             "type": "Audio",
             "id": instance.get_federation_id(),
-            "library": instance.library.get_federation_id(),
-            "name": instance.track.full_name,
+            "library": instance.library.fid,
+            "name": track.full_name,
             "published": instance.creation_date.isoformat(),
-            "metadata": {
-                "artist": {
-                    "musicbrainz_id": str(artist.mbid) if artist.mbid else None,
-                    "name": artist.name,
-                },
-                "release": {
-                    "musicbrainz_id": str(album.mbid) if album.mbid else None,
-                    "title": album.title,
-                },
-                "recording": {
-                    "musicbrainz_id": str(track.mbid) if track.mbid else None,
-                    "title": track.title,
-                },
-                "bitrate": instance.bitrate,
-                "size": instance.size,
-                "length": instance.duration,
-            },
+            "bitrate": instance.bitrate,
+            "size": instance.size,
+            "duration": instance.duration,
             "url": {
                 "href": utils.full_url(instance.listen_url),
                 "type": "Link",
                 "mediaType": instance.mimetype,
             },
+            "track": TrackSerializer(track, context={"include_ap_context": False}).data,
         }
         if instance.modification_date:
             d["updated"] = instance.modification_date.isoformat()
 
-        if self.context.get("include_ap_context", True):
+        if self.context.get("include_ap_context", self.parent is None):
             d["@context"] = AP_CONTEXT
         return d
 

api/funkwhale_api/federation/tasks.py

@@ -27,7 +27,7 @@ def clean_music_cache():
     limit = timezone.now() - datetime.timedelta(minutes=delay)
 
     candidates = (
-        music_models.TrackFile.objects.filter(
+        music_models.Upload.objects.filter(
             Q(audio_file__isnull=False)
             & (Q(accessed_date__lt=limit) | Q(accessed_date=None))
         )
@@ -36,13 +36,13 @@ def clean_music_cache():
         .only("audio_file", "id")
         .order_by("id")
     )
-    for tf in candidates:
-        tf.audio_file.delete()
+    for upload in candidates:
+        upload.audio_file.delete()
 
     # we also delete orphaned files, if any
     storage = models.LibraryTrack._meta.get_field("audio_file").storage
     files = get_files(storage, "federation_cache/tracks")
-    existing = music_models.TrackFile.objects.filter(audio_file__in=files)
+    existing = music_models.Upload.objects.filter(audio_file__in=files)
     missing = set(files) - set(existing.values_list("audio_file", flat=True))
     for m in missing:
         storage.delete(m)
@@ -70,61 +70,30 @@ def dispatch_inbox(activity):
     creation, etc.)
     """
 
-    try:
-        routes.inbox.dispatch(
-            activity.payload,
-            context={
-                "activity": activity,
-                "actor": activity.actor,
-                "inbox_items": (
-                    activity.inbox_items.local()
-                    .select_related()
-                    .select_related("actor__user")
-                    .prefetch_related("activity__object", "activity__target")
-                ),
-            },
-        )
-    except Exception:
-        activity.inbox_items.local().update(
-            delivery_attempts=F("delivery_attempts") + 1,
-            last_delivery_date=timezone.now(),
-        )
-        raise
-    else:
-        activity.inbox_items.local().update(
-            delivery_attempts=F("delivery_attempts") + 1,
-            last_delivery_date=timezone.now(),
-            is_delivered=True,
-        )
+    routes.inbox.dispatch(
+        activity.payload,
+        context={
+            "activity": activity,
+            "actor": activity.actor,
+            "inbox_items": activity.inbox_items.filter(is_read=False),
+        },
+    )
 
 
 @celery.app.task(name="federation.dispatch_outbox")
 @celery.require_instance(models.Activity.objects.select_related(), "activity")
 def dispatch_outbox(activity):
     """
-    Deliver a local activity to its recipients
+    Deliver a local activity to its recipients, both locally and remotely
     """
-    inbox_items = activity.inbox_items.all().select_related("actor")
-    local_recipients_items = [ii for ii in inbox_items if ii.actor.is_local]
-    if local_recipients_items:
-        dispatch_inbox.delay(activity_id=activity.pk)
-    remote_recipients_items = [ii for ii in inbox_items if not ii.actor.is_local]
+    inbox_items = activity.inbox_items.filter(is_read=False).select_related()
+    deliveries = activity.deliveries.filter(is_delivered=False)
 
-    shared_inbox_urls = {
-        ii.actor.shared_inbox_url
-        for ii in remote_recipients_items
-        if ii.actor.shared_inbox_url
-    }
-    inbox_urls = {
-        ii.actor.inbox_url
-        for ii in remote_recipients_items
-        if not ii.actor.shared_inbox_url
-    }
-    for url in shared_inbox_urls:
-        deliver_to_remote_inbox.delay(activity_id=activity.pk, shared_inbox_url=url)
+    if inbox_items.exists():
+        dispatch_inbox.delay(activity_id=activity.pk)
 
-    for url in inbox_urls:
-        deliver_to_remote_inbox.delay(activity_id=activity.pk, inbox_url=url)
+    for id in deliveries.values_list("pk", flat=True):
+        deliver_to_remote.delay(delivery_id=id)
 
 
 @celery.app.task(
@@ -133,22 +102,21 @@ def dispatch_outbox(activity):
     retry_backoff=30,
     max_retries=5,
 )
-@celery.require_instance(models.Activity.objects.select_related(), "activity")
-def deliver_to_remote_inbox(activity, inbox_url=None, shared_inbox_url=None):
-    url = inbox_url or shared_inbox_url
-    actor = activity.actor
-    inbox_items = activity.inbox_items.filter(is_delivered=False)
-    if inbox_url:
-        inbox_items = inbox_items.filter(actor__inbox_url=inbox_url)
-    else:
-        inbox_items = inbox_items.filter(actor__shared_inbox_url=shared_inbox_url)
-    logger.info("Preparing activity delivery to %s", url)
+@celery.require_instance(
+    models.Delivery.objects.filter(is_delivered=False).select_related(
+        "activity__actor"
+    ),
+    "delivery",
+)
+def deliver_to_remote(delivery):
+    actor = delivery.activity.actor
+    logger.info("Preparing activity delivery to %s", delivery.inbox_url)
     auth = signing.get_auth(actor.private_key, actor.private_key_id)
     try:
         response = session.get_session().post(
             auth=auth,
-            json=activity.payload,
-            url=url,
+            json=delivery.activity.payload,
+            url=delivery.inbox_url,
             timeout=5,
             verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
             headers={"Content-Type": "application/activity+json"},
@@ -156,10 +124,12 @@ def deliver_to_remote_inbox(activity, inbox_url=None, shared_inbox_url=None):
         logger.debug("Remote answered with %s", response.status_code)
         response.raise_for_status()
     except Exception:
-        inbox_items.update(
-            last_delivery_date=timezone.now(),
-            delivery_attempts=F("delivery_attempts") + 1,
-        )
+        delivery.last_attempt_date = timezone.now()
+        delivery.attempts = F("attempts") + 1
+        delivery.save(update_fields=["last_attempt_date", "attempts"])
         raise
     else:
-        inbox_items.update(last_delivery_date=timezone.now(), is_delivered=True)
+        delivery.last_attempt_date = timezone.now()
+        delivery.attempts = F("attempts") + 1
+        delivery.is_delivered = True
+        delivery.save(update_fields=["last_attempt_date", "attempts", "is_delivered"])

api/funkwhale_api/federation/urls.py

@@ -8,10 +8,15 @@ music_router = routers.SimpleRouter(trailing_slash=False)
 router.register(
     r"federation/instance/actors", views.InstanceActorViewSet, "instance-actors"
 )
+router.register(r"federation/shared", views.SharedViewSet, "shared")
 router.register(r"federation/actors", views.ActorViewSet, "actors")
 router.register(r".well-known", views.WellKnownViewSet, "well-known")
 
 music_router.register(r"libraries", views.MusicLibraryViewSet, "libraries")
+music_router.register(r"uploads", views.MusicUploadViewSet, "uploads")
+music_router.register(r"artists", views.MusicArtistViewSet, "artists")
+music_router.register(r"albums", views.MusicAlbumViewSet, "albums")
+music_router.register(r"tracks", views.MusicTrackViewSet, "tracks")
 urlpatterns = router.urls + [
     url("federation/music/", include((music_router.urls, "music"), namespace="music"))
 ]

api/funkwhale_api/federation/views.py

@@ -27,6 +27,22 @@ class FederationMixin(object):
         return super().dispatch(request, *args, **kwargs)
 
 
+class SharedViewSet(FederationMixin, viewsets.GenericViewSet):
+    permission_classes = []
+    authentication_classes = [authentication.SignatureAuthentication]
+    renderer_classes = [renderers.ActivityPubRenderer]
+
+    @list_route(methods=["post"])
+    def inbox(self, request, *args, **kwargs):
+        if request.method.lower() == "post" and request.actor is None:
+            raise exceptions.AuthenticationFailed(
+                "You need a valid signature to send an activity"
+            )
+        if request.method.lower() == "post":
+            activity.receive(activity=request.data, on_behalf_of=request.actor)
+        return response.Response({}, status=200)
+
+
 class ActorViewSet(FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet):
     lookup_field = "preferred_username"
     authentication_classes = [authentication.SignatureAuthentication]
@@ -49,6 +65,18 @@ class ActorViewSet(FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericV
     def outbox(self, request, *args, **kwargs):
         return response.Response({}, status=200)
 
+    @detail_route(methods=["get"])
+    def followers(self, request, *args, **kwargs):
+        self.get_object()
+        # XXX to implement
+        return response.Response({})
+
+    @detail_route(methods=["get"])
+    def following(self, request, *args, **kwargs):
+        self.get_object()
+        # XXX to implement
+        return response.Response({})
+
 
 class InstanceActorViewSet(FederationMixin, viewsets.GenericViewSet):
     lookup_field = "actor"
@@ -175,8 +203,8 @@ class MusicLibraryViewSet(
             "actor": lb.actor,
             "name": lb.name,
             "summary": lb.description,
-            "items": lb.files.order_by("-creation_date"),
-            "item_serializer": serializers.AudioSerializer,
+            "items": lb.uploads.order_by("-creation_date"),
+            "item_serializer": serializers.UploadSerializer,
         }
         page = request.GET.get("page")
         if page is None:
@@ -204,3 +232,49 @@ class MusicLibraryViewSet(
                 return response.Response(status=404)
 
         return response.Response(data)
+
+    @detail_route(methods=["get"])
+    def followers(self, request, *args, **kwargs):
+        self.get_object()
+        # XXX Implement this
+        return response.Response({})
+
+
+class MusicUploadViewSet(
+    FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet
+):
+    authentication_classes = [authentication.SignatureAuthentication]
+    permission_classes = []
+    renderer_classes = [renderers.ActivityPubRenderer]
+    queryset = music_models.Upload.objects.none()
+    lookup_field = "uuid"
+
+
+class MusicArtistViewSet(
+    FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet
+):
+    authentication_classes = [authentication.SignatureAuthentication]
+    permission_classes = []
+    renderer_classes = [renderers.ActivityPubRenderer]
+    queryset = music_models.Artist.objects.none()
+    lookup_field = "uuid"
+
+
+class MusicAlbumViewSet(
+    FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet
+):
+    authentication_classes = [authentication.SignatureAuthentication]
+    permission_classes = []
+    renderer_classes = [renderers.ActivityPubRenderer]
+    queryset = music_models.Album.objects.none()
+    lookup_field = "uuid"
+
+
+class MusicTrackViewSet(
+    FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet
+):
+    authentication_classes = [authentication.SignatureAuthentication]
+    permission_classes = []
+    renderer_classes = [renderers.ActivityPubRenderer]
+    queryset = music_models.Track.objects.none()
+    lookup_field = "uuid"

api/funkwhale_api/instance/stats.py

@@ -43,7 +43,7 @@ def get_artists():
 
 
 def get_music_duration():
-    seconds = models.TrackFile.objects.aggregate(d=Sum("duration"))["d"]
+    seconds = models.Upload.objects.aggregate(d=Sum("duration"))["d"]
     if seconds:
         return seconds / 3600
     return 0