See #170: Funkwhale federation

api/funkwhale_api/audio/models.py

@@ -107,6 +107,4 @@ def generate_actor(username, **kwargs):
 @receiver(post_delete, sender=Channel)
 def delete_channel_related_objs(instance, **kwargs):
     instance.library.delete()
-    if instance.actor != instance.attributed_to:
-        instance.actor.delete()
     instance.artist.delete()

api/funkwhale_api/audio/views.py

@@ -13,10 +13,12 @@ from django.utils import timezone
 from funkwhale_api.common import locales
 from funkwhale_api.common import permissions
 from funkwhale_api.common import preferences
+from funkwhale_api.common import utils as common_utils
 from funkwhale_api.common.mixins import MultipleLookupDetailMixin
 from funkwhale_api.federation import actors
 from funkwhale_api.federation import models as federation_models
 from funkwhale_api.federation import routes
+from funkwhale_api.federation import tasks as federation_tasks
 from funkwhale_api.federation import utils as federation_utils
 from funkwhale_api.music import models as music_models
 from funkwhale_api.music import views as music_views
@@ -128,6 +130,8 @@ class ChannelViewSet(
         )
         # prefetch stuff
         subscription = SubscriptionsViewSet.queryset.get(pk=subscription.pk)
+        if not object.actor.is_local:
+            routes.outbox.dispatch({"type": "Follow"}, context={"follow": subscription})
 
         data = serializers.SubscriptionSerializer(subscription).data
         return response.Response(data, status=201)
@@ -139,7 +143,15 @@ class ChannelViewSet(
     )
     def unsubscribe(self, request, *args, **kwargs):
         object = self.get_object()
-        request.user.actor.emitted_follows.filter(target=object.actor).delete()
+        follow_qs = request.user.actor.emitted_follows.filter(target=object.actor)
+        follow = follow_qs.first()
+        if follow:
+            if not object.actor.is_local:
+                routes.outbox.dispatch(
+                    {"type": "Undo", "object": {"type": "Follow"}},
+                    context={"follow": follow},
+                )
+            follow_qs.delete()
         return response.Response(status=204)
 
     @decorators.action(
@@ -248,11 +260,10 @@ class ChannelViewSet(
 
     @transaction.atomic
     def perform_destroy(self, instance):
-        routes.outbox.dispatch(
-            {"type": "Delete", "object": {"type": instance.actor.type}},
-            context={"actor": instance.actor},
-        )
         instance.__class__.objects.filter(pk=instance.pk).delete()
+        common_utils.on_commit(
+            federation_tasks.remove_actor.delay, actor_id=instance.actor.pk
+        )
 
 
 class SubscriptionsViewSet(

api/funkwhale_api/federation/api_serializers.py

@@ -7,6 +7,7 @@ from django.utils import timezone
 
 from rest_framework import serializers
 
+from funkwhale_api.audio import models as audio_models
 from funkwhale_api.common import fields as common_fields
 from funkwhale_api.common import serializers as common_serializers
 from funkwhale_api.music import models as music_models
@@ -171,6 +172,7 @@ FETCH_OBJECT_CONFIG = {
     "library": {"queryset": music_models.Library.objects.all(), "id_attr": "uuid"},
     "upload": {"queryset": music_models.Upload.objects.all(), "id_attr": "uuid"},
     "account": {"queryset": models.Actor.objects.all(), "id_attr": "full_username"},
+    "channel": {"queryset": audio_models.Channel.objects.all(), "id_attr": "uuid"},
 }
 FETCH_OBJECT_FIELD = common_fields.GenericRelation(FETCH_OBJECT_CONFIG)
 

api/funkwhale_api/federation/contexts.py

+from . import schema_org
+
 CONTEXTS = [
     {
         "shortId": "LDP",
@@ -218,6 +220,12 @@ CONTEXTS = [
             }
         },
     },
+    {
+        "shortId": "SC",
+        "contextUrl": None,
+        "documentUrl": "http://schema.org",
+        "document": {"@context": schema_org.CONTEXT},
+    },
     {
         "shortId": "SEC",
         "contextUrl": None,
@@ -280,6 +288,7 @@ CONTEXTS = [
                 "type": "@type",
                 "as": "https://www.w3.org/ns/activitystreams#",
                 "fw": "https://funkwhale.audio/ns#",
+                "schema": "http://schema.org#",
                 "xsd": "http://www.w3.org/2001/XMLSchema#",
                 "Album": "fw:Album",
                 "Track": "fw:Track",
@@ -298,6 +307,8 @@ CONTEXTS = [
                 "musicbrainzId": "fw:musicbrainzId",
                 "license": {"@id": "fw:license", "@type": "@id"},
                 "copyright": "fw:copyright",
+                "category": "schema:category",
+                "language": "schema:inLanguage",
             }
         },
     },
@@ -364,4 +375,5 @@ AS = NS(CONTEXTS_BY_ID["AS"])
 LDP = NS(CONTEXTS_BY_ID["LDP"])
 SEC = NS(CONTEXTS_BY_ID["SEC"])
 FW = NS(CONTEXTS_BY_ID["FW"])
+SC = NS(CONTEXTS_BY_ID["SC"])
 LITEPUB = NS(CONTEXTS_BY_ID["LITEPUB"])

api/funkwhale_api/federation/routes.py

 import logging
 
+from django.db.models import Q
+
 from funkwhale_api.music import models as music_models
 
 from . import activity
@@ -158,18 +160,26 @@ def outbox_create_audio(context):
 
 @inbox.register({"type": "Create", "object.type": "Audio"})
 def inbox_create_audio(payload, context):
-    serializer = serializers.UploadSerializer(
-        data=payload["object"],
-        context={"activity": context.get("activity"), "actor": context["actor"]},
-    )
-
+    is_channel = "library" not in payload["object"]
+    if is_channel:
+        channel = context["actor"].get_channel()
+        serializer = serializers.ChannelUploadSerializer(
+            data=payload["object"], context={"channel": channel},
+        )
+    else:
+        serializer = serializers.UploadSerializer(
+            data=payload["object"],
+            context={"activity": context.get("activity"), "actor": context["actor"]},
+        )
     if not serializer.is_valid(raise_exception=context.get("raise_exception", False)):
         logger.warn("Discarding invalid audio create")
         return
 
     upload = serializer.save()
-
-    return {"object": upload, "target": upload.library}
+    if is_channel:
+        return {"object": upload, "target": channel}
+    else:
+        return {"object": upload, "target": upload.library}
 
 
 @inbox.register({"type": "Delete", "object.type": "Library"})
@@ -252,9 +262,10 @@ def inbox_delete_audio(payload, context):
         # we did not receive a list of Ids, so we can probably use the value directly
         upload_fids = [payload["object"]["id"]]
 
-    candidates = music_models.Upload.objects.filter(
-        library__actor=actor, fid__in=upload_fids
+    query = Q(fid__in=upload_fids) & (
+        Q(library__actor=actor) | Q(track__artist__channel__actor=actor)
     )
+    candidates = music_models.Upload.objects.filter(query)
 
     total = candidates.count()
     logger.info("Deleting %s uploads with ids %s", total, upload_fids)
@@ -483,3 +494,44 @@ def outbox_flag(context):
             to=[{"type": "actor_inbox", "actor": report.target_owner}],
         ),
     }
+
+
+@inbox.register({"type": "Delete", "object.type": "Album"})
+def inbox_delete_album(payload, context):
+    actor = context["actor"]
+    album_id = payload["object"].get("id")
+    if not album_id:
+        logger.debug("Discarding deletion of empty library")
+        return
+
+    query = Q(fid=album_id) & (Q(attributed_to=actor) | Q(artist__channel__actor=actor))
+    try:
+        album = music_models.Album.objects.get(query)
+    except music_models.Album.DoesNotExist:
+        logger.debug("Discarding deletion of unkwnown album %s", album_id)
+        return
+
+    album.delete()
+
+
+@outbox.register({"type": "Delete", "object.type": "Album"})
+def outbox_delete_album(context):
+    album = context["album"]
+    actor = (
+        album.artist.channel.actor
+        if album.artist.get_channel()
+        else album.attributed_to
+    )
+    actor = actor or actors.get_service_actor()
+    serializer = serializers.ActivitySerializer(
+        {"type": "Delete", "object": {"type": "Album", "id": album.fid}}
+    )
+
+    yield {
+        "type": "Delete",
+        "actor": actor,
+        "payload": with_recipients(
+            serializer.data,
+            to=[activity.PUBLIC_ADDRESS, {"type": "instances_with_followers"}],
+        ),
+    }

api/funkwhale_api/federation/tasks.py

@@ -7,11 +7,14 @@ import requests
 from django.conf import settings
 from django.db import transaction
 from django.db.models import Q, F
+from django.db.models.deletion import Collector
 from django.utils import timezone
 from dynamic_preferences.registries import global_preferences_registry
 from requests.exceptions import RequestException
 
+from funkwhale_api.audio import models as audio_models
 from funkwhale_api.common import preferences
+from funkwhale_api.common import models as common_models
 from funkwhale_api.common import session
 from funkwhale_api.common import utils as common_utils
 from funkwhale_api.moderation import mrf
@@ -254,8 +257,11 @@ def handle_purge_actors(ids, only=[]):
 
     # purge audio content
     if not only or "media" in only:
+        delete_qs(common_models.Attachment.objects.filter(actor__in=ids))
         delete_qs(models.LibraryFollow.objects.filter(actor_id__in=ids))
         delete_qs(models.Follow.objects.filter(target_id__in=ids))
+        delete_qs(audio_models.Channel.objects.filter(attributed_to__in=ids))
+        delete_qs(audio_models.Channel.objects.filter(actor__in=ids))
         delete_qs(music_models.Upload.objects.filter(library__actor_id__in=ids))
         delete_qs(music_models.Library.objects.filter(actor_id__in=ids))
 
@@ -390,9 +396,76 @@ def fetch(fetch_obj):
         error("save", message=str(e))
         raise
 
+    # special case for channels
+    # when obj is an actor, we check if the actor has a channel associated with it
+    # if it is the case, we consider the fetch obj to be a channel instead
+    if isinstance(obj, models.Actor) and obj.get_channel():
+        obj = obj.get_channel()
     fetch_obj.object = obj
     fetch_obj.status = "finished"
     fetch_obj.fetch_date = timezone.now()
     return fetch_obj.save(
         update_fields=["fetch_date", "status", "object_id", "object_content_type"]
     )
+
+
+class PreserveSomeDataCollector(Collector):
+    """
+    We need to delete everything related to an actor. Well… Almost everything.
+    But definitely not the Delete Activity we send to announce the actor is deleted.
+    """
+
+    def __init__(self, *args, **kwargs):
+        self.creation_date = timezone.now()
+        super().__init__(*args, **kwargs)
+
+    def related_objects(self, related, *args, **kwargs):
+        qs = super().related_objects(related, *args, **kwargs)
+        if related.name == "outbox_activities":
+            # exclude the delete activity can be broadcasted properly
+            qs = qs.exclude(type="Delete", creation_date__gte=self.creation_date)
+
+        return qs
+
+
+@celery.app.task(name="federation.remove_actor")
+@transaction.atomic
+@celery.require_instance(
+    models.Actor.objects.all(), "actor",
+)
+def remove_actor(actor):
+    # Then we broadcast the info over federation. We do this *before* deleting objects
+    # associated with the actor, otherwise follows are removed and we don't know where
+    # to broadcast
+    logger.info("Broadcasting deletion to federation…")
+    collector = PreserveSomeDataCollector(using="default")
+    routes.outbox.dispatch(
+        {"type": "Delete", "object": {"type": actor.type}}, context={"actor": actor}
+    )
+
+    # then we delete any object associated with the actor object, but *not* the actor
+    # itself. We keep it for auditability and sending the Delete ActivityPub message
+    logger.info(
+        "Prepare deletion of objects associated with account %s…",
+        actor.preferred_username,
+    )
+    collector.collect([actor])
+    for model, instances in collector.data.items():
+        if issubclass(model, actor.__class__):
+            # we skip deletion of the actor itself
+            continue
+
+        to_delete = model.objects.filter(pk__in=[instance.pk for instance in instances])
+        logger.info(
+            "Deleting %s objects associated with account %s…",
+            len(instances),
+            actor.preferred_username,
+        )
+        to_delete.delete()
+
+    # Finally, we update the actor itself and mark it as removed
+    logger.info("Marking actor as Tombsone…")
+    actor.type = "Tombstone"
+    actor.name = None
+    actor.summary = None
+    actor.save(update_fields=["type", "name", "summary"])

api/funkwhale_api/federation/views.py

@@ -67,7 +67,11 @@ class ActorViewSet(FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericV
     lookup_field = "preferred_username"
     authentication_classes = [authentication.SignatureAuthentication]
     renderer_classes = renderers.get_ap_renderers()
-    queryset = models.Actor.objects.local().select_related("user")
+    queryset = (
+        models.Actor.objects.local()
+        .select_related("user", "channel__artist", "channel__attributed_to")
+        .prefetch_related("channel__artist__tagged_items__tag")
+    )
     serializer_class = serializers.ActorSerializer
 
     def get_queryset(self):

api/funkwhale_api/music/views.py

@@ -241,6 +241,14 @@ class AlbumViewSet(
             return serializers.AlbumCreateSerializer
         return super().get_serializer_class()
 
+    @transaction.atomic
+    def perform_destroy(self, instance):
+        routes.outbox.dispatch(
+            {"type": "Delete", "object": {"type": "Album"}},
+            context={"album": instance},
+        )
+        models.Album.objects.filter(pk=instance.pk).delete()
+
 
 class LibraryViewSet(
     mixins.CreateModelMixin,
@@ -380,6 +388,15 @@ class TrackViewSet(
         context["description"] = self.action in ["retrieve", "create", "update"]
         return context
 
+    @transaction.atomic
+    def perform_destroy(self, instance):
+        uploads = instance.uploads.order_by("id")
+        routes.outbox.dispatch(
+            {"type": "Delete", "object": {"type": "Audio"}},
+            context={"uploads": list(uploads)},
+        )
+        instance.delete()
+
 
 def strip_absolute_media_url(path):
     if (

api/funkwhale_api/users/tasks.py

 import logging
 
-from django.db.models.deletion import Collector
-
-from funkwhale_api.federation import routes
+from funkwhale_api.federation import tasks as federation_tasks
 from funkwhale_api.taskapp import celery
 
 from . import models
@@ -20,39 +18,6 @@ def delete_account(user):
     user.delete()
     logger.info("Deleted user object")
 
-    # Then we broadcast the info over federation. We do this *before* deleting objects
-    # associated with the actor, otherwise follows are removed and we don't know where
-    # to broadcast
-    logger.info("Broadcasting deletion to federation…")
-    routes.outbox.dispatch(
-        {"type": "Delete", "object": {"type": actor.type}}, context={"actor": actor}
-    )
-
-    # then we delete any object associated with the actor object, but *not* the actor
-    # itself. We keep it for auditability and sending the Delete ActivityPub message
-    collector = Collector(using="default")
-    logger.info(
-        "Prepare deletion of objects associated with account %s…", user.username
-    )
-    collector.collect([actor])
-
-    for model, instances in collector.data.items():
-        if issubclass(model, actor.__class__):
-            # we skip deletion of the actor itself
-            continue
-
-        logger.info(
-            "Deleting %s objects associated with account %s…",
-            len(instances),
-            user.username,
-        )
-        to_delete = model.objects.filter(pk__in=[instance.pk for instance in instances])
-        to_delete.delete()
-
-    # Finally, we update the actor itself and mark it as removed
-    logger.info("Marking actor as Tombsone…")
-    actor.type = "Tombstone"
-    actor.name = None
-    actor.summary = None
-    actor.save(update_fields=["type", "name", "summary"])
-    logger.info("Deletion of account done %s!", user.username)
+    # ensure actor is set to tombstone, activities are removed, etc.
+    federation_tasks.remove_actor(actor_id=actor.pk)
+    logger.info("Deletion of account done %s!", actor.preferred_username)

api/tests/audio/test_views.py

@@ -148,19 +148,17 @@ def test_channel_delete(logged_in_api_client, factories, mocker):
     channel = factories["audio.Channel"](attributed_to=actor)
 
     url = reverse("api:v1:channels-detail", kwargs={"composite": channel.uuid})
-    dispatch = mocker.patch("funkwhale_api.federation.routes.outbox.dispatch")
+    on_commit = mocker.patch("funkwhale_api.common.utils.on_commit")
     response = logged_in_api_client.delete(url)
 
     assert response.status_code == 204
 
+    on_commit.assert_called_once_with(
+        views.federation_tasks.remove_actor.delay, actor_id=channel.actor.pk
+    )
     with pytest.raises(channel.DoesNotExist):
         channel.refresh_from_db()
 
-    dispatch.assert_called_once_with(
-        {"type": "Delete", "object": {"type": channel.actor.type}},
-        context={"actor": channel.actor},
-    )
-
 
 def test_channel_delete_permission(logged_in_api_client, factories):
     logged_in_api_client.user.create_actor()
@@ -218,6 +216,38 @@ def test_channel_unsubscribe(factories, logged_in_api_client):
         subscription.refresh_from_db()
 
 
+def test_channel_subscribe_remote(factories, logged_in_api_client, mocker):
+    dispatch = mocker.patch("funkwhale_api.federation.routes.outbox.dispatch")
+    actor = logged_in_api_client.user.create_actor()
+    channel_actor = factories["federation.Actor"]()
+    channel = factories["audio.Channel"](artist__description=None, actor=channel_actor)
+    url = reverse("api:v1:channels-subscribe", kwargs={"composite": channel.uuid})
+
+    response = logged_in_api_client.post(url)
+
+    assert response.status_code == 201
+    subscription = actor.emitted_follows.latest("id")
+    dispatch.assert_called_once_with(
+        {"type": "Follow"}, context={"follow": subscription}
+    )
+
+
+def test_channel_unsubscribe_remote(factories, logged_in_api_client, mocker):
+    dispatch = mocker.patch("funkwhale_api.federation.routes.outbox.dispatch")
+    actor = logged_in_api_client.user.create_actor()
+    channel_actor = factories["federation.Actor"]()
+    channel = factories["audio.Channel"](actor=channel_actor)
+    subscription = factories["audio.Subscription"](target=channel.actor, actor=actor)
+    url = reverse("api:v1:channels-unsubscribe", kwargs={"composite": channel.uuid})
+
+    response = logged_in_api_client.post(url)
+
+    assert response.status_code == 204
+    dispatch.assert_called_once_with(
+        {"type": "Undo", "object": {"type": "Follow"}}, context={"follow": subscription}
+    )
+
+
 def test_subscriptions_list(factories, logged_in_api_client):
     actor = logged_in_api_client.user.create_actor()
     channel = factories["audio.Channel"](

api/tests/federation/test_api_serializers.py

@@ -167,6 +167,7 @@ def test_fetch_serializer_no_obj(factories, to_api_date):
         ("music.Track", "track", "id"),
         ("music.Library", "library", "uuid"),
         ("music.Upload", "upload", "uuid"),
+        ("audio.Channel", "channel", "uuid"),
         ("federation.Actor", "account", "full_username"),
     ],
 )

api/tests/federation/test_routes.py

@@ -26,6 +26,7 @@ from funkwhale_api.moderation import serializers as moderation_serializers
             routes.inbox_delete_library,
         ),
         ({"type": "Delete", "object": {"type": "Audio"}}, routes.inbox_delete_audio),
+        ({"type": "Delete", "object": {"type": "Album"}}, routes.inbox_delete_album),
         ({"type": "Undo", "object": {"type": "Follow"}}, routes.inbox_undo_follow),
         ({"type": "Update", "object": {"type": "Artist"}}, routes.inbox_update_artist),
         ({"type": "Update", "object": {"type": "Album"}}, routes.inbox_update_album),
@@ -58,6 +59,7 @@ def test_inbox_routes(route, handler):
             routes.outbox_delete_library,
         ),
         ({"type": "Delete", "object": {"type": "Audio"}}, routes.outbox_delete_audio),
+        ({"type": "Delete", "object": {"type": "Album"}}, routes.outbox_delete_album),
         ({"type": "Undo", "object": {"type": "Follow"}}, routes.outbox_undo_follow),
         ({"type": "Update", "object": {"type": "Track"}}, routes.outbox_update_track),
         (
@@ -349,6 +351,34 @@ def test_inbox_create_audio(factories, mocker):
     assert save.call_count == 1
 
 
+def test_inbox_create_audio_channel(factories, mocker):
+    activity = factories["federation.Activity"]()
+    channel = factories["audio.Channel"]()
+    album = factories["music.Album"](artist=channel.artist)
+    upload = factories["music.Upload"](track__album=album, library=channel.library,)
+    payload = {
+        "@context": jsonld.get_default_context(),
+        "type": "Create",
+        "actor": channel.actor.fid,
+        "object": serializers.ChannelUploadSerializer(upload).data,
+    }
+    upload.delete()
+    init = mocker.spy(serializers.ChannelUploadSerializer, "__init__")
+    save = mocker.spy(serializers.ChannelUploadSerializer, "save")
+    result = routes.inbox_create_audio(
+        payload,
+        context={"actor": channel.actor, "raise_exception": True, "activity": activity},
+    )
+    assert channel.library.uploads.count() == 1
+    assert result == {"object": channel.library.uploads.latest("id"), "target": channel}
+
+    assert init.call_count == 1
+    args = init.call_args
+    assert args[1]["data"] == payload["object"]
+    assert args[1]["context"] == {"channel": channel}
+    assert save.call_count == 1
+
+
 def test_inbox_delete_library(factories):
     activity = factories["federation.Activity"]()
 
@@ -368,6 +398,73 @@ def test_inbox_delete_library(factories):
         library.refresh_from_db()
 
 
+def test_inbox_delete_album(factories):
+
+    album = factories["music.Album"](attributed=True)
+    payload = {
+        "type": "Delete",
+        "actor": album.attributed_to.fid,
+        "object": {"type": "Album", "id": album.fid},
+    }
+
+    routes.inbox_delete_album(
+        payload,
+        context={
+            "actor": album.attributed_to,
+            "raise_exception": True,
+            "activity": activity,
+        },
+    )
+
+    with pytest.raises(album.__class__.DoesNotExist):
+        album.refresh_from_db()
+
+
+def test_inbox_delete_album_channel(factories):
+    channel = factories["audio.Channel"]()
+    album = factories["music.Album"](artist=channel.artist)
+    payload = {
+        "type": "Delete",
+        "actor": channel.actor.fid,
+        "object": {"type": "Album", "id": album.fid},
+    }
+
+    routes.inbox_delete_album(
+        payload,
+        context={"actor": channel.actor, "raise_exception": True, "activity": activity},
+    )
+
+    with pytest.raises(album.__class__.DoesNotExist):
+        album.refresh_from_db()
+
+
+def test_outbox_delete_album(factories):
+    album = factories["music.Album"](attributed=True)
+    a = list(routes.outbox_delete_album({"album": album}))[0]
+    expected = serializers.ActivitySerializer(
+        {"type": "Delete", "object": {"type": "Album"