See #170: federation for channels

api/funkwhale_api/audio/factories.py

@@ -18,6 +18,7 @@ class ChannelFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     library = factory.SubFactory(
         federation_factories.MusicLibraryFactory,
         actor=factory.SelfAttribute("..attributed_to"),
+        privacy_level="everyone",
     )
     actor = factory.LazyAttribute(set_actor)
     artist = factory.SubFactory(music_factories.ArtistFactory)
@@ -27,6 +28,8 @@ class ChannelFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
 
     class Params:
         local = factory.Trait(
-            attributed_to__fid=factory.Faker("federation_url", local=True),
+            attributed_to=factory.SubFactory(
+                federation_factories.ActorFactory, local=True
+            ),
             artist__local=True,
         )

api/funkwhale_api/audio/serializers.py

@@ -37,7 +37,7 @@ class ChannelCreateSerializer(serializers.Serializer):
 
         channel.library = music_models.Library.objects.create(
             name=channel.actor.preferred_username,
-            privacy_level="public",
+            privacy_level="everyone",
             actor=validated_data["attributed_to"],
         )
         channel.save()

api/funkwhale_api/federation/activity.py

@@ -118,7 +118,7 @@ def should_reject(fid, actor_id=None, payload={}):
 
 
 @transaction.atomic
-def receive(activity, on_behalf_of):
+def receive(activity, on_behalf_of, inbox_actor=None):
     from . import models
     from . import serializers
     from . import tasks
@@ -131,7 +131,12 @@ def receive(activity, on_behalf_of):
     # we ensure the activity has the bare minimum structure before storing
     # it in our database
     serializer = serializers.BaseActivitySerializer(
-        data=activity, context={"actor": on_behalf_of, "local_recipients": True}
+        data=activity,
+        context={
+            "actor": on_behalf_of,
+            "local_recipients": True,
+            "recipients": [inbox_actor] if inbox_actor else [],
+        },
     )
     serializer.is_valid(raise_exception=True)
 
@@ -161,14 +166,19 @@ def receive(activity, on_behalf_of):
 
     local_to_recipients = get_actors_from_audience(activity.get("to", []))
     local_to_recipients = local_to_recipients.exclude(user=None)
+    local_to_recipients = local_to_recipients.values_list("pk", flat=True)
+    local_to_recipients = list(local_to_recipients)
+    if inbox_actor:
+        local_to_recipients.append(inbox_actor.pk)
 
     local_cc_recipients = get_actors_from_audience(activity.get("cc", []))
     local_cc_recipients = local_cc_recipients.exclude(user=None)
+    local_cc_recipients = local_cc_recipients.values_list("pk", flat=True)
 
     inbox_items = []
     for recipients, type in [(local_to_recipients, "to"), (local_cc_recipients, "cc")]:
 
-        for r in recipients.values_list("pk", flat=True):
+        for r in recipients:
             inbox_items.append(models.InboxItem(actor_id=r, type=type, activity=copy))
 
     models.InboxItem.objects.bulk_create(inbox_items)

api/funkwhale_api/federation/api_serializers.py

@@ -86,7 +86,12 @@ class LibraryFollowSerializer(serializers.ModelSerializer):
 
 
 def serialize_generic_relation(activity, obj):
-    data = {"uuid": obj.uuid, "type": obj._meta.label}
+    data = {"type": obj._meta.label}
+    if data["type"] == "federation.Actor":
+        data["full_username"] = obj.full_username
+    else:
+        data["uuid"] = obj.uuid
+
     if data["type"] == "music.Library":
         data["name"] = obj.name
     if data["type"] == "federation.LibraryFollow":

api/funkwhale_api/federation/authentication.py

@@ -52,9 +52,13 @@ class SignatureAuthentication(authentication.BaseAuthentication):
             actor = actors.get_actor(actor_url)
         except Exception as e:
             logger.info(
-                "Discarding HTTP request from blocked actor/domain %s", actor_url
+                "Discarding HTTP request from blocked actor/domain %s, %s",
+                actor_url,
+                str(e),
+            )
+            raise rest_exceptions.AuthenticationFailed(
+                "Cannot fetch remote actor to authenticate signature"
             )
-            raise rest_exceptions.AuthenticationFailed(str(e))
 
         if not actor.public_key:
             raise rest_exceptions.AuthenticationFailed("No public key found")

api/funkwhale_api/federation/jsonld.py

@@ -214,14 +214,18 @@ def get_ids(v):
 
 
 def get_default_context():
-    return ["https://www.w3.org/ns/activitystreams", "https://w3id.org/security/v1", {}]
+    return [
+        "https://www.w3.org/ns/activitystreams",
+        "https://w3id.org/security/v1",
+        {"manuallyApprovesFollowers": "as:manuallyApprovesFollowers"},
+    ]
 
 
 def get_default_context_fw():
     return [
         "https://www.w3.org/ns/activitystreams",
         "https://w3id.org/security/v1",
-        {},
+        {"manuallyApprovesFollowers": "as:manuallyApprovesFollowers"},
         "https://funkwhale.audio/ns",
     ]
 

api/funkwhale_api/federation/migrations/0022_auto_20191204_1539.py

+# Generated by Django 2.2.7 on 2019-12-04 15:39
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('federation', '0021_auto_20191029_1257'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='actor',
+            name='inbox_url',
+            field=models.URLField(blank=True, max_length=500, null=True),
+        ),
+        migrations.AlterField(
+            model_name='actor',
+            name='outbox_url',
+            field=models.URLField(blank=True, max_length=500, null=True),
+        ),
+    ]

api/funkwhale_api/federation/models.py

@@ -180,8 +180,8 @@ class Actor(models.Model):
 
     fid = models.URLField(unique=True, max_length=500, db_index=True)
     url = models.URLField(max_length=500, null=True, blank=True)
-    outbox_url = models.URLField(max_length=500)
-    inbox_url = models.URLField(max_length=500)
+    outbox_url = models.URLField(max_length=500, null=True, blank=True)
+    inbox_url = models.URLField(max_length=500, null=True, blank=True)
     following_url = models.URLField(max_length=500, null=True, blank=True)
     followers_url = models.URLField(max_length=500, null=True, blank=True)
     shared_inbox_url = models.URLField(max_length=500, null=True, blank=True)

api/funkwhale_api/federation/renderers.py

@@ -6,6 +6,7 @@ def get_ap_renderers():
         ("APActivity", "application/activity+json"),
         ("APLD", "application/ld+json"),
         ("APJSON", "application/json"),
+        ("HTML", "text/html"),
     ]
 
     return [

api/funkwhale_api/federation/routes.py

@@ -131,21 +131,28 @@ def outbox_follow(context):
 @outbox.register({"type": "Create", "object.type": "Audio"})
 def outbox_create_audio(context):
     upload = context["upload"]
+    channel = upload.library.get_channel()
+    upload_serializer = (
+        serializers.ChannelUploadSerializer if channel else serializers.UploadSerializer
+    )
+    followers_target = channel.actor if channel else upload.library
+    actor = channel.actor if channel else upload.library.actor
+
     serializer = serializers.ActivitySerializer(
         {
             "type": "Create",
-            "actor": upload.library.actor.fid,
-            "object": serializers.UploadSerializer(upload).data,
+            "actor": actor.fid,
+            "object": upload_serializer(upload).data,
         }
     )
     yield {
         "type": "Create",
-        "actor": upload.library.actor,
+        "actor": actor,
         "payload": with_recipients(
-            serializer.data, to=[{"type": "followers", "target": upload.library}]
+            serializer.data, to=[{"type": "followers", "target": followers_target}]
         ),
         "object": upload,
-        "target": upload.library,
+        "target": None if channel else upload.library,
     }
 
 
@@ -258,6 +265,9 @@ def inbox_delete_audio(payload, context):
 def outbox_delete_audio(context):
     uploads = context["uploads"]
     library = uploads[0].library
+    channel = library.get_channel()
+    followers_target = channel.actor if channel else library
+    actor = channel.actor if channel else library.actor
     serializer = serializers.ActivitySerializer(
         {
             "type": "Delete",
@@ -266,9 +276,9 @@ def outbox_delete_audio(context):
     )
     yield {
         "type": "Delete",
-        "actor": library.actor,
+        "actor": actor,
         "payload": with_recipients(
-            serializer.data, to=[{"type": "followers", "target": library}]
+            serializer.data, to=[{"type": "followers", "target": followers_target}]
         ),
     }
 

api/funkwhale_api/federation/serializers.py

@@ -68,8 +68,8 @@ class PublicKeySerializer(jsonld.JsonLdSerializer):
 
 class ActorSerializer(jsonld.JsonLdSerializer):
     id = serializers.URLField(max_length=500)
-    outbox = serializers.URLField(max_length=500)
-    inbox = serializers.URLField(max_length=500)
+    outbox = serializers.URLField(max_length=500, required=False)
+    inbox = serializers.URLField(max_length=500, required=False)
     type = serializers.ChoiceField(
         choices=[getattr(contexts.AS, c[0]) for c in models.TYPE_CHOICES]
     )
@@ -77,7 +77,7 @@ class ActorSerializer(jsonld.JsonLdSerializer):
     manuallyApprovesFollowers = serializers.NullBooleanField(required=False)
     name = serializers.CharField(required=False, max_length=200)
     summary = serializers.CharField(max_length=None, required=False)
-    followers = serializers.URLField(max_length=500)
+    followers = serializers.URLField(max_length=500, required=False)
     following = serializers.URLField(max_length=500, required=False, allow_null=True)
     publicKey = PublicKeySerializer(required=False)
     endpoints = EndpointsSerializer(required=False)
@@ -142,8 +142,8 @@ class ActorSerializer(jsonld.JsonLdSerializer):
     def prepare_missing_fields(self):
         kwargs = {
             "fid": self.validated_data["id"],
-            "outbox_url": self.validated_data["outbox"],
-            "inbox_url": self.validated_data["inbox"],
+            "outbox_url": self.validated_data.get("outbox"),
+            "inbox_url": self.validated_data.get("inbox"),
             "following_url": self.validated_data.get("following"),
             "followers_url": self.validated_data.get("followers"),
             "summary": self.validated_data.get("summary"),
@@ -244,7 +244,7 @@ class BaseActivitySerializer(serializers.Serializer):
         to = payload.get("to", [])
         cc = payload.get("cc", [])
 
-        if not to and not cc:
+        if not to and not cc and not self.context.get("recipients"):
             raise serializers.ValidationError(
                 "We cannot handle an activity with no recipient"
             )
@@ -801,6 +801,10 @@ class TagSerializer(jsonld.JsonLdSerializer):
         return value
 
 
+def repr_tag(tag_name):
+    return {"type": "Hashtag", "name": "#{}".format(tag_name)}
+
+
 class MusicEntitySerializer(jsonld.JsonLdSerializer):
     id = serializers.URLField(max_length=500)
     published = serializers.DateTimeField()
@@ -831,7 +835,7 @@ class MusicEntitySerializer(jsonld.JsonLdSerializer):
 
     def get_tags_repr(self, instance):
         return [
-            {"type": "Hashtag", "name": "#{}".format(item.tag.name)}
+            repr_tag(item.tag.name)
             for item in sorted(instance.tagged_items.all(), key=lambda i: i.tag.name)
         ]
 
@@ -1182,3 +1186,71 @@ class NodeInfoLinkSerializer(serializers.Serializer):
 
 class NodeInfoSerializer(serializers.Serializer):
     links = serializers.ListField(child=NodeInfoLinkSerializer(), min_length=1)
+
+
+class ChannelOutboxSerializer(PaginatedCollectionSerializer):
+    type = serializers.ChoiceField(choices=[contexts.AS.OrderedCollection])
+
+    class Meta:
+        jsonld_mapping = PAGINATED_COLLECTION_JSONLD_MAPPING
+
+    def to_representation(self, channel):
+        conf = {
+            "id": channel.actor.outbox_url,
+            "page_size": 100,
+            "attributedTo": channel.actor,
+            "actor": channel.actor,
+            "items": channel.library.uploads.for_federation()
+            .order_by("-creation_date")
+            .filter(track__artist=channel.artist),
+            "type": "OrderedCollection",
+        }
+        r = super().to_representation(conf)
+        return r
+
+
+class ChannelUploadSerializer(serializers.Serializer):
+    def to_representation(self, upload):
+        data = {
+            "id": upload.fid,
+            "type": "Audio",
+            "name": upload.track.full_name,
+            "attributedTo": upload.library.channel.actor.fid,
+            "published": upload.creation_date.isoformat(),
+            "to": contexts.AS.Public
+            if upload.library.privacy_level == "everyone"
+            else "",
+            "url": [
+                {
+                    "type": "Link",
+                    "mimeType": upload.mimetype,
+                    "href": utils.full_url(upload.listen_url),
+                },
+                {
+                    "type": "Link",
+                    "mimeType": "text/html",
+                    "href": utils.full_url(upload.track.get_absolute_url()),
+                },
+            ],
+        }
+        tags = [item.tag.name for item in upload.get_all_tagged_items()]
+        if tags:
+            data["tag"] = [repr_tag(name) for name in tags]
+            data["summary"] = " ".join(["#{}".format(name) for name in tags])
+
+        if self.context.get("include_ap_context", True):
+            data["@context"] = jsonld.get_default_context()
+
+        return data
+
+
+class ChannelCreateUploadSerializer(serializers.Serializer):
+    def to_representation(self, upload):
+        return {
+            "@context": jsonld.get_default_context(),
+            "type": "Create",
+            "actor": upload.library.channel.actor.fid,
+            "object": ChannelUploadSerializer(
+                upload, context={"include_ap_context": False}
+            ).data,
+        }

api/funkwhale_api/federation/views.py

@@ -55,18 +55,57 @@ class ActorViewSet(FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericV
 
     @action(methods=["get", "post"], detail=True)
     def inbox(self, request, *args, **kwargs):
+        inbox_actor = self.get_object()
         if request.method.lower() == "post" and request.actor is None:
             raise exceptions.AuthenticationFailed(
                 "You need a valid signature to send an activity"
             )
         if request.method.lower() == "post":
-            activity.receive(activity=request.data, on_behalf_of=request.actor)
+            activity.receive(
+                activity=request.data,
+                on_behalf_of=request.actor,
+                inbox_actor=inbox_actor,
+            )
         return response.Response({}, status=200)
 
     @action(methods=["get", "post"], detail=True)
     def outbox(self, request, *args, **kwargs):
+        actor = self.get_object()
+        channel = actor.channel
+        if channel:
+            return self.get_channel_outbox_response(request, channel)
         return response.Response({}, status=200)
 
+    def get_channel_outbox_response(self, request, channel):
+        conf = {
+            "id": channel.actor.outbox_url,
+            "actor": channel.actor,
+            "items": channel.library.uploads.for_federation()
+            .order_by("-creation_date")
+            .prefetch_related("library__channel__actor", "track__artist"),
+            "item_serializer": serializers.ChannelCreateUploadSerializer,
+        }
+        page = request.GET.get("page")
+        if page is None:
+            serializer = serializers.ChannelOutboxSerializer(channel)
+            data = serializer.data
+        else:
+            try:
+                page_number = int(page)
+            except Exception:
+                return response.Response({"page": ["Invalid page number"]}, status=400)
+            conf["page_size"] = preferences.get("federation__collection_page_size")
+            p = paginator.Paginator(conf["items"], conf["page_size"])
+            try:
+                page = p.page(page_number)
+                conf["page"] = page
+                serializer = serializers.CollectionPageSerializer(conf)
+                data = serializer.data
+            except paginator.EmptyPage:
+                return response.Response(status=404)
+
+        return response.Response(data)
+
     @action(methods=["get"], detail=True)
     def followers(self, request, *args, **kwargs):
         self.get_object()
@@ -251,6 +290,11 @@ class MusicUploadViewSet(
         actor = music_utils.get_actor_from_request(self.request)
         return queryset.playable_by(actor)
 
+    def get_serializer(self, obj):
+        if obj.library.get_channel():
+            return serializers.ChannelUploadSerializer(obj)
+        return super().get_serializer(obj)
+
 
 class MusicArtistViewSet(
     FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet

api/funkwhale_api/music/models.py

@@ -634,7 +634,10 @@ class UploadQuerySet(common_models.NullsLastQuerySet):
         return self.exclude(library__in=libraries, import_status="finished")
 
     def local(self, include=True):
-        return self.exclude(library__actor__user__isnull=include)
+        query = models.Q(library__actor__domain_id=settings.FEDERATION_HOSTNAME)
+        if not include:
+            query = ~query
+        return self.filter(query)
 
     def for_federation(self):
         return self.filter(import_status="finished", mimetype__startswith="audio/")
@@ -904,6 +907,14 @@ class Upload(models.Model):
             # external storage
             return self.audio_file.name
 
+    def get_all_tagged_items(self):
+        track_tags = self.track.tagged_items.all()
+        album_tags = self.track.album.tagged_items.all()
+        artist_tags = self.track.artist.tagged_items.all()
+
+        items = (track_tags | album_tags | artist_tags).order_by("tag__name")
+        return items
+
 
 MIMETYPE_CHOICES = [(mt, ext) for ext, mt in utils.AUDIO_EXTENSIONS_AND_MIMETYPE]
 

api/funkwhale_api/music/spa_views.py

@@ -4,6 +4,7 @@ from django.conf import settings
 from django.urls import reverse
 from django.db.models import Q
 
+from funkwhale_api.common import preferences
 from funkwhale_api.common import utils
 from funkwhale_api.playlists import models as playlists_models
 
@@ -65,8 +66,9 @@ def library_track(request, pk):
                 "content": obj.album.attachment_cover.download_url_medium_square_crop,
             }
         )
-
-    if obj.uploads.playable_by(None).exists():
+    playable_uploads = obj.uploads.playable_by(None).order_by("id")
+    upload = playable_uploads.first()
+    if upload:
         metas.append(
             {
                 "tag": "meta",
@@ -74,7 +76,15 @@ def library_track(request, pk):
                 "content": utils.join_url(settings.FUNKWHALE_URL, obj.listen_url),
             }
         )
-
+        if preferences.get("federation__enabled"):
+            metas.append(
+                {
+                    "tag": "link",
+                    "rel": "alternate",
+                    "type": "application/activity+json",
+                    "href": upload.fid,
+                }
+            )
         metas.append(
             {
                 "tag": "link",
@@ -133,6 +143,15 @@ def library_album(request, pk):
             }
         )
 
+    if preferences.get("federation__enabled"):
+        metas.append(
+            {
+                "tag": "link",
+                "rel": "alternate",
+                "type": "application/activity+json",
+                "href": obj.fid,
+            }
+        )
     if models.Upload.objects.filter(track__album=obj).playable_by(None).exists():
         metas.append(
             {
@@ -179,6 +198,16 @@ def library_artist(request, pk):
             }
         )
 
+    if preferences.get("federation__enabled"):
+        metas.append(
+            {
+                "tag": "link",
+                "rel": "alternate",
+                "type": "application/activity+json",
+                "href": obj.fid,
+            }
+        )
+
     if (
         models.Upload.objects.filter(Q(track__artist=obj) | Q(track__album__artist=obj))
         .playable_by(None)

api/requirements/local.txt

@@ -9,6 +9,7 @@ django-debug-toolbar>=1.11,<1.12
 
 # improved REPL
 ipdb==0.11
+prompt_toolkit<3
 black
 profiling
 

api/tests/audio/test_serializers.py

@@ -29,7 +29,7 @@ def test_channel_serializer_create(factories):
     assert channel.actor.summary == data["summary"]
     assert channel.actor.preferred_username == data["username"]
     assert channel.actor.name == data["name"]
-    assert channel.library.privacy_level == "public"
+    assert channel.library.privacy_level == "everyone"
     assert channel.library.actor == attributed_to
 
 

api/tests/audio/test_views.py

@@ -34,7 +34,7 @@ def test_channel_create(logged_in_api_client):
     assert channel.attributed_to == actor
     assert channel.actor.summary == data["summary"]
     assert channel.actor.preferred_username == data["username"]
-    assert channel.library.privacy_level == "public"
+    assert channel.library.privacy_level == "everyone"
     assert channel.library.actor == actor
 
 

api/tests/federation/test_activity.py

@@ -16,11 +16,14 @@ from funkwhale_api.federation import (
 def test_receive_validates_basic_attributes_and_stores_activity(
     mrf_inbox_registry, factories, now, mocker
 ):
+
     mocker.patch.object(
         activity.InboxRouter, "get_matching_handlers", return_value=True
     )
     mrf_inbox_registry_apply = mocker.spy(mrf_inbox_registry, "apply")
+    serializer_init = mocker.spy(serializers.BaseActivitySerializer, "__init__")
     mocked_dispatch = mocker.patch("funkwhale_api.common.utils.on_commit")
+    inbox_actor = factories["federation.Actor"]()
     local_to_actor = factories["users.User"]().create_actor()
     local_cc_actor = factories["users.User"]().create_actor()
     remote_actor = factories["federation.Actor"]()
@@ -33,7 +36,9 @@ def test_receive_validates_basic_attributes_and_stores_activity(
         "cc": [local_cc_actor.fid, activity.PUBLIC_ADDRESS],
     }
 
-    copy = activity.receive(activity=a, on_behalf_of=remote_actor)
+    copy = activity.receive(
+        activity=a, on_behalf_of=remote_actor, inbox_actor=inbox_actor
+    )
     mrf_inbox_registry_apply.assert_called_once_with(a, sender_id=a["actor"])
 
     assert copy.payload == a
@@ -45,13 +50,24 @@ def test_receive_validates_basic_attributes_and_stores_activity(
         tasks.dispatch_inbox.delay, activity_id=copy.pk
     )
 
-    assert models.InboxItem.objects.count() == 2
-    for actor, t in [(local_to_actor, "to"), (local_cc_actor, "cc")]:
+    assert models.InboxItem.objects.count() == 3
+    for actor, t in [
+        (local_to_actor, "to"),
+        (inbox_actor, "to"),
+        (local_cc_actor, "cc"),