Allowing to follow entire instances / domains

.env.dev

-DJANGO_ALLOWED_HOSTS=.funkwhale.test,localhost,nginx,0.0.0.0,127.0.0.1
-DJANGO_SETTINGS_MODULE=config.settings.local
-DJANGO_SECRET_KEY=dev
-C_FORCE_ROOT=true
-FUNKWHALE_HOSTNAME=localhost
-FUNKWHALE_PROTOCOL=http
-PYTHONDONTWRITEBYTECODE=true
-VUE_PORT=8080
-MUSIC_DIRECTORY_PATH=/music
-BROWSABLE_API_ENABLED=True
-FORWARDED_PROTO=http
-LDAP_ENABLED=False
-FUNKWHALE_SPA_HTML_ROOT=http://nginx/front/
-PYTHONTRACEMALLOC=0
-
-# Uncomment this if you're using traefik/https
-# FORCE_HTTPS_URLS=True
-
-# Customize to your needs
-POSTGRES_VERSION=11
-DEBUG=true

api/Dockerfile

@@ -4,37 +4,38 @@ FROM alpine:3.14 as pre-build
 # dependencies. This is only required until alpine 3.16 is released, since this
 # allows us to install poetry as package.
 
-RUN apk add --no-cache python3 py3-cryptography py3-pip && \
-		pip3 install poetry
+RUN --mount=type=cache,target=/root/.cache/pip \
+  apk add --no-cache python3 py3-cryptography py3-pip && \
+  pip3 install poetry
 COPY pyproject.toml poetry.lock /
 RUN poetry export --without-hashes > requirements.txt
 RUN poetry export --dev --without-hashes > dev-requirements.txt
 
 FROM alpine:3.14 as builder
 
-RUN \
-    echo 'installing dependencies' && \
-    apk add --no-cache \
-    git \
-    musl-dev \
-    gcc \
-    postgresql-dev \
-    python3-dev \
-    py3-psycopg2 \
-    py3-cryptography \
-    libldap \
-    libffi-dev \
-    make \
-    zlib-dev \
-    jpeg-dev \
-    openldap-dev \
-    openssl-dev \
-    cargo \
-    libxml2-dev \
-    libxslt-dev \
-    curl \
-    && \
-    ln -s /usr/bin/python3 /usr/bin/python
+RUN --mount=type=cache,target=/root/.cache/pip \
+  echo 'installing dependencies' && \
+  apk add --no-cache \
+  git \
+  musl-dev \
+  gcc \
+  postgresql-dev \
+  python3-dev \
+  py3-psycopg2 \
+  py3-cryptography \
+  libldap \
+  libffi-dev \
+  make \
+  zlib-dev \
+  jpeg-dev \
+  openldap-dev \
+  openssl-dev \
+  cargo \
+  libxml2-dev \
+  libxslt-dev \
+  curl \
+  && \
+  ln -s /usr/bin/python3 /usr/bin/python
 
 # create virtual env for next stage
 RUN python -m venv --system-site-packages /venv
@@ -44,26 +45,26 @@ ENV PATH="/venv/bin:/root/.local/bin:$PATH" VIRTUAL_ENV=/venv
 COPY --from=0 /requirements.txt /requirements.txt
 COPY --from=0 /dev-requirements.txt /dev-requirements.txt
 # hack around https://github.com/pypa/pip/issues/6158#issuecomment-456619072
-ENV PIP_DOWNLOAD_CACHE=/noop/
-RUN \
-    echo 'installing pip requirements' && \
-    pip3 install --upgrade pip && \
-    pip3 install setuptools wheel && \
-    # Currently we are unable to relieably build cryptography on armv7. This
-    # is why we need to use the package shipped by Alpine Linux, which is currently
-    # version 3.3.2. Since poetry does not allow in-place dependency pinning, we need
-    # to install the deps using pip.
-    cat /requirements.txt | grep -Ev 'cryptography|autobahn' | pip3 install -r /dev/stdin cryptography==3.3.2 autobahn==21.2.1 && \
-    rm -rf $PIP_DOWNLOAD_CACHE
+#ENV PIP_DOWNLOAD_CACHE=/noop/
+RUN --mount=type=cache,target=/root/.cache/pip \
+  echo 'installing pip requirements' && \
+  pip3 install --upgrade pip && \
+  pip3 install setuptools wheel && \
+  # Currently we are unable to relieably build cryptography on armv7. This
+  # is why we need to use the package shipped by Alpine Linux, which is currently
+  # version 3.3.2. Since poetry does not allow in-place dependency pinning, we need
+  # to install the deps using pip.
+  cat /requirements.txt | grep -Ev 'cryptography|autobahn' | pip3 install -r /dev/stdin cryptography==3.3.2 autobahn==21.2.1 && \
+  #rm -rf $PIP_DOWNLOAD_CACHE
 
-ARG install_dev_deps=0
+  ARG install_dev_deps=0
 RUN \
-    if [ "$install_dev_deps" = "1" ] ; then \
-      echo "Installing dev dependencies" && \
-    cat /dev-requirements.txt | grep -Ev 'cryptography|autobahn' | pip3 install -r /dev/stdin cryptography==3.3.2 autobahn==21.2.1 \
-    ; else  \
-      echo "Skipping dev deps installation" \
-    ; fi
+  if [ "$install_dev_deps" = "1" ] ; then \
+  echo "Installing dev dependencies" && \
+  cat /dev-requirements.txt | grep -Ev 'cryptography|autobahn' | pip3 install -r /dev/stdin cryptography==3.3.2 autobahn==21.2.1 \
+  ; else  \
+  echo "Skipping dev deps installation" \
+  ; fi
 
 
 FROM alpine:3.14 as build-image
@@ -73,18 +74,18 @@ COPY --from=builder /venv /venv
 ENV PATH="/venv/bin:$PATH"
 
 RUN apk add --no-cache \
-    libmagic \
-    bash \
-    gettext \
-    python3 \
-    jpeg-dev \
-    ffmpeg \
-    libpq \
-    libxml2 \
-    libxslt \
-    py3-cryptography \
-    && \
-    ln -s /usr/bin/python3 /usr/bin/python
+  libmagic \
+  bash \
+  gettext \
+  python3 \
+  jpeg-dev \
+  ffmpeg \
+  libpq \
+  libxml2 \
+  libxslt \
+  py3-cryptography \
+  && \
+  ln -s /usr/bin/python3 /usr/bin/python
 
 COPY . /app
 WORKDIR /app

api/config/settings/common.py

@@ -190,6 +190,7 @@ ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=[]) + [FUNKWHALE_HOSTNA
 """
 List of allowed hostnames for which the Funkwhale server will answer.
 """
+GLOBAL_FEDERATION = env.bool("GLOBAL_FEDERATION", default=True)
 
 # APP CONFIGURATION
 # ------------------------------------------------------------------------------

api/config/settings/local.py

@@ -92,7 +92,7 @@ if DEBUG_TOOLBAR_ENABLED:
 # ------------------------------------------------------------------------------
 TEST_RUNNER = "django.test.runner.DiscoverRunner"
 
-# CELERY
+# CELERY Set to true to debug
 CELERY_TASK_ALWAYS_EAGER = False
 # END CELERY
 

api/funkwhale_api/federation/activity.py

@@ -12,6 +12,11 @@ from funkwhale_api.common import utils as funkwhale_utils
 
 from . import contexts
 
+import logging
+
+logger = logging.getLogger(__name__)
+
+
 recursive_getattr = funkwhale_utils.recursive_getattr
 
 
@@ -119,13 +124,16 @@ def should_reject(fid, actor_id=None, payload={}):
 
 @transaction.atomic
 def receive(activity, on_behalf_of, inbox_actor=None):
+    """
+    Receive an activity, find his recipients and save it to the database before dispatching it
+    """
     from . import models
     from . import serializers
     from . import tasks
     from .routes import inbox
     from funkwhale_api.moderation import mrf
 
-    logger.debug(
+    logger.info(
         "[federation] Received activity from %s : %s", on_behalf_of.fid, activity
     )
     # we ensure the activity has the bare minimum structure before storing
@@ -339,6 +347,10 @@ class OutboxRouter(Router):
             deliveries_by_activity_uuid = {}
             prepared_activities = []
             for activity_data in activities_data:
+                # If its a domain update we change the object type of the library to "Domain" to allow specific routing
+                for recipient in activity_data["payload"]["to"]:
+                    if recipient is dict and "domain_followers" in recipient["type"]:
+                        activity_data["payload"]["object"]["type"] = "Domain"
                 activity_data["payload"]["actor"] = activity_data["actor"].fid
                 to = activity_data["payload"].pop("to", [])
                 cc = activity_data["payload"].pop("cc", [])
@@ -461,6 +473,19 @@ def prepare_deliveries_and_inbox_items(recipient_list, type, allowed_domains=Non
                 else:
                     remote_inbox_urls.add(actor.shared_inbox_url or actor.inbox_url)
             urls.append(r["target"].followers_url)
+
+        elif isinstance(r, dict) and r["type"] == "domain_followers":
+            received_follows = (
+                r["target"]
+                .domainfollow_received_follows.filter(approved=True)
+                .select_related("actor__user")
+            )
+            logger.info("received_follows" + str(received_follows))
+            for follow in received_follows:
+                actor = follow.actor
+                remote_inbox_urls.add(actor.shared_inbox_url or actor.inbox_url)
+                urls.append(actor.shared_inbox_url)
+
         elif isinstance(r, dict) and r["type"] == "actor_inbox":
             actor = r["actor"]
             urls.append(actor.fid)

api/funkwhale_api/federation/admin.py

@@ -78,6 +78,14 @@ class LibraryFollowAdmin(admin.ModelAdmin):
     list_select_related = True
 
 
+@admin.register(models.DomainFollow)
+class DomainFollowAdmin(admin.ModelAdmin):
+    list_display = ["actor", "target", "approved", "creation_date"]
+    list_filter = ["approved"]
+    search_fields = ["actor__fid", "target__name"]
+    list_select_related = True
+
+
 @admin.register(models.InboxItem)
 class InboxItemAdmin(admin.ModelAdmin):
     list_display = ["actor", "activity", "type", "is_read"]

api/funkwhale_api/federation/api_serializers.py

@@ -12,10 +12,15 @@ from funkwhale_api.common import fields as common_fields
 from funkwhale_api.common import serializers as common_serializers
 from funkwhale_api.music import models as music_models
 from funkwhale_api.users import serializers as users_serializers
-
+from funkwhale_api.federation import models as federation_models
 from . import filters
 from . import models
 from . import serializers as federation_serializers
+from . import tasks
+
+import logging
+
+logger = logging.getLogger(__name__)
 
 
 class NestedLibraryFollowSerializer(serializers.ModelSerializer):
@@ -24,6 +29,17 @@ class NestedLibraryFollowSerializer(serializers.ModelSerializer):
         fields = ["creation_date", "uuid", "fid", "approved", "modification_date"]
 
 
+class NestedDomainFollowSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = models.DomainFollow
+        fields = [
+            "creation_date",
+            "fid",
+            "approved",
+            "modification_date",
+        ]
+
+
 class LibraryScanSerializer(serializers.ModelSerializer):
     class Meta:
         model = music_models.LibraryScan
@@ -39,6 +55,33 @@ class LibraryScanSerializer(serializers.ModelSerializer):
 
 class DomainSerializer(serializers.Serializer):
     name = serializers.CharField()
+    follow = serializers.SerializerMethodField()
+
+    class Meta:
+        model = federation_models.Domain
+        fields = [
+            "fid",
+            "name",
+            "nodeinfo",
+        ]
+
+    def get_follow(self, o):
+        try:
+            return NestedDomainFollowSerializer(o._follows[0]).data
+        except (AttributeError, IndexError):
+            return None
+
+
+class LibraryFollowListSerializer(serializers.ListSerializer):
+    def create(self, validated_data):
+        libs = [models.LibraryFollow(**lib) for lib in validated_data]
+        return models.LibraryFollow.objects.bulk_create(libs, batch_size=1000)
+
+
+class LibraryListSerializer(serializers.ListSerializer):
+    def create(self, validated_data):
+        libs = [models.Library(**lib) for lib in validated_data]
+        return models.Library.objects.bulk_create(libs, batch_size=1000)
 
 
 class LibrarySerializer(serializers.ModelSerializer):
@@ -61,6 +104,7 @@ class LibrarySerializer(serializers.ModelSerializer):
             "follow",
             "latest_scan",
         ]
+        list_serializer_class = LibraryListSerializer
 
     def get_uploads_count(self, o):
         return max(getattr(o, "_uploads_count", 0), o.uploads_count)
@@ -80,14 +124,28 @@ class LibrarySerializer(serializers.ModelSerializer):
 class LibraryFollowSerializer(serializers.ModelSerializer):
     target = common_serializers.RelatedField("uuid", LibrarySerializer(), required=True)
     actor = serializers.SerializerMethodField()
+    follow_context = serializers.ChoiceField(
+        choices=["Domain", "Library"], required=False
+    )
 
     class Meta:
         model = models.LibraryFollow
-        fields = ["creation_date", "actor", "uuid", "target", "approved"]
+        fields = [
+            "creation_date",
+            "actor",
+            "uuid",
+            "target",
+            "approved",
+            "follow_context",
+        ]
         read_only_fields = ["uuid", "actor", "approved", "creation_date"]
+        list_serializer_class = LibraryFollowListSerializer
 
     def validate_target(self, v):
-        actor = self.context["actor"]
+        try:
+            actor = self.context["actor"]
+        except KeyError:
+            raise KeyError
         if v.actor == actor:
             raise serializers.ValidationError("You cannot follow your own library")
 
@@ -99,16 +157,54 @@ class LibraryFollowSerializer(serializers.ModelSerializer):
         return federation_serializers.APIActorSerializer(o.actor).data
 
 
+class DomainFollowSerializer(serializers.ModelSerializer):
+    target = common_serializers.RelatedField("name", DomainSerializer(), required=True)
+    actor = serializers.SerializerMethodField()
+
+    class Meta:
+        model = models.DomainFollow
+        fields = ["uuid", "creation_date", "actor", "target", "approved"]
+        read_only_fields = ["uuid", "actor", "approved", "creation_date"]
+
+    def validate_target(self, v):
+        actor = self.context["actor"]
+        if v.domainfollow_received_follows.filter(actor=actor).exists():
+            raise serializers.ValidationError("You are already following this Domain")
+        domain = models.Domain.objects.get(name=v.name)
+        now = datetime.datetime.now(timezone.utc)
+        if (
+            domain.nodeinfo_fetch_date - now
+        ).total_seconds() > 86400 or "error" in domain.nodeinfo:
+            tasks.refresh_actor_data()
+            tasks.update_domain_nodeinfo(domain_name=domain)
+            domain = models.Domain.objects.get(name=v.name)
+        if "error" in domain.nodeinfo:
+            e = domain.nodeinfo["error"]
+            logger.error(f"Could not get domain information because of : {e!r}")
+            raise serializers.ValidationError(domain.nodeinfo["error"])
+        return domain
+
+    def get_actor(self, o):
+        logger.info("o is :" + str(o))
+        return federation_serializers.APIActorSerializer(o.actor).data
+
+
 def serialize_generic_relation(activity, obj):
     data = {"type": obj._meta.label}
+    logger.info("data[type]" + str(data["type"]))
     if data["type"] == "federation.Actor":
         data["full_username"] = obj.full_username
+    elif data["type"] == "federation.Domain":
+        data["name"] = obj.name
     else:
         data["uuid"] = obj.uuid
-
     if data["type"] == "music.Library":
         data["name"] = obj.name
-    if data["type"] == "federation.LibraryFollow":
+    # to do : test if this is works
+    if (
+        data["type"] == "federation.LibraryFollow"
+        or data["type"] == "federation.DomainFollow"
+    ):
         data["approved"] = obj.approved
 
     return data

api/funkwhale_api/federation/api_urls.py

@@ -5,6 +5,7 @@ from . import api_views
 router = routers.OptionalSlashRouter()
 router.register(r"fetches", api_views.FetchViewSet, "fetches")
 router.register(r"follows/library", api_views.LibraryFollowViewSet, "library-follows")
+router.register(r"follows/domain", api_views.DomainFollowViewSet, "domain-follows")
 router.register(r"inbox", api_views.InboxItemViewSet, "inbox")
 router.register(r"libraries", api_views.LibraryViewSet, "libraries")
 router.register(r"domains", api_views.DomainViewSet, "domains")

api/funkwhale_api/federation/api_views.py

+from urllib import request
 import requests.exceptions
+import requests
 
 from django.conf import settings
 from django.db import transaction
@@ -11,6 +13,7 @@ from rest_framework import response
 from rest_framework import viewsets
 
 from funkwhale_api.common import preferences
+from funkwhale_api.common import permissions as common_permissions
 from funkwhale_api.common import utils as common_utils
 from funkwhale_api.common.permissions import ConditionalAuthentication
 from funkwhale_api.music import models as music_models
@@ -27,6 +30,10 @@ from . import serializers
 from . import tasks
 from . import utils
 
+import logging
+
+logger = logging.getLogger(__name__)
+
 
 @transaction.atomic
 def update_follow(follow, approved):
@@ -263,6 +270,93 @@ class DomainViewSet(
         return qs
 
 
+class DomainFollowViewSet(
+    mixins.CreateModelMixin,
+    mixins.ListModelMixin,
+    mixins.RetrieveModelMixin,
+    mixins.DestroyModelMixin,
+    viewsets.GenericViewSet,
+):
+    lookup_field = "uuid"
+    queryset = (
+        models.DomainFollow.objects.all()
+        .order_by("-creation_date")
+        .select_related("actor")
+    )
+    serializer_class = api_serializers.DomainFollowSerializer
+    permission_classes = [oauth_permissions.ScopePermission]
+    required_scope = "follows"
+    filterset_class = filters.DomainFollowFilter
+    ordering_fields = ("creation_date",)
+
+    def get_queryset(self):
+        qs = super().get_queryset()
+        return qs.filter(actor=self.request.user.actor).exclude(approved=False)
+
+    def perform_create(self, serializer):
+        follow = serializer.save(actor=self.request.user.actor)
+        routes.outbox.dispatch({"type": "Follow"}, context={"follow": follow})
+
+    @transaction.atomic
+    def perform_destroy(self, instance):
+        routes.outbox.dispatch(
+            {"type": "Undo", "object": {"type": "Follow"}},
+            context={"follow": instance},
+        )
+        tasks.domain_bulk_unfollow_libraries.delay(
+            instance.target.name, actor_id=self.request.user.actor.pk
+        )
+        instance.delete()
+
+    def get_serializer_context(self):
+        context = super().get_serializer_context()
+        context["actor"] = self.request.user.actor
+        return context
+
+    @decorators.action(methods=["post"], detail=True)
+    def accept(self, request, *args, **kwargs):
+        try:
+            follow = self.queryset.get(
+                target__actor=self.request.user.actor, uuid=kwargs["uuid"]
+            )
+        except models.DomainFollow.DoesNotExist:
+            return response.Response({}, status=404)
+        update_follow(follow, approved=True)
+        return response.Response(status=204)
+
+    @decorators.action(methods=["post"], detail=True)
+    def reject(self, request, *args, **kwargs):
+        try:
+            follow = self.queryset.get(
+                target__actor=self.request.user.actor, uuid=kwargs["uuid"]
+            )
+        except models.DomainFollow.DoesNotExist:
+            return response.Response({}, status=404)
+
+        update_follow(follow, approved=False)
+        return response.Response(status=204)
+
+    @decorators.action(methods=["get"], detail=False)
+    def all(self, request, *args, **kwargs):
+        """
+        Return all the subscriptions of the current user, with only limited data
+        to have a performant endpoint and avoid lots of queries just to display
+        subscription status in the UI
+        """
+        follows = list(
+            self.get_queryset().values_list("uuid", "target__name", "approved")
+        )
+
+        payload = {
+            "results": [
+                {"uuid": str(u[0]), "domain": str(u[1]), "approved": u[2]}
+                for u in follows
+            ],
+            "count": len(follows),
+        }
+        return response.Response(payload, status=200)
+
+
 class ActorViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet):
     queryset = models.Actor.objects.select_related(
         "user", "channel", "summary_obj", "attachment_icon"

api/funkwhale_api/federation/factories.py

 import uuid
 
 import factory
+
+# see https://stackoverflow.com/questions/60914361/python-import-error-module-factory-has-no-attribute-fuzzy
+from factory import fuzzy
 import requests
 import requests_http_message_signatures
 from django.conf import settings
@@ -238,11 +241,23 @@ class DeliveryFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
 class LibraryFollowFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     target = factory.SubFactory(MusicLibraryFactory)
     actor = factory.SubFactory(ActorFactory)
+    follow_context = fuzzy.FuzzyChoice(
+        models.LibraryFollow.FOLLOW_CONTEXT_CHOICES, getter=lambda c: c[0]
+    )
 
     class Meta:
         model = "federation.LibraryFollow"
 
 
+@registry.register
+class DomainollowFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
+    target = factory.SubFactory(DomainFactory)
+    actor = factory.SubFactory(ActorFactory)
+
+    class Meta:
+        model = "federation.DomainFollow"
+
+
 class ArtistMetadataFactory(factory.Factory):
     name = factory.Faker("name")
 

api/funkwhale_api/federation/filters.py

@@ -34,6 +34,12 @@ class LibraryFollowFilter(django_filters.FilterSet):
         fields = ["approved"]
 
 
+class DomainFollowFilter(django_filters.FilterSet):
+    class Meta:
+        model = models.DomainFollow
+        fields = ["approved"]
+
+
 class InboxItemFilter(django_filters.FilterSet):
     is_read = django_filters.BooleanFilter(
         "is_read", widget=django_filters.widgets.BooleanWidget()

api/funkwhale_api/federation/migrations/0027_auto_20220527_2006.py

+# Generated by Django 3.2.13 on 2022-05-27 20:06
+
+import django.core.serializers.json
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import funkwhale_api.federation.models
+import uuid
+