Merge branch 'domain-actor-block' into 'develop'

Domains/actor blocking and instance-level moderation tools

See merge request funkwhale/funkwhale!521

api/config/settings/common.py

@@ -156,6 +156,7 @@ LOCAL_APPS = (
     "funkwhale_api.requests",
     "funkwhale_api.favorites",
     "funkwhale_api.federation",
+    "funkwhale_api.moderation",
     "funkwhale_api.radios",
     "funkwhale_api.history",
     "funkwhale_api.playlists",

api/config/settings/local.py

@@ -38,7 +38,7 @@ DEBUG_TOOLBAR_CONFIG = {
     "DISABLE_PANELS": ["debug_toolbar.panels.redirects.RedirectsPanel"],
     "SHOW_TEMPLATE_CONTEXT": True,
     "SHOW_TOOLBAR_CALLBACK": lambda request: True,
-    "JQUERY_URL": "",
+    "JQUERY_URL": "/staticfiles/admin/js/vendor/jquery/jquery.js",
 }
 
 # django-extensions

api/funkwhale_api/common/decorators.py

+from rest_framework import response
+from rest_framework.decorators import list_route
+
+
+def action_route(serializer_class):
+    @list_route(methods=["post"])
+    def action(self, request, *args, **kwargs):
+        queryset = self.get_queryset()
+        serializer = serializer_class(request.data, queryset=queryset)
+        serializer.is_valid(raise_exception=True)
+        result = serializer.save()
+        return response.Response(result, status=200)
+
+    return action

api/funkwhale_api/common/serializers.py

@@ -123,7 +123,7 @@ class ActionSerializer(serializers.Serializer):
         if type(value) in [list, tuple]:
             return self.queryset.filter(
                 **{"{}__in".format(self.pk_field): value}
-            ).order_by("id")
+            ).order_by(self.pk_field)
 
         raise serializers.ValidationError(
             "{} is not a valid value for objects. You must provide either a "

api/funkwhale_api/common/validators.py

 import mimetypes
 from os.path import splitext
 
+from django.core import validators
 from django.core.exceptions import ValidationError
 from django.core.files.images import get_image_dimensions
 from django.template.defaultfilters import filesizeformat
@@ -150,3 +151,17 @@ class FileValidator(object):
             }
 
             raise ValidationError(message)
+
+
+class DomainValidator(validators.URLValidator):
+    message = "Enter a valid domain name."
+
+    def __call__(self, value):
+        """
+        This is a bit hackish but since we don't have any built-in domain validator,
+        we use the url one, and prepend http:// in front of it.
+
+        If it fails, we know the domain is not valid.
+        """
+        super().__call__("http://{}".format(value))
+        return value

api/funkwhale_api/federation/activity.py

@@ -80,6 +80,30 @@ OBJECT_TYPES = (
 BROADCAST_TO_USER_ACTIVITIES = ["Follow", "Accept"]
 
 
+def should_reject(id, actor_id=None, payload={}):
+    from funkwhale_api.moderation import models as moderation_models
+
+    policies = moderation_models.InstancePolicy.objects.active()
+
+    media_types = ["Audio", "Artist", "Album", "Track", "Library", "Image"]
+    relevant_values = [
+        recursive_gettattr(payload, "type", permissive=True),
+        recursive_gettattr(payload, "object.type", permissive=True),
+        recursive_gettattr(payload, "target.type", permissive=True),
+    ]
+    # if one of the payload types match our internal media types, then
+    # we apply policies that reject media
+    if set(media_types) & set(relevant_values):
+        policy_type = Q(block_all=True) | Q(reject_media=True)
+    else:
+        policy_type = Q(block_all=True)
+
+    query = policies.matching_url_query(id) & policy_type
+    if actor_id:
+        query |= policies.matching_url_query(actor_id) & policy_type
+    return policies.filter(query).exists()
+
+
 @transaction.atomic
 def receive(activity, on_behalf_of):
     from . import models
@@ -92,6 +116,16 @@ def receive(activity, on_behalf_of):
         data=activity, context={"actor": on_behalf_of, "local_recipients": True}
     )
     serializer.is_valid(raise_exception=True)
+    if should_reject(
+        id=serializer.validated_data["id"],
+        actor_id=serializer.validated_data["actor"].fid,
+        payload=activity,
+    ):
+        logger.info(
+            "[federation] Discarding activity due to instance policies %s",
+            serializer.validated_data.get("id"),
+        )
+        return
     try:
         copy = serializer.save()
     except IntegrityError:
@@ -283,7 +317,7 @@ class OutboxRouter(Router):
             return activities
 
 
-def recursive_gettattr(obj, key):
+def recursive_gettattr(obj, key, permissive=False):
     """
     Given a dictionary such as {'user': {'name': 'Bob'}} and
     a dotted string such as user.name, returns 'Bob'.
@@ -292,7 +326,12 @@ def recursive_gettattr(obj, key):
     """
     v = obj
     for k in key.split("."):
-        v = v.get(k)
+        try:
+            v = v.get(k)
+        except (TypeError, AttributeError):
+            if not permissive:
+                raise
+            return
         if v is None:
             return
 

api/funkwhale_api/federation/api_views.py

@@ -13,6 +13,7 @@ from funkwhale_api.music import models as music_models
 
 from . import activity
 from . import api_serializers
+from . import exceptions
 from . import filters
 from . import models
 from . import routes
@@ -128,11 +129,16 @@ class LibraryViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet):
         except KeyError:
             return response.Response({"fid": ["This field is required"]})
         try:
-            library = utils.retrieve(
+            library = utils.retrieve_ap_object(
                 fid,
                 queryset=self.queryset,
                 serializer_class=serializers.LibrarySerializer,
             )
+        except exceptions.BlockedActorOrDomain:
+            return response.Response(
+                {"detail": "This domain/account is blocked on your instance."},
+                status=400,
+            )
         except requests.exceptions.RequestException as e:
             return response.Response(
                 {"detail": "Error while fetching the library: {}".format(str(e))},

api/funkwhale_api/federation/authentication.py

 import cryptography
+import logging
+
 from django.contrib.auth.models import AnonymousUser
-from rest_framework import authentication, exceptions
+from rest_framework import authentication, exceptions as rest_exceptions
+
+from funkwhale_api.moderation import models as moderation_models
+from . import actors, exceptions, keys, signing, utils
+
 
-from . import actors, keys, signing, utils
+logger = logging.getLogger(__name__)
 
 
 class SignatureAuthentication(authentication.BaseAuthentication):
@@ -14,20 +20,36 @@ class SignatureAuthentication(authentication.BaseAuthentication):
         except KeyError:
             return
         except ValueError as e:
-            raise exceptions.AuthenticationFailed(str(e))
+            raise rest_exceptions.AuthenticationFailed(str(e))
+
+        try:
+            actor_url = key_id.split("#")[0]
+        except (TypeError, IndexError, AttributeError):
+            raise rest_exceptions.AuthenticationFailed("Invalid key id")
+
+        policies = (
+            moderation_models.InstancePolicy.objects.active()
+            .filter(block_all=True)
+            .matching_url(actor_url)
+        )
+        if policies.exists():
+            raise exceptions.BlockedActorOrDomain()
 
         try:
-            actor = actors.get_actor(key_id.split("#")[0])
+            actor = actors.get_actor(actor_url)
         except Exception as e:
-            raise exceptions.AuthenticationFailed(str(e))
+            logger.info(
+                "Discarding HTTP request from blocked actor/domain %s", actor_url
+            )
+            raise rest_exceptions.AuthenticationFailed(str(e))
 
         if not actor.public_key:
-            raise exceptions.AuthenticationFailed("No public key found")
+            raise rest_exceptions.AuthenticationFailed("No public key found")
 
         try:
             signing.verify_django(request, actor.public_key.encode("utf-8"))
         except cryptography.exceptions.InvalidSignature:
-            raise exceptions.AuthenticationFailed("Invalid signature")
+            raise rest_exceptions.AuthenticationFailed("Invalid signature")
 
         return actor
 

api/funkwhale_api/federation/exceptions.py

+from rest_framework import exceptions
+
+
 class MalformedPayload(ValueError):
     pass
 
 
 class MissingSignature(KeyError):
     pass
+
+
+class BlockedActorOrDomain(exceptions.AuthenticationFailed):
+    pass

api/funkwhale_api/federation/factories.py

@@ -67,7 +67,7 @@ def create_user(actor):
 
 
 @registry.register
-class Domain(NoUpdateOnCreate, factory.django.DjangoModelFactory):
+class DomainFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     name = factory.Faker("domain_name")
 
     class Meta:
@@ -81,7 +81,7 @@ class ActorFactory(NoUpdateOnCreate, factory.DjangoModelFactory):
     private_key = None
     preferred_username = factory.Faker("user_name")
     summary = factory.Faker("paragraph")
-    domain = factory.SubFactory(Domain)
+    domain = factory.SubFactory(DomainFactory)
     fid = factory.LazyAttribute(
         lambda o: "https://{}/users/{}".format(o.domain.name, o.preferred_username)
     )

api/funkwhale_api/federation/fields.py

+from rest_framework import serializers
+
+from . import models
+
+
+class ActorRelatedField(serializers.EmailField):
+    def to_representation(self, value):
+        return value.full_username
+
+    def to_internal_value(self, value):
+        value = super().to_internal_value(value)
+        username, domain = value.split("@")
+        try:
+            return models.Actor.objects.get(
+                preferred_username=username, domain_id=domain
+            )
+        except models.Actor.DoesNotExist:
+            raise serializers.ValidationError("Invalid actor name")

api/funkwhale_api/federation/models.py

@@ -13,6 +13,7 @@ from django.urls import reverse
 
 from funkwhale_api.common import session
 from funkwhale_api.common import utils as common_utils
+from funkwhale_api.common import validators as common_validators
 from funkwhale_api.music import utils as music_utils
 
 from . import utils as federation_utils
@@ -83,7 +84,11 @@ class DomainQuerySet(models.QuerySet):
 
 
 class Domain(models.Model):
-    name = models.CharField(primary_key=True, max_length=255)
+    name = models.CharField(
+        primary_key=True,
+        max_length=255,
+        validators=[common_validators.DomainValidator()],
+    )
     creation_date = models.DateTimeField(default=timezone.now)
     nodeinfo_fetch_date = models.DateTimeField(default=None, null=True, blank=True)
     nodeinfo = JSONField(default=empty_dict, max_length=50000, blank=True)

api/funkwhale_api/federation/serializers.py

@@ -567,7 +567,7 @@ class LibrarySerializer(PaginatedCollectionSerializer):
         return r
 
     def create(self, validated_data):
-        actor = utils.retrieve(
+        actor = utils.retrieve_ap_object(
             validated_data["actor"],
             queryset=models.Actor,
             serializer_class=ActorSerializer,

api/funkwhale_api/federation/tasks.py

@@ -186,3 +186,46 @@ def update_domain_nodeinfo(domain):
     domain.nodeinfo_fetch_date = now
     domain.nodeinfo = nodeinfo
     domain.save(update_fields=["nodeinfo", "nodeinfo_fetch_date"])
+
+
+def delete_qs(qs):
+    label = qs.model._meta.label
+    result = qs.delete()
+    related = sum(result[1].values())
+
+    logger.info(
+        "Purged %s %s objects (and %s related entities)", result[0], label, related
+    )
+
+
+def handle_purge_actors(ids, only=[]):
+    """
+    Empty only means we purge everything
+    Otherwise, we purge only the requested bits: media
+    """
+    # purge follows (received emitted)
+    if not only:
+        delete_qs(models.LibraryFollow.objects.filter(target__actor_id__in=ids))
+        delete_qs(models.Follow.objects.filter(actor_id__in=ids))
+
+    # purge audio content
+    if not only or "media" in only:
+        delete_qs(models.LibraryFollow.objects.filter(actor_id__in=ids))
+        delete_qs(models.Follow.objects.filter(target_id__in=ids))
+        delete_qs(music_models.Upload.objects.filter(library__actor_id__in=ids))
+        delete_qs(music_models.Library.objects.filter(actor_id__in=ids))
+
+    # purge remaining activities / deliveries
+    if not only:
+        delete_qs(models.InboxItem.objects.filter(actor_id__in=ids))
+        delete_qs(models.Activity.objects.filter(actor_id__in=ids))
+
+
+@celery.app.task(name="federation.purge_actors")
+def purge_actors(ids=[], domains=[], only=[]):
+    actors = models.Actor.objects.filter(
+        Q(id__in=ids) | Q(domain_id__in=domains)
+    ).order_by("id")
+    found_ids = list(actors.values_list("id", flat=True))
+    logger.info("Starting purging %s accounts", len(found_ids))
+    handle_purge_actors(ids=found_ids, only=only)

api/funkwhale_api/federation/utils.py

@@ -3,7 +3,9 @@ import re
 from django.conf import settings
 
 from funkwhale_api.common import session
+from funkwhale_api.moderation import models as moderation_models
 
+from . import exceptions
 from . import signing
 
 
@@ -58,7 +60,14 @@ def slugify_username(username):
     return re.sub(r"[-\s]+", "_", value)
 
 
-def retrieve(fid, actor=None, serializer_class=None, queryset=None):
+def retrieve_ap_object(
+    fid, actor=None, serializer_class=None, queryset=None, apply_instance_policies=True
+):
+    from . import activity
+
+    policies = moderation_models.InstancePolicy.objects.active().filter(block_all=True)
+    if apply_instance_policies and policies.matching_url(fid):
+        raise exceptions.BlockedActorOrDomain()
     if queryset:
         try:
             # queryset can also be a Model class
@@ -83,6 +92,16 @@ def retrieve(fid, actor=None, serializer_class=None, queryset=None):
     )
     response.raise_for_status()
     data = response.json()
+
+    # we match against moderation policies here again, because the FID of the returned
+    # object may not be the same as the URL used to access it
+    try:
+        id = data["id"]
+    except KeyError:
+        pass
+    else:
+        if apply_instance_policies and activity.should_reject(id=id, payload=data):
+            raise exceptions.BlockedActorOrDomain()
     if not serializer_class:
         return data
     serializer = serializer_class(data=data)

api/funkwhale_api/manage/filters.py

@@ -4,6 +4,7 @@ from funkwhale_api.common import fields
 from funkwhale_api.common import search
 
 from funkwhale_api.federation import models as federation_models
+from funkwhale_api.moderation import models as moderation_models
 from funkwhale_api.music import models as music_models
 from funkwhale_api.users import models as users_models
 
@@ -87,3 +88,24 @@ class ManageInvitationFilterSet(filters.FilterSet):
         if value is None:
             return queryset
         return queryset.open(value)
+
+
+class ManageInstancePolicyFilterSet(filters.FilterSet):
+    q = fields.SearchFilter(
+        search_fields=[
+            "summary",
+            "target_domain__name",
+            "target_actor__username",
+            "target_actor__domain__name",
+        ]
+    )
+
+    class Meta:
+        model = moderation_models.InstancePolicy
+        fields = [
+            "q",
+            "block_all",
+            "silence_activity",
+            "silence_notifications",
+            "reject_media",
+        ]

api/funkwhale_api/manage/serializers.py

@@ -3,7 +3,11 @@ from django.db import transaction
 from rest_framework import serializers
 
 from funkwhale_api.common import serializers as common_serializers
+from funkwhale_api.common import utils as common_utils
 from funkwhale_api.federation import models as federation_models
+from funkwhale_api.federation import fields as federation_fields
+from funkwhale_api.federation import tasks as federation_tasks
+from funkwhale_api.moderation import models as moderation_models
 from funkwhale_api.music import models as music_models
 from funkwhale_api.users import models as users_models
 
@@ -185,6 +189,13 @@ class ManageDomainSerializer(serializers.ModelSerializer):
             "outbox_activities_count",
             "nodeinfo",
             "nodeinfo_fetch_date",
+            "instance_policy",
+        ]
+        read_only_fields = [
+            "creation_date",
+            "instance_policy",
+            "nodeinfo",
+            "nodeinfo_fetch_date",
         ]
 
     def get_actors_count(self, o):
@@ -194,6 +205,17 @@ class ManageDomainSerializer(serializers.ModelSerializer):
         return getattr(o, "outbox_activities_count", 0)
 
 
+class ManageDomainActionSerializer(common_serializers.ActionSerializer):
+    actions = [common_serializers.Action("purge", allow_all=False)]
+    filterset_class = filters.ManageDomainFilterSet
+    pk_field = "name"
+
+    @transaction.atomic
+    def handle_purge(self, objects):
+        ids = objects.values_list("pk", flat=True)
+        common_utils.on_commit(federation_tasks.purge_actors.delay, domains=list(ids))
+
+
 class ManageActorSerializer(serializers.ModelSerializer):
     uploads_count = serializers.SerializerMethodField()
     user = ManageUserSerializer()
@@ -218,7 +240,102 @@ class ManageActorSerializer(serializers.ModelSerializer):
             "manually_approves_followers",
             "uploads_count",
             "user",
+            "instance_policy",
         ]
+        read_only_fields = ["creation_date", "instance_policy"]
 
     def get_uploads_count(self, o):
         return getattr(o, "uploads_count", 0)
+
+
+class ManageActorActionSerializer(common_serializers.ActionSerializer):
+    actions = [common_serializers.Action("purge", allow_all=False)]
+    filterset_class = filters.ManageActorFilterSet
+
+    @transaction.atomic
+    def handle_purge(self, objects):
+        ids = objects.values_list("id", flat=True)
+        common_utils.on_commit(federation_tasks.purge_actors.delay, ids=list(ids))
+
+
+class TargetSerializer(serializers.Serializer):
+    type = serializers.ChoiceField(choices=["domain", "actor"])
+    id = serializers.CharField()
+
+    def to_representation(self, value):
+        if value["type"] == "domain":
+            return {"type": "domain", "id": value["obj"].name}
+        if value["type"] == "actor":
+            return {"type": "actor", "id": value["obj"].full_username}
+
+    def to_internal_value(self, value):
+        if value["type"] == "domain":
+            field = serializers.PrimaryKeyRelatedField(
+                queryset=federation_models.Domain.objects.external()
+            )
+        if value["type"] == "actor":
+            field = federation_fields.ActorRelatedField()
+        value["obj"] = field.to_internal_value(value["id"])
+        return value
+
+
+class ManageInstancePolicySerializer(serializers.ModelSerializer):
+    target = TargetSerializer()
+    actor = federation_fields.ActorRelatedField(read_only=True)
+
+    class Meta:
+        model = moderation_models.InstancePolicy
+        fields = [
+            "id",
+            "uuid",
+            "target",
+            "creation_date",
+            "actor",
+            "summary",
+            "is_active",
+            "block_all",
+            "silence_activity",
+            "silence_notifications",
+            "reject_media",
+        ]
+
+        read_only_fields = ["uuid", "id", "creation_date", "actor", "target"]
+
+    def validate(self, data):
+        try:
+            target = data.pop("target")
+        except KeyError:
+            # partial update
+            return data
+        if target["type"] == "domain":
+            data["target_domain"] = target["obj"]
+        if target["type"] == "actor":
+            data["target_actor"] = target["obj"]
+
+        return data
+
+    @transaction.atomic
+    def save(self, *args, **kwargs):
+        instance = super().save(*args, **kwargs)
+        need_purge = self.instance.is_active and (
+            self.instance.block_all or self.instance.reject_media
+        )
+        if need_purge:
+            only = []
+            if self.instance.reject_media:
+                only.append("media")
+            target = instance.target
+            if target["type"] == "domain":
+                common_utils.on_commit(
+                    federation_tasks.purge_actors.delay,
+                    domains=[target["obj"].pk],
+                    only=only,
+                )
+            if target["type"] == "actor":
+                common_utils.on_commit(
+                    federation_tasks.purge_actors.delay,
+                    ids=[target["obj"].pk],
+                    only=only,
+                )
+
+        return instance

api/funkwhale_api/manage/urls.py

@@ -5,8 +5,15 @@ from . import views
 
 federation_router = routers.SimpleRouter()
 federation_router.register(r"domains", views.ManageDomainViewSet, "domains")
+
 library_router = routers.SimpleRouter()
 library_router.register(r"uploads", views.ManageUploadViewSet, "uploads")
+
+moderation_router = routers.SimpleRouter()
+moderation_router.register(
+    r"instance-policies", views.ManageInstancePolicyViewSet, "instance-policies"
+)
+
 users_router = routers.SimpleRouter()
 users_router.register(r"users", views.ManageUserViewSet, "users")
 users_router.register(r"invitations", views.ManageInvitationViewSet, "invitations")
@@ -20,5 +27,9 @@ urlpatterns = [
         include((federation_router.urls, "federation"), namespace="federation"),
     ),
     url(r"^library/", include((library_router.urls, "instance"), namespace="library")),
+    url(
+        r"^moderation/",
+        include((moderation_router.urls, "moderation"), namespace="moderation"),
+    ),
     url(r"^users/", include((users_router.urls, "instance"), namespace="users")),
 ] + other_router.urls

api/funkwhale_api/manage/views.py

@@ -2,10 +2,11 @@ from rest_framework import mixins, response, viewsets
 from rest_framework.decorators import detail_route, list_route
 from django.shortcuts import get_object_or_404
 
-from funkwhale_api.common import preferences
+from funkwhale_api.common import preferences, decorators
 from funkwhale_api.federation import models as federation_models
 from funkwhale_api.federation import tasks as federation_tasks
 from funkwhale_api.music import models as music_models
+from funkwhale_api.moderation import models as moderation_models
 from funkwhale_api.users import models as users_models
 from funkwhale_api.users.permissions import HasUserPermission
 
@@ -98,13 +99,17 @@ class ManageInvitationViewSet(
 
 
 class ManageDomainViewSet(
-    mixins.ListModelMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet
+    mixins.CreateModelMixin,
+    mixins.ListModelMixin,
+    mixins.RetrieveModelMixin,
+    viewsets.GenericViewSet,
 ):
     lookup_value_regex = r"[a-zA-Z0-9\-\.]+"
     queryset = (
         federation_models.Domain.objects.external()
         .with_actors_count()
         .with_outbox_activities_count()
+        .prefetch_related("instance_policy")
         .order_by("name")
     )
     serializer_class = serializers.ManageDomainSerializer
@@ -117,6 +122,7 @@ class ManageDomainViewSet(
         "nodeinfo_fetch_date",
         "actors_count",
         "outbox_activities_count",
+        "instance_policy",
     ]
 
     @detail_route(methods=["get"])
@@ -131,6 +137,8 @@ class ManageDomainViewSet(
         domain = self.get_object()
         return response.Response(domain.get_stats(), status=200)
 
+    action = decorators.action_route(serializers.ManageDomainActionSerializer)
+
 
 class ManageActorViewSet(
     mixins.ListModelMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet
@@ -141,6 +149,7 @@ class ManageActorViewSet(
         .with_uploads_count()
         .order_by("-creation_date")
         .select_related("user")
+        .prefetch_related("instance_policy")
     )
     serializer_class = serializers.ManageActorSerializer
     filter_class = filters.ManageActorFilterSet
@@ -155,6 +164,7 @@ class ManageActorViewSet(
         "last_fetch_date",
         "uploads_count",
         "outbox_activities_count",
+        "instance_policy",
     ]
 
     def get_object(self):
@@ -170,3 +180,28 @@ class ManageActorViewSet(
     def stats(self, request, *args, **kwargs):
         domain = self.get_object()
         return response.Response(domain.get_stats(), status=200)
+
+    action = decorators.action_route(serializers.ManageActorActionSerializer)
+
+
+class ManageInstancePolicyViewSet(
+    mixins.ListModelMixin,
+    mixins.RetrieveModelMixin,
+    mixins.DestroyModelMixin,
+    mixins.CreateModelMixin,
+    mixins.UpdateModelMixin,
+    viewsets.GenericViewSet,
+):
+    queryset = (
+        moderation_models.InstancePolicy.objects.all()
+        .order_by("-creation_date")
+        .select_related()
+    )
+    serializer_class = serializers.ManageInstancePolicySerializer
+    filter_class = filters.ManageInstancePolicyFilterSet
+    permission_classes = (HasUserPermission,)
+    required_permissions = ["moderation"]
+    ordering_fields = ["id", "creation_date"]
+
+    def perform_create(self, serializer):
+        serializer.save(actor=self.request.user.actor)