From 233ac870beaa1f807827d02ae8a5969db37d95fa Mon Sep 17 00:00:00 2001
From: Eliot Berriot <contact@eliotberriot.com>
Date: Wed, 9 Jan 2019 14:18:32 +0100
Subject: [PATCH] Added actions and tasks to purge domains and actors
---
api/funkwhale_api/common/decorators.py | 14 +++
api/funkwhale_api/common/serializers.py | 2 +-
api/funkwhale_api/federation/tasks.py | 36 ++++++++
api/funkwhale_api/manage/serializers.py | 54 +++++++++++-
api/funkwhale_api/manage/views.py | 6 +-
api/tests/federation/test_tasks.py | 41 +++++++++
api/tests/manage/test_serializers.py | 87 +++++++++++++++++++
.../manage/moderation/AccountsTable.vue | 11 +--
.../manage/moderation/DomainsTable.vue | 12 +--
.../manage/moderation/InstancePolicyForm.vue | 2 +-
10 files changed, 251 insertions(+), 14 deletions(-)
create mode 100644 api/funkwhale_api/common/decorators.py
diff --git a/api/funkwhale_api/common/decorators.py b/api/funkwhale_api/common/decorators.py
new file mode 100644
index 0000000000..5ecedc5121
--- /dev/null
+++ b/api/funkwhale_api/common/decorators.py
@@ -0,0 +1,14 @@
+from rest_framework import response
+from rest_framework.decorators import list_route
+
+
+def action_route(serializer_class):
+ @list_route(methods=["post"])
+ def action(self, request, *args, **kwargs):
+ queryset = self.get_queryset()
+ serializer = serializer_class(request.data, queryset=queryset)
+ serializer.is_valid(raise_exception=True)
+ result = serializer.save()
+ return response.Response(result, status=200)
+
+ return action
diff --git a/api/funkwhale_api/common/serializers.py b/api/funkwhale_api/common/serializers.py
index e7bbf8f1f0..fafa6152d0 100644
--- a/api/funkwhale_api/common/serializers.py
+++ b/api/funkwhale_api/common/serializers.py
@@ -123,7 +123,7 @@ class ActionSerializer(serializers.Serializer):
if type(value) in [list, tuple]:
return self.queryset.filter(
**{"{}__in".format(self.pk_field): value}
- ).order_by("id")
+ ).order_by(self.pk_field)
raise serializers.ValidationError(
"{} is not a valid value for objects. You must provide either a "
diff --git a/api/funkwhale_api/federation/tasks.py b/api/funkwhale_api/federation/tasks.py
index 4ed07aa25f..67f8fabc95 100644
--- a/api/funkwhale_api/federation/tasks.py
+++ b/api/funkwhale_api/federation/tasks.py
@@ -186,3 +186,39 @@ def update_domain_nodeinfo(domain):
domain.nodeinfo_fetch_date = now
domain.nodeinfo = nodeinfo
domain.save(update_fields=["nodeinfo", "nodeinfo_fetch_date"])
+
+
+def delete_qs(qs):
+ label = qs.model._meta.label
+ result = qs.delete()
+ related = sum(result[1].values())
+
+ logger.info(
+ "Purged %s %s objects (and %s related entities)", result[0], label, related
+ )
+
+
+def handle_purge_actors(ids):
+ # purge follows (received emitted)
+ delete_qs(models.LibraryFollow.objects.filter(target__actor_id__in=ids))
+ delete_qs(models.LibraryFollow.objects.filter(actor_id__in=ids))
+ delete_qs(models.Follow.objects.filter(target_id__in=ids))
+ delete_qs(models.Follow.objects.filter(actor_id__in=ids))
+
+ # purge audio content
+ delete_qs(music_models.Upload.objects.filter(library__actor_id__in=ids))
+ delete_qs(music_models.Library.objects.filter(actor_id__in=ids))
+
+ # purge remaining activities / deliveries
+ delete_qs(models.InboxItem.objects.filter(actor_id__in=ids))
+ delete_qs(models.Activity.objects.filter(actor_id__in=ids))
+
+
+@celery.app.task(name="federation.purge_actors")
+def purge_actors(ids=[], domains=[]):
+ actors = models.Actor.objects.filter(
+ Q(id__in=ids) | Q(domain_id__in=domains)
+ ).order_by("id")
+ found_ids = list(actors.values_list("id", flat=True))
+ logger.info("Starting purging %s accounts", len(found_ids))
+ handle_purge_actors(ids=found_ids)
diff --git a/api/funkwhale_api/manage/serializers.py b/api/funkwhale_api/manage/serializers.py
index 009f5c31da..6795b30dfe 100644
--- a/api/funkwhale_api/manage/serializers.py
+++ b/api/funkwhale_api/manage/serializers.py
@@ -3,8 +3,10 @@ from django.db import transaction
from rest_framework import serializers
from funkwhale_api.common import serializers as common_serializers
+from funkwhale_api.common import utils as common_utils
from funkwhale_api.federation import models as federation_models
from funkwhale_api.federation import fields as federation_fields
+from funkwhale_api.federation import tasks as federation_tasks
from funkwhale_api.moderation import models as moderation_models
from funkwhale_api.music import models as music_models
from funkwhale_api.users import models as users_models
@@ -203,6 +205,17 @@ class ManageDomainSerializer(serializers.ModelSerializer):
return getattr(o, "outbox_activities_count", 0)
+class ManageDomainActionSerializer(common_serializers.ActionSerializer):
+ actions = [common_serializers.Action("purge", allow_all=False)]
+ filterset_class = filters.ManageDomainFilterSet
+ pk_field = "name"
+
+ @transaction.atomic
+ def handle_purge(self, objects):
+ ids = objects.values_list("pk", flat=True)
+ common_utils.on_commit(federation_tasks.purge_actors.delay, domains=list(ids))
+
+
class ManageActorSerializer(serializers.ModelSerializer):
uploads_count = serializers.SerializerMethodField()
user = ManageUserSerializer()
@@ -235,6 +248,16 @@ class ManageActorSerializer(serializers.ModelSerializer):
return getattr(o, "uploads_count", 0)
+class ManageActorActionSerializer(common_serializers.ActionSerializer):
+ actions = [common_serializers.Action("purge", allow_all=False)]
+ filterset_class = filters.ManageActorFilterSet
+
+ @transaction.atomic
+ def handle_purge(self, objects):
+ ids = objects.values_list("id", flat=True)
+ common_utils.on_commit(federation_tasks.purge_actors.delay, ids=list(ids))
+
+
class TargetSerializer(serializers.Serializer):
type = serializers.ChoiceField(choices=["domain", "actor"])
id = serializers.CharField()
@@ -279,10 +302,39 @@ class ManageInstancePolicySerializer(serializers.ModelSerializer):
read_only_fields = ["uuid", "id", "creation_date", "actor", "target"]
def validate(self, data):
- target = data.pop("target")
+ try:
+ target = data.pop("target")
+ except KeyError:
+ # partial update
+ return data
if target["type"] == "domain":
data["target_domain"] = target["obj"]
if target["type"] == "actor":
data["target_actor"] = target["obj"]
return data
+
+ @transaction.atomic
+ def save(self, *args, **kwargs):
+ block_all = self.validated_data.get("block_all", False)
+ need_purge = (
+ # we purge when we create with block all
+ (not self.instance and block_all)
+ or
+ # or when block all value switch from False to True
+ (self.instance and block_all and not self.instance.block_all)
+ )
+ instance = super().save(*args, **kwargs)
+
+ if need_purge:
+ target = instance.target
+ if target["type"] == "domain":
+ common_utils.on_commit(
+ federation_tasks.purge_actors.delay, domains=[target["obj"].pk]
+ )
+ if target["type"] == "actor":
+ common_utils.on_commit(
+ federation_tasks.purge_actors.delay, ids=[target["obj"].pk]
+ )
+
+ return instance
diff --git a/api/funkwhale_api/manage/views.py b/api/funkwhale_api/manage/views.py
index f1fbf01a49..d460cf91cc 100644
--- a/api/funkwhale_api/manage/views.py
+++ b/api/funkwhale_api/manage/views.py
@@ -2,7 +2,7 @@ from rest_framework import mixins, response, viewsets
from rest_framework.decorators import detail_route, list_route
from django.shortcuts import get_object_or_404
-from funkwhale_api.common import preferences
+from funkwhale_api.common import preferences, decorators
from funkwhale_api.federation import models as federation_models
from funkwhale_api.federation import tasks as federation_tasks
from funkwhale_api.music import models as music_models
@@ -135,6 +135,8 @@ class ManageDomainViewSet(
domain = self.get_object()
return response.Response(domain.get_stats(), status=200)
+ action = decorators.action_route(serializers.ManageDomainActionSerializer)
+
class ManageActorViewSet(
mixins.ListModelMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet
@@ -175,6 +177,8 @@ class ManageActorViewSet(
domain = self.get_object()
return response.Response(domain.get_stats(), status=200)
+ action = decorators.action_route(serializers.ManageActorActionSerializer)
+
class ManageInstancePolicyViewSet(
mixins.ListModelMixin,
diff --git a/api/tests/federation/test_tasks.py b/api/tests/federation/test_tasks.py
index ad7a577ef0..e539810693 100644
--- a/api/tests/federation/test_tasks.py
+++ b/api/tests/federation/test_tasks.py
@@ -190,3 +190,44 @@ def test_update_domain_nodeinfo_error(factories, r_mock, now):
"status": "error",
"error": "500 Server Error: None for url: {}".format(wellknown_url),
}
+
+
+def test_handle_purge_actors(factories, mocker):
+ to_purge = factories["federation.Actor"]()
+ keeped = [
+ factories["music.Upload"](),
+ factories["federation.Activity"](),
+ factories["federation.InboxItem"](),
+ factories["federation.Follow"](),
+ factories["federation.LibraryFollow"](),
+ ]
+
+ library = factories["music.Library"](actor=to_purge)
+ deleted = [
+ library,
+ factories["music.Upload"](library=library),
+ factories["federation.Activity"](actor=to_purge),
+ factories["federation.InboxItem"](actor=to_purge),
+ factories["federation.Follow"](actor=to_purge),
+ factories["federation.LibraryFollow"](actor=to_purge),
+ ]
+
+ tasks.handle_purge_actors([to_purge.pk])
+
+ for k in keeped:
+ # this should not be deleted
+ k.refresh_from_db()
+
+ for d in deleted:
+ with pytest.raises(d.__class__.DoesNotExist):
+ d.refresh_from_db()
+
+
+def test_purge_actors(factories, mocker):
+ handle_purge_actors = mocker.spy(tasks, "handle_purge_actors")
+ factories["federation.Actor"]()
+ to_delete = factories["federation.Actor"]()
+ to_delete_domain = factories["federation.Actor"]()
+ tasks.purge_actors(ids=[to_delete.pk], domains=[to_delete_domain.domain.name])
+
+ handle_purge_actors.assert_called_once_with(ids=[to_delete.pk, to_delete_domain.pk])
diff --git a/api/tests/manage/test_serializers.py b/api/tests/manage/test_serializers.py
index 6dbd7ac3a1..36dbe509c8 100644
--- a/api/tests/manage/test_serializers.py
+++ b/api/tests/manage/test_serializers.py
@@ -1,6 +1,7 @@
import pytest
from funkwhale_api.manage import serializers
+from funkwhale_api.federation import tasks as federation_tasks
def test_manage_upload_action_delete(factories):
@@ -138,3 +139,89 @@ def test_instance_policy_serializer_save_domain(factories):
policy = serializer.save()
assert policy.target_domain == domain
+
+
+def test_manage_actor_action_purge(factories, mocker):
+ actors = factories["federation.Actor"].create_batch(size=3)
+ s = serializers.ManageActorActionSerializer(queryset=None)
+ on_commit = mocker.patch("funkwhale_api.common.utils.on_commit")
+
+ s.handle_purge(actors[0].__class__.objects.all())
+ on_commit.assert_called_once_with(
+ federation_tasks.purge_actors.delay, ids=[a.pk for a in actors]
+ )
+
+
+def test_manage_domain_action_purge(factories, mocker):
+ domains = factories["federation.Domain"].create_batch(size=3)
+ s = serializers.ManageDomainActionSerializer(queryset=None)
+ on_commit = mocker.patch("funkwhale_api.common.utils.on_commit")
+
+ s.handle_purge(domains[0].__class__.objects.all())
+ on_commit.assert_called_once_with(
+ federation_tasks.purge_actors.delay, domains=[d.pk for d in domains]
+ )
+
+
+def test_instance_policy_serializer_purges_target_domain(factories, mocker):
+ policy = factories["moderation.InstancePolicy"](for_domain=True, block_all=False)
+ on_commit = mocker.patch("funkwhale_api.common.utils.on_commit")
+
+ serializer = serializers.ManageInstancePolicySerializer(
+ policy, data={"block_all": True}, partial=True
+ )
+ serializer.is_valid(raise_exception=True)
+ serializer.save()
+
+ policy.refresh_from_db()
+
+ assert policy.block_all is True
+ on_commit.assert_called_once_with(
+ federation_tasks.purge_actors.delay, domains=[policy.target_domain_id]
+ )
+
+ on_commit.reset_mock()
+
+ # setting to false should have no effect
+ serializer = serializers.ManageInstancePolicySerializer(
+ policy, data={"block_all": False}, partial=True
+ )
+ serializer.is_valid(raise_exception=True)
+ serializer.save()
+
+ policy.refresh_from_db()
+
+ assert policy.block_all is False
+ assert on_commit.call_count == 0
+
+
+def test_instance_policy_serializer_purges_target_actor(factories, mocker):
+ policy = factories["moderation.InstancePolicy"](for_actor=True, block_all=False)
+ on_commit = mocker.patch("funkwhale_api.common.utils.on_commit")
+
+ serializer = serializers.ManageInstancePolicySerializer(
+ policy, data={"block_all": True}, partial=True
+ )
+ serializer.is_valid(raise_exception=True)
+ serializer.save()
+
+ policy.refresh_from_db()
+
+ assert policy.block_all is True
+ on_commit.assert_called_once_with(
+ federation_tasks.purge_actors.delay, ids=[policy.target_actor_id]
+ )
+
+ on_commit.reset_mock()
+
+ # setting to false should have no effect
+ serializer = serializers.ManageInstancePolicySerializer(
+ policy, data={"block_all": False}, partial=True
+ )
+ serializer.is_valid(raise_exception=True)
+ serializer.save()
+
+ policy.refresh_from_db()
+
+ assert policy.block_all is False
+ assert on_commit.call_count == 0
diff --git a/front/src/components/manage/moderation/AccountsTable.vue b/front/src/components/manage/moderation/AccountsTable.vue
index 8750b4ec97..a0bf541606 100644
--- a/front/src/components/manage/moderation/AccountsTable.vue
+++ b/front/src/components/manage/moderation/AccountsTable.vue
@@ -78,6 +78,7 @@
:current="page"
:paginate-by="paginateBy"
:total="result.count"
+ action-url="manage/accounts/action/"
></pagination>
<span v-if="result && result.results.length > 0">
@@ -178,11 +179,11 @@ export default {
},
actions () {
return [
- // {
- // name: 'delete',
- // label: this.$gettext('Delete'),
- // isDangerous: true
- // }
+ {
+ name: 'purge',
+ label: this.$gettext('Purge'),
+ isDangerous: true
+ }
]
}
},
diff --git a/front/src/components/manage/moderation/DomainsTable.vue b/front/src/components/manage/moderation/DomainsTable.vue
index fd6a2bd46a..7b9a1c0233 100644
--- a/front/src/components/manage/moderation/DomainsTable.vue
+++ b/front/src/components/manage/moderation/DomainsTable.vue
@@ -32,6 +32,8 @@
@action-launched="fetchData"
:objects-data="result"
:actions="actions"
+ action-url="manage/federation/domains/action/"
+ idField="name"
:filters="actionFilters">
<template slot="header-cells">
<th><translate>Name</translate></th>
@@ -157,11 +159,11 @@ export default {
},
actions () {
return [
- // {
- // name: 'delete',
- // label: this.$gettext('Delete'),
- // isDangerous: true
- // }
+ {
+ name: 'purge',
+ label: this.$gettext('Purge'),
+ isDangerous: true
+ }
]
}
},
diff --git a/front/src/components/manage/moderation/InstancePolicyForm.vue b/front/src/components/manage/moderation/InstancePolicyForm.vue
index d3c8d6d6e7..fceb8884e0 100644
--- a/front/src/components/manage/moderation/InstancePolicyForm.vue
+++ b/front/src/components/manage/moderation/InstancePolicyForm.vue
@@ -107,7 +107,7 @@ export default {
return {
summaryHelp: this.$gettext("Explain why you're applying this policy. Depending on your instance configuration, this will help you remember why you acted on this account or domain, and may be displayed publicly to help users understand what moderation rules are in place."),
isActiveHelp: this.$gettext("Use this setting to temporarily enable/disable the policy without completely removing it."),
- blockAllHelp: this.$gettext("Block everything from this account or domain. This will prevent any interaction with the entity."),
+ blockAllHelp: this.$gettext("Block everything from this account or domain. This will prevent any interaction with the entity, and purge related content (uploads, libraries, follows, etc.)"),
silenceActivity: {
help: this.$gettext("Hide account or domain content, except from followers."),
label: this.$gettext("Silence activity"),
--
GitLab