Commit 9aa12db6 authored by Eliot Berriot's avatar Eliot Berriot
Browse files

See #170: Funkwhale federation

parent fce4d875
...@@ -107,6 +107,4 @@ def generate_actor(username, **kwargs): ...@@ -107,6 +107,4 @@ def generate_actor(username, **kwargs):
@receiver(post_delete, sender=Channel) @receiver(post_delete, sender=Channel)
def delete_channel_related_objs(instance, **kwargs): def delete_channel_related_objs(instance, **kwargs):
instance.library.delete() instance.library.delete()
if instance.actor != instance.attributed_to:
instance.actor.delete()
instance.artist.delete() instance.artist.delete()
...@@ -13,10 +13,12 @@ from django.utils import timezone ...@@ -13,10 +13,12 @@ from django.utils import timezone
from funkwhale_api.common import locales from funkwhale_api.common import locales
from funkwhale_api.common import permissions from funkwhale_api.common import permissions
from funkwhale_api.common import preferences from funkwhale_api.common import preferences
from funkwhale_api.common import utils as common_utils
from funkwhale_api.common.mixins import MultipleLookupDetailMixin from funkwhale_api.common.mixins import MultipleLookupDetailMixin
from funkwhale_api.federation import actors from funkwhale_api.federation import actors
from funkwhale_api.federation import models as federation_models from funkwhale_api.federation import models as federation_models
from funkwhale_api.federation import routes from funkwhale_api.federation import routes
from funkwhale_api.federation import tasks as federation_tasks
from funkwhale_api.federation import utils as federation_utils from funkwhale_api.federation import utils as federation_utils
from funkwhale_api.music import models as music_models from funkwhale_api.music import models as music_models
from funkwhale_api.music import views as music_views from funkwhale_api.music import views as music_views
...@@ -128,6 +130,8 @@ class ChannelViewSet( ...@@ -128,6 +130,8 @@ class ChannelViewSet(
) )
# prefetch stuff # prefetch stuff
subscription = SubscriptionsViewSet.queryset.get(pk=subscription.pk) subscription = SubscriptionsViewSet.queryset.get(pk=subscription.pk)
if not object.actor.is_local:
routes.outbox.dispatch({"type": "Follow"}, context={"follow": subscription})
data = serializers.SubscriptionSerializer(subscription).data data = serializers.SubscriptionSerializer(subscription).data
return response.Response(data, status=201) return response.Response(data, status=201)
...@@ -139,7 +143,15 @@ class ChannelViewSet( ...@@ -139,7 +143,15 @@ class ChannelViewSet(
) )
def unsubscribe(self, request, *args, **kwargs): def unsubscribe(self, request, *args, **kwargs):
object = self.get_object() object = self.get_object()
request.user.actor.emitted_follows.filter(target=object.actor).delete() follow_qs = request.user.actor.emitted_follows.filter(target=object.actor)
follow = follow_qs.first()
if follow:
if not object.actor.is_local:
routes.outbox.dispatch(
{"type": "Undo", "object": {"type": "Follow"}},
context={"follow": follow},
)
follow_qs.delete()
return response.Response(status=204) return response.Response(status=204)
@decorators.action( @decorators.action(
...@@ -248,11 +260,10 @@ class ChannelViewSet( ...@@ -248,11 +260,10 @@ class ChannelViewSet(
@transaction.atomic @transaction.atomic
def perform_destroy(self, instance): def perform_destroy(self, instance):
routes.outbox.dispatch(
{"type": "Delete", "object": {"type": instance.actor.type}},
context={"actor": instance.actor},
)
instance.__class__.objects.filter(pk=instance.pk).delete() instance.__class__.objects.filter(pk=instance.pk).delete()
common_utils.on_commit(
federation_tasks.remove_actor.delay, actor_id=instance.actor.pk
)
class SubscriptionsViewSet( class SubscriptionsViewSet(
......
...@@ -7,6 +7,7 @@ from django.utils import timezone ...@@ -7,6 +7,7 @@ from django.utils import timezone
from rest_framework import serializers from rest_framework import serializers
from funkwhale_api.audio import models as audio_models
from funkwhale_api.common import fields as common_fields from funkwhale_api.common import fields as common_fields
from funkwhale_api.common import serializers as common_serializers from funkwhale_api.common import serializers as common_serializers
from funkwhale_api.music import models as music_models from funkwhale_api.music import models as music_models
...@@ -171,6 +172,7 @@ FETCH_OBJECT_CONFIG = { ...@@ -171,6 +172,7 @@ FETCH_OBJECT_CONFIG = {
"library": {"queryset": music_models.Library.objects.all(), "id_attr": "uuid"}, "library": {"queryset": music_models.Library.objects.all(), "id_attr": "uuid"},
"upload": {"queryset": music_models.Upload.objects.all(), "id_attr": "uuid"}, "upload": {"queryset": music_models.Upload.objects.all(), "id_attr": "uuid"},
"account": {"queryset": models.Actor.objects.all(), "id_attr": "full_username"}, "account": {"queryset": models.Actor.objects.all(), "id_attr": "full_username"},
"channel": {"queryset": audio_models.Channel.objects.all(), "id_attr": "uuid"},
} }
FETCH_OBJECT_FIELD = common_fields.GenericRelation(FETCH_OBJECT_CONFIG) FETCH_OBJECT_FIELD = common_fields.GenericRelation(FETCH_OBJECT_CONFIG)
......
from . import schema_org
CONTEXTS = [ CONTEXTS = [
{ {
"shortId": "LDP", "shortId": "LDP",
...@@ -218,6 +220,12 @@ CONTEXTS = [ ...@@ -218,6 +220,12 @@ CONTEXTS = [
} }
}, },
}, },
{
"shortId": "SC",
"contextUrl": None,
"documentUrl": "http://schema.org",
"document": {"@context": schema_org.CONTEXT},
},
{ {
"shortId": "SEC", "shortId": "SEC",
"contextUrl": None, "contextUrl": None,
...@@ -280,6 +288,7 @@ CONTEXTS = [ ...@@ -280,6 +288,7 @@ CONTEXTS = [
"type": "@type", "type": "@type",
"as": "https://www.w3.org/ns/activitystreams#", "as": "https://www.w3.org/ns/activitystreams#",
"fw": "https://funkwhale.audio/ns#", "fw": "https://funkwhale.audio/ns#",
"schema": "http://schema.org#",
"xsd": "http://www.w3.org/2001/XMLSchema#", "xsd": "http://www.w3.org/2001/XMLSchema#",
"Album": "fw:Album", "Album": "fw:Album",
"Track": "fw:Track", "Track": "fw:Track",
...@@ -298,6 +307,8 @@ CONTEXTS = [ ...@@ -298,6 +307,8 @@ CONTEXTS = [
"musicbrainzId": "fw:musicbrainzId", "musicbrainzId": "fw:musicbrainzId",
"license": {"@id": "fw:license", "@type": "@id"}, "license": {"@id": "fw:license", "@type": "@id"},
"copyright": "fw:copyright", "copyright": "fw:copyright",
"category": "schema:category",
"language": "schema:inLanguage",
} }
}, },
}, },
...@@ -364,4 +375,5 @@ AS = NS(CONTEXTS_BY_ID["AS"]) ...@@ -364,4 +375,5 @@ AS = NS(CONTEXTS_BY_ID["AS"])
LDP = NS(CONTEXTS_BY_ID["LDP"]) LDP = NS(CONTEXTS_BY_ID["LDP"])
SEC = NS(CONTEXTS_BY_ID["SEC"]) SEC = NS(CONTEXTS_BY_ID["SEC"])
FW = NS(CONTEXTS_BY_ID["FW"]) FW = NS(CONTEXTS_BY_ID["FW"])
SC = NS(CONTEXTS_BY_ID["SC"])
LITEPUB = NS(CONTEXTS_BY_ID["LITEPUB"]) LITEPUB = NS(CONTEXTS_BY_ID["LITEPUB"])
import logging import logging
from django.db.models import Q
from funkwhale_api.music import models as music_models from funkwhale_api.music import models as music_models
from . import activity from . import activity
...@@ -158,18 +160,26 @@ def outbox_create_audio(context): ...@@ -158,18 +160,26 @@ def outbox_create_audio(context):
@inbox.register({"type": "Create", "object.type": "Audio"}) @inbox.register({"type": "Create", "object.type": "Audio"})
def inbox_create_audio(payload, context): def inbox_create_audio(payload, context):
serializer = serializers.UploadSerializer( is_channel = "library" not in payload["object"]
data=payload["object"], if is_channel:
context={"activity": context.get("activity"), "actor": context["actor"]}, channel = context["actor"].get_channel()
) serializer = serializers.ChannelUploadSerializer(
data=payload["object"], context={"channel": channel},
)
else:
serializer = serializers.UploadSerializer(
data=payload["object"],
context={"activity": context.get("activity"), "actor": context["actor"]},
)
if not serializer.is_valid(raise_exception=context.get("raise_exception", False)): if not serializer.is_valid(raise_exception=context.get("raise_exception", False)):
logger.warn("Discarding invalid audio create") logger.warn("Discarding invalid audio create")
return return
upload = serializer.save() upload = serializer.save()
if is_channel:
return {"object": upload, "target": upload.library} return {"object": upload, "target": channel}
else:
return {"object": upload, "target": upload.library}
@inbox.register({"type": "Delete", "object.type": "Library"}) @inbox.register({"type": "Delete", "object.type": "Library"})
...@@ -252,9 +262,10 @@ def inbox_delete_audio(payload, context): ...@@ -252,9 +262,10 @@ def inbox_delete_audio(payload, context):
# we did not receive a list of Ids, so we can probably use the value directly # we did not receive a list of Ids, so we can probably use the value directly
upload_fids = [payload["object"]["id"]] upload_fids = [payload["object"]["id"]]
candidates = music_models.Upload.objects.filter( query = Q(fid__in=upload_fids) & (
library__actor=actor, fid__in=upload_fids Q(library__actor=actor) | Q(track__artist__channel__actor=actor)
) )
candidates = music_models.Upload.objects.filter(query)
total = candidates.count() total = candidates.count()
logger.info("Deleting %s uploads with ids %s", total, upload_fids) logger.info("Deleting %s uploads with ids %s", total, upload_fids)
...@@ -483,3 +494,44 @@ def outbox_flag(context): ...@@ -483,3 +494,44 @@ def outbox_flag(context):
to=[{"type": "actor_inbox", "actor": report.target_owner}], to=[{"type": "actor_inbox", "actor": report.target_owner}],
), ),
} }
@inbox.register({"type": "Delete", "object.type": "Album"})
def inbox_delete_album(payload, context):
actor = context["actor"]
album_id = payload["object"].get("id")
if not album_id:
logger.debug("Discarding deletion of empty library")
return
query = Q(fid=album_id) & (Q(attributed_to=actor) | Q(artist__channel__actor=actor))
try:
album = music_models.Album.objects.get(query)
except music_models.Album.DoesNotExist:
logger.debug("Discarding deletion of unkwnown album %s", album_id)
return
album.delete()
@outbox.register({"type": "Delete", "object.type": "Album"})
def outbox_delete_album(context):
album = context["album"]
actor = (
album.artist.channel.actor
if album.artist.get_channel()
else album.attributed_to
)
actor = actor or actors.get_service_actor()
serializer = serializers.ActivitySerializer(
{"type": "Delete", "object": {"type": "Album", "id": album.fid}}
)
yield {
"type": "Delete",
"actor": actor,
"payload": with_recipients(
serializer.data,
to=[activity.PUBLIC_ADDRESS, {"type": "instances_with_followers"}],
),
}
This diff is collapsed.
This diff is collapsed.
...@@ -7,11 +7,14 @@ import requests ...@@ -7,11 +7,14 @@ import requests
from django.conf import settings from django.conf import settings
from django.db import transaction from django.db import transaction
from django.db.models import Q, F from django.db.models import Q, F
from django.db.models.deletion import Collector
from django.utils import timezone from django.utils import timezone
from dynamic_preferences.registries import global_preferences_registry from dynamic_preferences.registries import global_preferences_registry
from requests.exceptions import RequestException from requests.exceptions import RequestException
from funkwhale_api.audio import models as audio_models
from funkwhale_api.common import preferences from funkwhale_api.common import preferences
from funkwhale_api.common import models as common_models
from funkwhale_api.common import session from funkwhale_api.common import session
from funkwhale_api.common import utils as common_utils from funkwhale_api.common import utils as common_utils
from funkwhale_api.moderation import mrf from funkwhale_api.moderation import mrf
...@@ -254,8 +257,11 @@ def handle_purge_actors(ids, only=[]): ...@@ -254,8 +257,11 @@ def handle_purge_actors(ids, only=[]):
# purge audio content # purge audio content
if not only or "media" in only: if not only or "media" in only:
delete_qs(common_models.Attachment.objects.filter(actor__in=ids))
delete_qs(models.LibraryFollow.objects.filter(actor_id__in=ids)) delete_qs(models.LibraryFollow.objects.filter(actor_id__in=ids))
delete_qs(models.Follow.objects.filter(target_id__in=ids)) delete_qs(models.Follow.objects.filter(target_id__in=ids))
delete_qs(audio_models.Channel.objects.filter(attributed_to__in=ids))
delete_qs(audio_models.Channel.objects.filter(actor__in=ids))
delete_qs(music_models.Upload.objects.filter(library__actor_id__in=ids)) delete_qs(music_models.Upload.objects.filter(library__actor_id__in=ids))
delete_qs(music_models.Library.objects.filter(actor_id__in=ids)) delete_qs(music_models.Library.objects.filter(actor_id__in=ids))
...@@ -390,9 +396,76 @@ def fetch(fetch_obj): ...@@ -390,9 +396,76 @@ def fetch(fetch_obj):
error("save", message=str(e)) error("save", message=str(e))
raise raise
# special case for channels
# when obj is an actor, we check if the actor has a channel associated with it
# if it is the case, we consider the fetch obj to be a channel instead
if isinstance(obj, models.Actor) and obj.get_channel():
obj = obj.get_channel()
fetch_obj.object = obj fetch_obj.object = obj
fetch_obj.status = "finished" fetch_obj.status = "finished"
fetch_obj.fetch_date = timezone.now() fetch_obj.fetch_date = timezone.now()
return fetch_obj.save( return fetch_obj.save(
update_fields=["fetch_date", "status", "object_id", "object_content_type"] update_fields=["fetch_date", "status", "object_id", "object_content_type"]
) )
class PreserveSomeDataCollector(Collector):
"""
We need to delete everything related to an actor. Well… Almost everything.
But definitely not the Delete Activity we send to announce the actor is deleted.
"""
def __init__(self, *args, **kwargs):
self.creation_date = timezone.now()
super().__init__(*args, **kwargs)
def related_objects(self, related, *args, **kwargs):
qs = super().related_objects(related, *args, **kwargs)
if related.name == "outbox_activities":
# exclude the delete activity can be broadcasted properly
qs = qs.exclude(type="Delete", creation_date__gte=self.creation_date)
return qs
@celery.app.task(name="federation.remove_actor")
@transaction.atomic
@celery.require_instance(
models.Actor.objects.all(), "actor",
)
def remove_actor(actor):
# Then we broadcast the info over federation. We do this *before* deleting objects
# associated with the actor, otherwise follows are removed and we don't know where
# to broadcast
logger.info("Broadcasting deletion to federation…")
collector = PreserveSomeDataCollector(using="default")
routes.outbox.dispatch(
{"type": "Delete", "object": {"type": actor.type}}, context={"actor": actor}
)
# then we delete any object associated with the actor object, but *not* the actor
# itself. We keep it for auditability and sending the Delete ActivityPub message
logger.info(
"Prepare deletion of objects associated with account %s…",
actor.preferred_username,
)
collector.collect([actor])
for model, instances in collector.data.items():
if issubclass(model, actor.__class__):
# we skip deletion of the actor itself
continue
to_delete = model.objects.filter(pk__in=[instance.pk for instance in instances])
logger.info(
"Deleting %s objects associated with account %s…",
len(instances),
actor.preferred_username,
)
to_delete.delete()
# Finally, we update the actor itself and mark it as removed
logger.info("Marking actor as Tombsone…")
actor.type = "Tombstone"
actor.name = None
actor.summary = None
actor.save(update_fields=["type", "name", "summary"])
...@@ -67,7 +67,11 @@ class ActorViewSet(FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericV ...@@ -67,7 +67,11 @@ class ActorViewSet(FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericV
lookup_field = "preferred_username" lookup_field = "preferred_username"
authentication_classes = [authentication.SignatureAuthentication] authentication_classes = [authentication.SignatureAuthentication]
renderer_classes = renderers.get_ap_renderers() renderer_classes = renderers.get_ap_renderers()
queryset = models.Actor.objects.local().select_related("user") queryset = (
models.Actor.objects.local()
.select_related("user", "channel__artist", "channel__attributed_to")
.prefetch_related("channel__artist__tagged_items__tag")
)
serializer_class = serializers.ActorSerializer serializer_class = serializers.ActorSerializer
def get_queryset(self): def get_queryset(self):
......
...@@ -241,6 +241,14 @@ class AlbumViewSet( ...@@ -241,6 +241,14 @@ class AlbumViewSet(
return serializers.AlbumCreateSerializer return serializers.AlbumCreateSerializer
return super().get_serializer_class() return super().get_serializer_class()
@transaction.atomic
def perform_destroy(self, instance):
routes.outbox.dispatch(
{"type": "Delete", "object": {"type": "Album"}},
context={"album": instance},
)
models.Album.objects.filter(pk=instance.pk).delete()
class LibraryViewSet( class LibraryViewSet(
mixins.CreateModelMixin, mixins.CreateModelMixin,
...@@ -380,6 +388,15 @@ class TrackViewSet( ...@@ -380,6 +388,15 @@ class TrackViewSet(
context["description"] = self.action in ["retrieve", "create", "update"] context["description"] = self.action in ["retrieve", "create", "update"]
return context return context
@transaction.atomic
def perform_destroy(self, instance):
uploads = instance.uploads.order_by("id")
routes.outbox.dispatch(
{"type": "Delete", "object": {"type": "Audio"}},
context={"uploads": list(uploads)},
)
instance.delete()
def strip_absolute_media_url(path): def strip_absolute_media_url(path):
if ( if (
......
import logging import logging
from django.db.models.deletion import Collector from funkwhale_api.federation import tasks as federation_tasks
from funkwhale_api.federation import routes
from funkwhale_api.taskapp import celery from funkwhale_api.taskapp import celery
from . import models from . import models
...@@ -20,39 +18,6 @@ def delete_account(user): ...@@ -20,39 +18,6 @@ def delete_account(user):
user.delete() user.delete()
logger.info("Deleted user object") logger.info("Deleted user object")
# Then we broadcast the info over federation. We do this *before* deleting objects # ensure actor is set to tombstone, activities are removed, etc.
# associated with the actor, otherwise follows are removed and we don't know where federation_tasks.remove_actor(actor_id=actor.pk)
# to broadcast logger.info("Deletion of account done %s!", actor.preferred_username)
logger.info("Broadcasting deletion to federation…")
routes.outbox.dispatch(
{"type": "Delete", "object": {"type": actor.type}}, context={"actor": actor}
)
# then we delete any object associated with the actor object, but *not* the actor
# itself. We keep it for auditability and sending the Delete ActivityPub message
collector = Collector(using="default")
logger.info(
"Prepare deletion of objects associated with account %s…", user.username
)
collector.collect([actor])
for model, instances in collector.data.items():
if issubclass(model, actor.__class__):
# we skip deletion of the actor itself
continue
logger.info(
"Deleting %s objects associated with account %s…",
len(instances),
user.username,
)
to_delete = model.objects.filter(pk__in=[instance.pk for instance in instances])
to_delete.delete()
# Finally, we update the actor itself and mark it as removed
logger.info("Marking actor as Tombsone…")
actor.type = "Tombstone"
actor.name = None
actor.summary = None
actor.save(update_fields=["type", "name", "summary"])
logger.info("Deletion of account done %s!", user.username)
...@@ -148,19 +148,17 @@ def test_channel_delete(logged_in_api_client, factories, mocker): ...@@ -148,19 +148,17 @@ def test_channel_delete(logged_in_api_client, factories, mocker):
channel = factories["audio.Channel"](attributed_to=actor) channel = factories["audio.Channel"](attributed_to=actor)
url = reverse("api:v1:channels-detail", kwargs={"composite": channel.uuid}) url = reverse("api:v1:channels-detail", kwargs={"composite": channel.uuid})
dispatch = mocker.patch("funkwhale_api.federation.routes.outbox.dispatch") on_commit = mocker.patch("funkwhale_api.common.utils.on_commit")
response = logged_in_api_client.delete(url) response = logged_in_api_client.delete(url)
assert response.status_code == 204 assert response.status_code == 204
on_commit.assert_called_once_with(
views.federation_tasks.remove_actor.delay, actor_id=channel.actor.pk
)
with pytest.raises(channel.DoesNotExist): with pytest.raises(channel.DoesNotExist):
channel.refresh_from_db() channel.refresh_from_db()
dispatch.assert_called_once_with(
{"type": "Delete", "object": {"type": channel.actor.type}},
context={"actor": channel.actor},
)
def test_channel_delete_permission(logged_in_api_client, factories): def test_channel_delete_permission(logged_in_api_client, factories):
logged_in_api_client.user.create_actor() logged_in_api_client.user.create_actor()
...@@ -218,6 +216,38 @@ def test_channel_unsubscribe(factories, logged_in_api_client): ...@@ -218,6 +216,38 @@ def test_channel_unsubscribe(factories, logged_in_api_client):
subscription.refresh_from_db() subscription.refresh_from_db()
def test_channel_subscribe_remote(factories, logged_in_api_client, mocker):
dispatch = mocker.patch("funkwhale_api.federation.routes.outbox.dispatch")
actor = logged_in_api_client.user.create_actor()
channel_actor = factories["federation.Actor"]()
channel = factories["audio.Channel"](artist__description=None, actor=channel_actor)
url = reverse("api:v1:channels-subscribe", kwargs={"composite": channel.uuid})
response = logged_in_api_client.post(url)
assert response.status_code == 201
subscription = actor.emitted_follows.latest("id")
dispatch.assert_called_once_with(
{"type": "F