Commit 9aa12db6 authored by Agate's avatar Agate 💬

See #170: Funkwhale federation

parent fce4d875
......@@ -107,6 +107,4 @@ def generate_actor(username, **kwargs):
@receiver(post_delete, sender=Channel)
def delete_channel_related_objs(instance, **kwargs):
instance.library.delete()
if instance.actor != instance.attributed_to:
instance.actor.delete()
instance.artist.delete()
......@@ -13,10 +13,12 @@ from django.utils import timezone
from funkwhale_api.common import locales
from funkwhale_api.common import permissions
from funkwhale_api.common import preferences
from funkwhale_api.common import utils as common_utils
from funkwhale_api.common.mixins import MultipleLookupDetailMixin
from funkwhale_api.federation import actors
from funkwhale_api.federation import models as federation_models
from funkwhale_api.federation import routes
from funkwhale_api.federation import tasks as federation_tasks
from funkwhale_api.federation import utils as federation_utils
from funkwhale_api.music import models as music_models
from funkwhale_api.music import views as music_views
......@@ -128,6 +130,8 @@ class ChannelViewSet(
)
# prefetch stuff
subscription = SubscriptionsViewSet.queryset.get(pk=subscription.pk)
if not object.actor.is_local:
routes.outbox.dispatch({"type": "Follow"}, context={"follow": subscription})
data = serializers.SubscriptionSerializer(subscription).data
return response.Response(data, status=201)
......@@ -139,7 +143,15 @@ class ChannelViewSet(
)
def unsubscribe(self, request, *args, **kwargs):
object = self.get_object()
request.user.actor.emitted_follows.filter(target=object.actor).delete()
follow_qs = request.user.actor.emitted_follows.filter(target=object.actor)
follow = follow_qs.first()
if follow:
if not object.actor.is_local:
routes.outbox.dispatch(
{"type": "Undo", "object": {"type": "Follow"}},
context={"follow": follow},
)
follow_qs.delete()
return response.Response(status=204)
@decorators.action(
......@@ -248,11 +260,10 @@ class ChannelViewSet(
@transaction.atomic
def perform_destroy(self, instance):
routes.outbox.dispatch(
{"type": "Delete", "object": {"type": instance.actor.type}},
context={"actor": instance.actor},
)
instance.__class__.objects.filter(pk=instance.pk).delete()
common_utils.on_commit(
federation_tasks.remove_actor.delay, actor_id=instance.actor.pk
)
class SubscriptionsViewSet(
......
......@@ -7,6 +7,7 @@ from django.utils import timezone
from rest_framework import serializers
from funkwhale_api.audio import models as audio_models
from funkwhale_api.common import fields as common_fields
from funkwhale_api.common import serializers as common_serializers
from funkwhale_api.music import models as music_models
......@@ -171,6 +172,7 @@ FETCH_OBJECT_CONFIG = {
"library": {"queryset": music_models.Library.objects.all(), "id_attr": "uuid"},
"upload": {"queryset": music_models.Upload.objects.all(), "id_attr": "uuid"},
"account": {"queryset": models.Actor.objects.all(), "id_attr": "full_username"},
"channel": {"queryset": audio_models.Channel.objects.all(), "id_attr": "uuid"},
}
FETCH_OBJECT_FIELD = common_fields.GenericRelation(FETCH_OBJECT_CONFIG)
......
from . import schema_org
CONTEXTS = [
{
"shortId": "LDP",
......@@ -218,6 +220,12 @@ CONTEXTS = [
}
},
},
{
"shortId": "SC",
"contextUrl": None,
"documentUrl": "http://schema.org",
"document": {"@context": schema_org.CONTEXT},
},
{
"shortId": "SEC",
"contextUrl": None,
......@@ -280,6 +288,7 @@ CONTEXTS = [
"type": "@type",
"as": "https://www.w3.org/ns/activitystreams#",
"fw": "https://funkwhale.audio/ns#",
"schema": "http://schema.org#",
"xsd": "http://www.w3.org/2001/XMLSchema#",
"Album": "fw:Album",
"Track": "fw:Track",
......@@ -298,6 +307,8 @@ CONTEXTS = [
"musicbrainzId": "fw:musicbrainzId",
"license": {"@id": "fw:license", "@type": "@id"},
"copyright": "fw:copyright",
"category": "schema:category",
"language": "schema:inLanguage",
}
},
},
......@@ -364,4 +375,5 @@ AS = NS(CONTEXTS_BY_ID["AS"])
LDP = NS(CONTEXTS_BY_ID["LDP"])
SEC = NS(CONTEXTS_BY_ID["SEC"])
FW = NS(CONTEXTS_BY_ID["FW"])
SC = NS(CONTEXTS_BY_ID["SC"])
LITEPUB = NS(CONTEXTS_BY_ID["LITEPUB"])
import logging
from django.db.models import Q
from funkwhale_api.music import models as music_models
from . import activity
......@@ -158,18 +160,26 @@ def outbox_create_audio(context):
@inbox.register({"type": "Create", "object.type": "Audio"})
def inbox_create_audio(payload, context):
serializer = serializers.UploadSerializer(
data=payload["object"],
context={"activity": context.get("activity"), "actor": context["actor"]},
)
is_channel = "library" not in payload["object"]
if is_channel:
channel = context["actor"].get_channel()
serializer = serializers.ChannelUploadSerializer(
data=payload["object"], context={"channel": channel},
)
else:
serializer = serializers.UploadSerializer(
data=payload["object"],
context={"activity": context.get("activity"), "actor": context["actor"]},
)
if not serializer.is_valid(raise_exception=context.get("raise_exception", False)):
logger.warn("Discarding invalid audio create")
return
upload = serializer.save()
return {"object": upload, "target": upload.library}
if is_channel:
return {"object": upload, "target": channel}
else:
return {"object": upload, "target": upload.library}
@inbox.register({"type": "Delete", "object.type": "Library"})
......@@ -252,9 +262,10 @@ def inbox_delete_audio(payload, context):
# we did not receive a list of Ids, so we can probably use the value directly
upload_fids = [payload["object"]["id"]]
candidates = music_models.Upload.objects.filter(
library__actor=actor, fid__in=upload_fids
query = Q(fid__in=upload_fids) & (
Q(library__actor=actor) | Q(track__artist__channel__actor=actor)
)
candidates = music_models.Upload.objects.filter(query)
total = candidates.count()
logger.info("Deleting %s uploads with ids %s", total, upload_fids)
......@@ -483,3 +494,44 @@ def outbox_flag(context):
to=[{"type": "actor_inbox", "actor": report.target_owner}],
),
}
@inbox.register({"type": "Delete", "object.type": "Album"})
def inbox_delete_album(payload, context):
actor = context["actor"]
album_id = payload["object"].get("id")
if not album_id:
logger.debug("Discarding deletion of empty library")
return
query = Q(fid=album_id) & (Q(attributed_to=actor) | Q(artist__channel__actor=actor))
try:
album = music_models.Album.objects.get(query)
except music_models.Album.DoesNotExist:
logger.debug("Discarding deletion of unkwnown album %s", album_id)
return
album.delete()
@outbox.register({"type": "Delete", "object.type": "Album"})
def outbox_delete_album(context):
album = context["album"]
actor = (
album.artist.channel.actor
if album.artist.get_channel()
else album.attributed_to
)
actor = actor or actors.get_service_actor()
serializer = serializers.ActivitySerializer(
{"type": "Delete", "object": {"type": "Album", "id": album.fid}}
)
yield {
"type": "Delete",
"actor": actor,
"payload": with_recipients(
serializer.data,
to=[activity.PUBLIC_ADDRESS, {"type": "instances_with_followers"}],
),
}
This source diff could not be displayed because it is too large. You can view the blob instead.
This diff is collapsed.
......@@ -7,11 +7,14 @@ import requests
from django.conf import settings
from django.db import transaction
from django.db.models import Q, F
from django.db.models.deletion import Collector
from django.utils import timezone
from dynamic_preferences.registries import global_preferences_registry
from requests.exceptions import RequestException
from funkwhale_api.audio import models as audio_models
from funkwhale_api.common import preferences
from funkwhale_api.common import models as common_models
from funkwhale_api.common import session
from funkwhale_api.common import utils as common_utils
from funkwhale_api.moderation import mrf
......@@ -254,8 +257,11 @@ def handle_purge_actors(ids, only=[]):
# purge audio content
if not only or "media" in only:
delete_qs(common_models.Attachment.objects.filter(actor__in=ids))
delete_qs(models.LibraryFollow.objects.filter(actor_id__in=ids))
delete_qs(models.Follow.objects.filter(target_id__in=ids))
delete_qs(audio_models.Channel.objects.filter(attributed_to__in=ids))
delete_qs(audio_models.Channel.objects.filter(actor__in=ids))
delete_qs(music_models.Upload.objects.filter(library__actor_id__in=ids))
delete_qs(music_models.Library.objects.filter(actor_id__in=ids))
......@@ -390,9 +396,76 @@ def fetch(fetch_obj):
error("save", message=str(e))
raise
# special case for channels
# when obj is an actor, we check if the actor has a channel associated with it
# if it is the case, we consider the fetch obj to be a channel instead
if isinstance(obj, models.Actor) and obj.get_channel():
obj = obj.get_channel()
fetch_obj.object = obj
fetch_obj.status = "finished"
fetch_obj.fetch_date = timezone.now()
return fetch_obj.save(
update_fields=["fetch_date", "status", "object_id", "object_content_type"]
)
class PreserveSomeDataCollector(Collector):
"""
We need to delete everything related to an actor. Well… Almost everything.
But definitely not the Delete Activity we send to announce the actor is deleted.
"""
def __init__(self, *args, **kwargs):
self.creation_date = timezone.now()
super().__init__(*args, **kwargs)
def related_objects(self, related, *args, **kwargs):
qs = super().related_objects(related, *args, **kwargs)
if related.name == "outbox_activities":
# exclude the delete activity can be broadcasted properly
qs = qs.exclude(type="Delete", creation_date__gte=self.creation_date)
return qs
@celery.app.task(name="federation.remove_actor")
@transaction.atomic
@celery.require_instance(
models.Actor.objects.all(), "actor",
)
def remove_actor(actor):
# Then we broadcast the info over federation. We do this *before* deleting objects
# associated with the actor, otherwise follows are removed and we don't know where
# to broadcast
logger.info("Broadcasting deletion to federation…")
collector = PreserveSomeDataCollector(using="default")
routes.outbox.dispatch(
{"type": "Delete", "object": {"type": actor.type}}, context={"actor": actor}
)
# then we delete any object associated with the actor object, but *not* the actor
# itself. We keep it for auditability and sending the Delete ActivityPub message
logger.info(
"Prepare deletion of objects associated with account %s…",
actor.preferred_username,
)
collector.collect([actor])
for model, instances in collector.data.items():
if issubclass(model, actor.__class__):
# we skip deletion of the actor itself
continue
to_delete = model.objects.filter(pk__in=[instance.pk for instance in instances])
logger.info(
"Deleting %s objects associated with account %s…",
len(instances),
actor.preferred_username,
)
to_delete.delete()
# Finally, we update the actor itself and mark it as removed
logger.info("Marking actor as Tombsone…")
actor.type = "Tombstone"
actor.name = None
actor.summary = None
actor.save(update_fields=["type", "name", "summary"])
......@@ -67,7 +67,11 @@ class ActorViewSet(FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericV
lookup_field = "preferred_username"
authentication_classes = [authentication.SignatureAuthentication]
renderer_classes = renderers.get_ap_renderers()
queryset = models.Actor.objects.local().select_related("user")
queryset = (
models.Actor.objects.local()
.select_related("user", "channel__artist", "channel__attributed_to")
.prefetch_related("channel__artist__tagged_items__tag")
)
serializer_class = serializers.ActorSerializer
def get_queryset(self):
......
......@@ -241,6 +241,14 @@ class AlbumViewSet(
return serializers.AlbumCreateSerializer
return super().get_serializer_class()
@transaction.atomic
def perform_destroy(self, instance):
routes.outbox.dispatch(
{"type": "Delete", "object": {"type": "Album"}},
context={"album": instance},
)
models.Album.objects.filter(pk=instance.pk).delete()
class LibraryViewSet(
mixins.CreateModelMixin,
......@@ -380,6 +388,15 @@ class TrackViewSet(
context["description"] = self.action in ["retrieve", "create", "update"]
return context
@transaction.atomic
def perform_destroy(self, instance):
uploads = instance.uploads.order_by("id")
routes.outbox.dispatch(
{"type": "Delete", "object": {"type": "Audio"}},
context={"uploads": list(uploads)},
)
instance.delete()
def strip_absolute_media_url(path):
if (
......
import logging
from django.db.models.deletion import Collector
from funkwhale_api.federation import routes
from funkwhale_api.federation import tasks as federation_tasks
from funkwhale_api.taskapp import celery
from . import models
......@@ -20,39 +18,6 @@ def delete_account(user):
user.delete()
logger.info("Deleted user object")
# Then we broadcast the info over federation. We do this *before* deleting objects
# associated with the actor, otherwise follows are removed and we don't know where
# to broadcast
logger.info("Broadcasting deletion to federation…")
routes.outbox.dispatch(
{"type": "Delete", "object": {"type": actor.type}}, context={"actor": actor}
)
# then we delete any object associated with the actor object, but *not* the actor
# itself. We keep it for auditability and sending the Delete ActivityPub message
collector = Collector(using="default")
logger.info(
"Prepare deletion of objects associated with account %s…", user.username
)
collector.collect([actor])
for model, instances in collector.data.items():
if issubclass(model, actor.__class__):
# we skip deletion of the actor itself
continue
logger.info(
"Deleting %s objects associated with account %s…",
len(instances),
user.username,
)
to_delete = model.objects.filter(pk__in=[instance.pk for instance in instances])
to_delete.delete()
# Finally, we update the actor itself and mark it as removed
logger.info("Marking actor as Tombsone…")
actor.type = "Tombstone"
actor.name = None
actor.summary = None
actor.save(update_fields=["type", "name", "summary"])
logger.info("Deletion of account done %s!", user.username)
# ensure actor is set to tombstone, activities are removed, etc.
federation_tasks.remove_actor(actor_id=actor.pk)
logger.info("Deletion of account done %s!", actor.preferred_username)
......@@ -148,19 +148,17 @@ def test_channel_delete(logged_in_api_client, factories, mocker):
channel = factories["audio.Channel"](attributed_to=actor)
url = reverse("api:v1:channels-detail", kwargs={"composite": channel.uuid})
dispatch = mocker.patch("funkwhale_api.federation.routes.outbox.dispatch")
on_commit = mocker.patch("funkwhale_api.common.utils.on_commit")
response = logged_in_api_client.delete(url)
assert response.status_code == 204
on_commit.assert_called_once_with(
views.federation_tasks.remove_actor.delay, actor_id=channel.actor.pk
)
with pytest.raises(channel.DoesNotExist):
channel.refresh_from_db()
dispatch.assert_called_once_with(
{"type": "Delete", "object": {"type": channel.actor.type}},
context={"actor": channel.actor},
)
def test_channel_delete_permission(logged_in_api_client, factories):
logged_in_api_client.user.create_actor()
......@@ -218,6 +216,38 @@ def test_channel_unsubscribe(factories, logged_in_api_client):
subscription.refresh_from_db()
def test_channel_subscribe_remote(factories, logged_in_api_client, mocker):
dispatch = mocker.patch("funkwhale_api.federation.routes.outbox.dispatch")
actor = logged_in_api_client.user.create_actor()
channel_actor = factories["federation.Actor"]()
channel = factories["audio.Channel"](artist__description=None, actor=channel_actor)
url = reverse("api:v1:channels-subscribe", kwargs={"composite": channel.uuid})
response = logged_in_api_client.post(url)
assert response.status_code == 201
subscription = actor.emitted_follows.latest("id")
dispatch.assert_called_once_with(
{"type": "Follow"}, context={"follow": subscription}
)
def test_channel_unsubscribe_remote(factories, logged_in_api_client, mocker):
dispatch = mocker.patch("funkwhale_api.federation.routes.outbox.dispatch")
actor = logged_in_api_client.user.create_actor()
channel_actor = factories["federation.Actor"]()
channel = factories["audio.Channel"](actor=channel_actor)
subscription = factories["audio.Subscription"](target=channel.actor, actor=actor)
url = reverse("api:v1:channels-unsubscribe", kwargs={"composite": channel.uuid})
response = logged_in_api_client.post(url)
assert response.status_code == 204
dispatch.assert_called_once_with(
{"type": "Undo", "object": {"type": "Follow"}}, context={"follow": subscription}
)
def test_subscriptions_list(factories, logged_in_api_client):
actor = logged_in_api_client.user.create_actor()
channel = factories["audio.Channel"](
......
......@@ -167,6 +167,7 @@ def test_fetch_serializer_no_obj(factories, to_api_date):
("music.Track", "track", "id"),
("music.Library", "library", "uuid"),
("music.Upload", "upload", "uuid"),
("audio.Channel", "channel", "uuid"),
("federation.Actor", "account", "full_username"),
],
)
......
......@@ -26,6 +26,7 @@ from funkwhale_api.moderation import serializers as moderation_serializers
routes.inbox_delete_library,
),
({"type": "Delete", "object": {"type": "Audio"}}, routes.inbox_delete_audio),
({"type": "Delete", "object": {"type": "Album"}}, routes.inbox_delete_album),
({"type": "Undo", "object": {"type": "Follow"}}, routes.inbox_undo_follow),
({"type": "Update", "object": {"type": "Artist"}}, routes.inbox_update_artist),
({"type": "Update", "object": {"type": "Album"}}, routes.inbox_update_album),
......@@ -58,6 +59,7 @@ def test_inbox_routes(route, handler):
routes.outbox_delete_library,
),
({"type": "Delete", "object": {"type": "Audio"}}, routes.outbox_delete_audio),
({"type": "Delete", "object": {"type": "Album"}}, routes.outbox_delete_album),
({"type": "Undo", "object": {"type": "Follow"}}, routes.outbox_undo_follow),
({"type": "Update", "object": {"type": "Track"}}, routes.outbox_update_track),
(
......@@ -349,6 +351,34 @@ def test_inbox_create_audio(factories, mocker):
assert save.call_count == 1
def test_inbox_create_audio_channel(factories, mocker):
activity = factories["federation.Activity"]()
channel = factories["audio.Channel"]()
album = factories["music.Album"](artist=channel.artist)
upload = factories["music.Upload"](track__album=album, library=channel.library,)
payload = {
"@context": jsonld.get_default_context(),
"type": "Create",
"actor": channel.actor.fid,
"object": serializers.ChannelUploadSerializer(upload).data,
}
upload.delete()
init = mocker.spy(serializers.ChannelUploadSerializer, "__init__")
save = mocker.spy(serializers.ChannelUploadSerializer, "save")
result = routes.inbox_create_audio(
payload,
context={"actor": channel.actor, "raise_exception": True, "activity": activity},
)
assert channel.library.uploads.count() == 1
assert result == {"object": channel.library.uploads.latest("id"), "target": channel}
assert init.call_count == 1
args = init.call_args
assert args[1]["data"] == payload["object"]
assert args[1]["context"] == {"channel": channel}
assert save.call_count == 1
def test_inbox_delete_library(factories):
activity = factories["federation.Activity"]()
......@@ -368,6 +398,73 @@ def test_inbox_delete_library(factories):
library.refresh_from_db()
def test_inbox_delete_album(factories):
album = factories["music.Album"](attributed=True)
payload = {
"type": "Delete",
"actor": album.attributed_to.fid,
"object": {"type": "Album", "id": album.fid},
}
routes.inbox_delete_album(
payload,
context={
"actor": album.attributed_to,
"raise_exception": True,
"activity": activity,
},
)
with pytest.raises(album.__class__.DoesNotExist):
album.refresh_from_db()
def test_inbox_delete_album_channel(factories):
channel = factories["audio.Channel"]()
album = factories["music.Album"](artist=channel.artist)
payload = {
"type": "Delete",
"actor": channel.actor.fid,
"object": {"type": "Album", "id": album.fid},
}
routes.inbox_delete_album(
payload,
context={"actor": channel.actor, "raise_exception": True, "activity": activity},
)
with pytest.raises(album.__class__.DoesNotExist):
album.refresh_from_db()
def test_outbox_delete_album(factories):
album = factories["music.Album"](attributed=True)
a = list(routes.outbox_delete_album({"album": album}))[0]
expected = serializers.ActivitySerializer(
{"type": "Delete", "object": {"type": "Album", "id": album.fid}}
).data
expected["to"] = [activity.PUBLIC_ADDRESS, {"type": "instances_with_followers"}]
assert dict(a["payload"]) == dict(expected)
assert a["actor"] == album.attributed_to
def test_outbox_delete_album_channel(factories):
channel = factories["audio.Channel"]()
album = factories["music.Album"](artist=channel.artist)
a = list(routes.outbox_delete_album({"album": album}))[0]
expected = serializers.ActivitySerializer(
{"type": "Delete", "object": {"type": "Album", "id": album.fid}}
).data
expected["to"] = [activity.PUBLIC_ADDRESS, {"type": "instances_with_followers"}]
assert dict(a["payload"]) == dict(expected)
assert a["actor"] == channel.actor
def test_inbox_delete_library_impostor(factories):
activity = factories["federation.Activity"]()
impostor = factories["federation.Actor"]()
......@@ -469,6 +566,25 @@ def test_inbox_delete_audio(factories):
upload.refresh_from_db()
def test_inbox_delete_audio_channel(factories):
activity = factories["federation.Activity"]()
channel = factories["audio.Channel"]()
upload = factories["music.Upload"](track__artist=channel.artist)
payload = {
"type": "Delete",
"actor": channel.actor.fid,
"object": {"type": "Audio", "id": [upload.fid]},
}
routes.inbox_delete_audio(
payload,
context={"actor": channel.actor, "raise_exception": True, "activity": activity},
)
with pytest.raises(upload.__class__.DoesNotExist):
upload.refresh_from_db()
def test_inbox_delete_audio_impostor(factories):
activity = factories["federation.Activity"]()
impostor = factories["federation.Actor"]()
......
This diff is collapsed.
......@@ -491,6 +491,21 @@ def test_fetch_url(factory_name, serializer_class, factories, r_mock, mocker):
assert save.call_count == 1
def test_fetch_channel_actor_returns_channel(factories, r_mock):
obj = factories["audio.Channel"]()
fetch = factories["federation.Fetch"](url=obj.actor.fid)
payload = serializers.ActorSerializer(obj.actor).data
r_mock.get(obj.fid, json=payload)
tasks.fetch(fetch_id=fetch.pk)
fetch.refresh_from_db()
assert fetch.status == "finished"
assert fetch.object == obj
def test_fetch_honor_instance_policy_domain(factories):
domain = factories["moderation.InstancePolicy"](
block_all=True, for_domain=True
......
......@@ -1407,7 +1407,8 @@ def test_channel_owner_can_create_album(factories, logged_in_api_client):
assert album.description.text == "hello world"
def test_channel_owner_can_delete_album(factories, logged_in_api_client):
def test_channel_owner_can_delete_album(factories, logged_in_api_client, mocker):
dispatch = mocker.patch("funkwhale_api.federation.routes.outbox.dispatch")
actor = logged_in_api_client.user.create_actor()
channel = factories["audio.Channel"](attributed_to=actor)
album = factories["music.Album"](artist=channel.artist)
......@@ -1416,6 +1417,10 @@ def test_channel_owner_can_delete_album(factories, logged_in_api_client):
response = logged_in_api_client.delete(url)
assert response.status_code == 204
dispatch.assert_called_once_with(
{"type": "Delete", "object": {"type": "Album"}}, context={"album": album}
)
with pytest.raises(album.DoesNotExist):
album.refresh_from_db()
......@@ -1452,15 +1457,22 @@ def test_other_user_cannot_delete_album(factories, logged_in_api_client):
album.refresh_from_db()