diff --git a/api/Dockerfile b/api/Dockerfile index 2b1e831a2e11e6474de950ce5664796e7b6f1c22..eea83be6a4b8f0fc204262d909a459863686d562 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -5,7 +5,7 @@ FROM alpine:3.14 as pre-build # allows us to install poetry as package. RUN apk add --no-cache python3 py3-cryptography py3-pip && \ - pip3 install poetry + pip3 install poetry COPY pyproject.toml poetry.lock / RUN poetry export --without-hashes > requirements.txt RUN poetry export --dev --without-hashes > dev-requirements.txt @@ -13,28 +13,28 @@ RUN poetry export --dev --without-hashes > dev-requirements.txt FROM alpine:3.14 as builder RUN \ - echo 'installing dependencies' && \ - apk add --no-cache \ - git \ - musl-dev \ - gcc \ - postgresql-dev \ - python3-dev \ - py3-psycopg2 \ - py3-cryptography \ - libldap \ - libffi-dev \ - make \ - zlib-dev \ - jpeg-dev \ - openldap-dev \ - openssl-dev \ - cargo \ - libxml2-dev \ - libxslt-dev \ - curl \ - && \ - ln -s /usr/bin/python3 /usr/bin/python + echo 'installing dependencies' && \ + apk add --no-cache \ + git \ + musl-dev \ + gcc \ + postgresql-dev \ + python3-dev \ + py3-psycopg2 \ + py3-cryptography \ + libldap \ + libffi-dev \ + make \ + zlib-dev \ + jpeg-dev \ + openldap-dev \ + openssl-dev \ + cargo \ + libxml2-dev \ + libxslt-dev \ + curl \ + && \ + ln -s /usr/bin/python3 /usr/bin/python # create virtual env for next stage RUN python -m venv --system-site-packages /venv @@ -46,24 +46,24 @@ COPY --from=0 /dev-requirements.txt /dev-requirements.txt # hack around https://github.com/pypa/pip/issues/6158#issuecomment-456619072 ENV PIP_DOWNLOAD_CACHE=/noop/ RUN \ - echo 'installing pip requirements' && \ - pip3 install --upgrade pip && \ - pip3 install setuptools wheel && \ - # Currently we are unable to relieably build cryptography on armv7. This - # is why we need to use the package shipped by Alpine Linux, which is currently - # version 3.3.2. Since poetry does not allow in-place dependency pinning, we need - # to install the deps using pip. - cat /requirements.txt | grep -Ev 'cryptography|autobahn' | pip3 install -r /dev/stdin cryptography==3.3.2 autobahn==21.2.1 && \ - rm -rf $PIP_DOWNLOAD_CACHE + echo 'installing pip requirements' && \ + pip3 install --upgrade pip && \ + pip3 install setuptools wheel && \ + # Currently we are unable to relieably build cryptography on armv7. This + # is why we need to use the package shipped by Alpine Linux, which is currently + # version 3.3.2. Since poetry does not allow in-place dependency pinning, we need + # to install the deps using pip. + cat /requirements.txt | grep -Ev 'cryptography|autobahn' | pip3 install -r /dev/stdin cryptography==3.3.2 autobahn==21.2.1 && \ + rm -rf $PIP_DOWNLOAD_CACHE ARG install_dev_deps=0 RUN \ - if [ "$install_dev_deps" = "1" ] ; then \ - echo "Installing dev dependencies" && \ - cat /dev-requirements.txt | grep -Ev 'cryptography|autobahn' | pip3 install -r /dev/stdin cryptography==3.3.2 autobahn==21.2.1 \ - ; else \ - echo "Skipping dev deps installation" \ - ; fi + if [ "$install_dev_deps" = "1" ] ; then \ + echo "Installing dev dependencies" && \ + cat /dev-requirements.txt | grep -Ev 'cryptography|autobahn' | pip3 install -r /dev/stdin cryptography==3.3.2 autobahn==21.2.1 \ + ; else \ + echo "Skipping dev deps installation" \ + ; fi FROM alpine:3.14 as build-image @@ -73,18 +73,18 @@ COPY --from=builder /venv /venv ENV PATH="/venv/bin:$PATH" RUN apk add --no-cache \ - libmagic \ - bash \ - gettext \ - python3 \ - jpeg-dev \ - ffmpeg \ - libpq \ - libxml2 \ - libxslt \ - py3-cryptography \ - && \ - ln -s /usr/bin/python3 /usr/bin/python + libmagic \ + bash \ + gettext \ + python3 \ + jpeg-dev \ + ffmpeg \ + libpq \ + libxml2 \ + libxslt \ + py3-cryptography \ + && \ + ln -s /usr/bin/python3 /usr/bin/python COPY . /app WORKDIR /app diff --git a/api/config/settings/common.py b/api/config/settings/common.py index 179c01be4b466cd62f741569e6a0c9a8dc3ca134..8e9e3259325ed8bdf89f561269dd48ebafeb5c18 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -190,6 +190,7 @@ ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=[]) + [FUNKWHALE_HOSTNA """ List of allowed hostnames for which the Funkwhale server will answer. """ +GLOBAL_FEDERATION = env.bool("GLOBAL_FEDERATION", default=True) # APP CONFIGURATION # ------------------------------------------------------------------------------ @@ -824,6 +825,21 @@ CELERY_BEAT_SCHEDULE = { ), "options": {"expires": 60 * 60}, }, + "federation.follow_all_domains": { + "task": "federation.follow_all_domains", + "schedule": crontab(minute="30", hour="*/12"), + "options": {"expires": 60 * 60 * 24}, + }, + "federation.discover_domains": { + "task": "federation.discover_domains", + "schedule": crontab(minute="0", hour="2", day_of_week="1"), + "options": {"expires": 60 * 60 * 24}, + }, + "federation.discover_domains_from_funkwhale.audio": { + "task": "federation.discover_domains_from_funkwhale.audio", + "schedule": crontab(minute="0", hour="1", day_of_week="1"), + "options": {"expires": 60 * 60 * 24}, + }, } if env.bool("ADD_ALBUM_TAGS_FROM_TRACKS", default=True): diff --git a/api/config/settings/local.py b/api/config/settings/local.py index f0a64f0fa8192885b73f5a753926c40a796a3457..176c7912a4fec9c5ecd5aad9fa136d1aee924ca2 100644 --- a/api/config/settings/local.py +++ b/api/config/settings/local.py @@ -92,7 +92,7 @@ if DEBUG_TOOLBAR_ENABLED: # ------------------------------------------------------------------------------ TEST_RUNNER = "django.test.runner.DiscoverRunner" -# CELERY +# CELERY Set to true to debug CELERY_TASK_ALWAYS_EAGER = False # END CELERY diff --git a/api/funkwhale_api/federation/activity.py b/api/funkwhale_api/federation/activity.py index 7725953e69706e746247c2f754afcf28e943c618..fc8ce550834ac8b264998e2c36517b53d535863d 100644 --- a/api/funkwhale_api/federation/activity.py +++ b/api/funkwhale_api/federation/activity.py @@ -12,6 +12,11 @@ from funkwhale_api.common import utils as funkwhale_utils from . import contexts +import logging + +logger = logging.getLogger(__name__) + + recursive_getattr = funkwhale_utils.recursive_getattr @@ -119,13 +124,16 @@ def should_reject(fid, actor_id=None, payload={}): @transaction.atomic def receive(activity, on_behalf_of, inbox_actor=None): + """ + Receive an activity, find his recipients and save it to the database before dispatching it + """ from . import models from . import serializers from . import tasks from .routes import inbox from funkwhale_api.moderation import mrf - logger.debug( + logger.info( "[federation] Received activity from %s : %s", on_behalf_of.fid, activity ) # we ensure the activity has the bare minimum structure before storing @@ -226,6 +234,9 @@ class InboxRouter(Router): from . import api_serializers from . import models + logger.info( + f"[federation] Inbox dispatch payload : {payload} with context : {context}" + ) handlers = self.get_matching_handlers(payload) for handler in handlers: if call_handlers: @@ -308,6 +319,8 @@ class OutboxRouter(Router): from . import models from . import tasks + logger.info(f"[federation] Outbox dispatch context : {context}") + allow_list_enabled = preferences.get("moderation__allow_list_enabled") allowed_domains = None if allow_list_enabled: @@ -339,6 +352,14 @@ class OutboxRouter(Router): deliveries_by_activity_uuid = {} prepared_activities = [] for activity_data in activities_data: + # If its a domain update we change the object type of the library to "Domain" to allow specific routing + for recipient in activity_data["payload"]["to"]: + if ( + type(recipient) is dict + and "domain_followers" in recipient["type"] + ): + + activity_data["payload"]["object"]["type"] = "Domain" activity_data["payload"]["actor"] = activity_data["actor"].fid to = activity_data["payload"].pop("to", []) cc = activity_data["payload"].pop("cc", []) @@ -461,6 +482,18 @@ def prepare_deliveries_and_inbox_items(recipient_list, type, allowed_domains=Non else: remote_inbox_urls.add(actor.shared_inbox_url or actor.inbox_url) urls.append(r["target"].followers_url) + + elif isinstance(r, dict) and r["type"] == "domain_followers": + received_follows = ( + r["target"] + .domainfollow_received_follows.filter(approved=True) + .select_related("actor__user") + ) + for follow in received_follows: + actor = follow.actor + remote_inbox_urls.add(actor.shared_inbox_url or actor.inbox_url) + urls.append(actor.inbox_url) + elif isinstance(r, dict) and r["type"] == "actor_inbox": actor = r["actor"] urls.append(actor.fid) diff --git a/api/funkwhale_api/federation/admin.py b/api/funkwhale_api/federation/admin.py index 8e66708cfcd13054ff57f5477a3feef9788a446b..b71fad9716db7cccc483f9459bac4082ec1e7892 100644 --- a/api/funkwhale_api/federation/admin.py +++ b/api/funkwhale_api/federation/admin.py @@ -27,7 +27,7 @@ redeliver_activities.short_description = "Redeliver" @admin.register(models.Domain) class DomainAdmin(admin.ModelAdmin): list_display = ["name", "allowed", "creation_date"] - list_filter = ["allowed"] + list_filter = ["allowed", "is_funkwhale_instance"] search_fields = ["name"] @@ -78,6 +78,14 @@ class LibraryFollowAdmin(admin.ModelAdmin): list_select_related = True +@admin.register(models.DomainFollow) +class DomainFollowAdmin(admin.ModelAdmin): + list_display = ["actor", "target", "approved", "creation_date"] + list_filter = ["approved"] + search_fields = ["actor__fid", "target__name"] + list_select_related = True + + @admin.register(models.InboxItem) class InboxItemAdmin(admin.ModelAdmin): list_display = ["actor", "activity", "type", "is_read"] diff --git a/api/funkwhale_api/federation/api_serializers.py b/api/funkwhale_api/federation/api_serializers.py index 6ff1e1b2bc32978ba3c9441a623496a375b847b7..fbb81e12b02a9c30f87d212bff04e937888143dd 100644 --- a/api/funkwhale_api/federation/api_serializers.py +++ b/api/funkwhale_api/federation/api_serializers.py @@ -12,10 +12,15 @@ from funkwhale_api.common import fields as common_fields from funkwhale_api.common import serializers as common_serializers from funkwhale_api.music import models as music_models from funkwhale_api.users import serializers as users_serializers - +from funkwhale_api.federation import models as federation_models from . import filters from . import models from . import serializers as federation_serializers +from . import tasks + +import logging + +logger = logging.getLogger(__name__) class NestedLibraryFollowSerializer(serializers.ModelSerializer): @@ -24,6 +29,17 @@ class NestedLibraryFollowSerializer(serializers.ModelSerializer): fields = ["creation_date", "uuid", "fid", "approved", "modification_date"] +class NestedDomainFollowSerializer(serializers.ModelSerializer): + class Meta: + model = models.DomainFollow + fields = [ + "creation_date", + "fid", + "approved", + "modification_date", + ] + + class LibraryScanSerializer(serializers.ModelSerializer): class Meta: model = music_models.LibraryScan @@ -37,8 +53,53 @@ class LibraryScanSerializer(serializers.ModelSerializer): ] -class DomainSerializer(serializers.Serializer): +class DomainSerializer(serializers.ModelSerializer): name = serializers.CharField() + follow = serializers.SerializerMethodField() + is_funkwhale_instance = serializers.SerializerMethodField() + + class Meta: + model = federation_models.Domain + fields = [ + "name", + "nodeinfo", + "follow", + "is_funkwhale_instance", + ] + + def get_follow(self, o): + try: + return NestedDomainFollowSerializer(o._follows[0]).data + except (AttributeError, IndexError): + return None + + def get_is_funkwhale_instance(self, o): + try: + if o.nodeinfo["payload"]["software"]["name"] == "funkwhale": + return True + else: + return False + except Exception as e: + logger.info(f"Exception {e}") + return False + + +class LibraryFollowListSerializer(serializers.ListSerializer): + def create(self, validated_data): + libs = [models.LibraryFollow(**lib) for lib in validated_data] + return models.LibraryFollow.objects.bulk_create(libs, batch_size=1000) + + +class LibraryListSerializer(serializers.ListSerializer): + def create(self, validated_data): + for index, lib in enumerate(validated_data): + actor, status = models.Actor.objects.update_or_create(**lib["actor"]) + lib["actor"] = actor + validated_data[index] = lib + logger.info(f"EEEEEH actor is {actor} and libs are {lib}") + + libs = [music_models.Library(**lib) for lib in validated_data] + return music_models.Library.objects.bulk_create(libs, batch_size=1000) class LibrarySerializer(serializers.ModelSerializer): @@ -61,6 +122,7 @@ class LibrarySerializer(serializers.ModelSerializer): "follow", "latest_scan", ] + list_serializer_class = LibraryListSerializer def get_uploads_count(self, o): return max(getattr(o, "_uploads_count", 0), o.uploads_count) @@ -80,14 +142,26 @@ class LibrarySerializer(serializers.ModelSerializer): class LibraryFollowSerializer(serializers.ModelSerializer): target = common_serializers.RelatedField("uuid", LibrarySerializer(), required=True) actor = serializers.SerializerMethodField() + follow_context = serializers.ChoiceField( + choices=["Domain", "Library"], required=False + ) class Meta: model = models.LibraryFollow - fields = ["creation_date", "actor", "uuid", "target", "approved"] + fields = [ + "creation_date", + "actor", + "uuid", + "target", + "approved", + "follow_context", + ] read_only_fields = ["uuid", "actor", "approved", "creation_date"] + list_serializer_class = LibraryFollowListSerializer def validate_target(self, v): actor = self.context["actor"] + if v.actor == actor: raise serializers.ValidationError("You cannot follow your own library") @@ -99,16 +173,56 @@ class LibraryFollowSerializer(serializers.ModelSerializer): return federation_serializers.APIActorSerializer(o.actor).data +class DomainFollowSerializer(serializers.ModelSerializer): + target = common_serializers.RelatedField("name", DomainSerializer(), required=True) + actor = serializers.SerializerMethodField() + + class Meta: + model = models.DomainFollow + fields = ["uuid", "creation_date", "actor", "target", "approved"] + read_only_fields = ["uuid", "actor", "approved", "creation_date"] + + def validate_target(self, v): + actor = self.context["actor"] + if v.domainfollow_received_follows.filter(actor=actor).exists(): + raise serializers.ValidationError("You are already following this Domain") + domain = models.Domain.objects.get(name=v.name) + now = datetime.datetime.now(timezone.utc) + if ( + (domain.nodeinfo_fetch_date - now).total_seconds() > 86400 + or "error" in domain.nodeinfo + or not domain.service_actor + ): + + tasks.update_domain_nodeinfo.delay(domain_name=domain.name) + domain = models.Domain.objects.get(name=v.name) + + if "error" in domain.nodeinfo: + e = domain.nodeinfo["error"] + logger.error(f"Could not get domain information because of : {e!r}") + raise serializers.ValidationError(domain.nodeinfo["error"]) + + return domain + + def get_actor(self, o): + return federation_serializers.APIActorSerializer(o.actor).data + + def serialize_generic_relation(activity, obj): data = {"type": obj._meta.label} if data["type"] == "federation.Actor": data["full_username"] = obj.full_username + elif data["type"] == "federation.Domain": + data["name"] = obj.name else: data["uuid"] = obj.uuid - if data["type"] == "music.Library": data["name"] = obj.name - if data["type"] == "federation.LibraryFollow": + + if ( + data["type"] == "federation.LibraryFollow" + or data["type"] == "federation.DomainFollow" + ): data["approved"] = obj.approved return data diff --git a/api/funkwhale_api/federation/api_urls.py b/api/funkwhale_api/federation/api_urls.py index df5bfb2f0396af3da21357aba967d51d941cad9a..1e9799521f5a11769d9e266ba7dcad457b728424 100644 --- a/api/funkwhale_api/federation/api_urls.py +++ b/api/funkwhale_api/federation/api_urls.py @@ -5,6 +5,7 @@ from . import api_views router = routers.OptionalSlashRouter() router.register(r"fetches", api_views.FetchViewSet, "fetches") router.register(r"follows/library", api_views.LibraryFollowViewSet, "library-follows") +router.register(r"follows/domain", api_views.DomainFollowViewSet, "domain-follows") router.register(r"inbox", api_views.InboxItemViewSet, "inbox") router.register(r"libraries", api_views.LibraryViewSet, "libraries") router.register(r"domains", api_views.DomainViewSet, "domains") diff --git a/api/funkwhale_api/federation/api_views.py b/api/funkwhale_api/federation/api_views.py index f49c51c3575869e2417b672ea6adfb1d88c823d1..eb35f23be12875845599fbf2317ba5b68fe25f60 100644 --- a/api/funkwhale_api/federation/api_views.py +++ b/api/funkwhale_api/federation/api_views.py @@ -1,4 +1,6 @@ +from urllib import request import requests.exceptions +import requests from django.conf import settings from django.db import transaction @@ -11,6 +13,7 @@ from rest_framework import response from rest_framework import viewsets from funkwhale_api.common import preferences +from funkwhale_api.common import permissions as common_permissions from funkwhale_api.common import utils as common_utils from funkwhale_api.common.permissions import ConditionalAuthentication from funkwhale_api.music import models as music_models @@ -27,6 +30,10 @@ from . import serializers from . import tasks from . import utils +import logging + +logger = logging.getLogger(__name__) + @transaction.atomic def update_follow(follow, approved): @@ -262,6 +269,96 @@ class DomainViewSet( qs = qs.filter(allowed=True) return qs + def perform_create(self, serializer): + domain = serializer.save(actor=self.request.user.actor) + + +class DomainFollowViewSet( + mixins.CreateModelMixin, + mixins.ListModelMixin, + mixins.RetrieveModelMixin, + mixins.DestroyModelMixin, + viewsets.GenericViewSet, +): + lookup_field = "uuid" + queryset = ( + models.DomainFollow.objects.all() + .order_by("-creation_date") + .select_related("actor") + ) + serializer_class = api_serializers.DomainFollowSerializer + permission_classes = [oauth_permissions.ScopePermission] + required_scope = "follows" + filterset_class = filters.DomainFollowFilter + ordering_fields = ("creation_date",) + + def get_queryset(self): + qs = super().get_queryset() + return qs.filter(actor=self.request.user.actor).exclude(approved=False) + + def perform_create(self, serializer): + follow = serializer.save(actor=self.request.user.actor) + routes.outbox.dispatch({"type": "Follow"}, context={"follow": follow}) + + @transaction.atomic + def perform_destroy(self, instance): + routes.outbox.dispatch( + {"type": "Undo", "object": {"type": "Follow"}}, + context={"follow": instance}, + ) + tasks.domain_bulk_unfollow_libraries.delay( + instance.target.name, actor_id=self.request.user.actor.pk + ) + instance.delete() + + def get_serializer_context(self): + context = super().get_serializer_context() + context["actor"] = self.request.user.actor + return context + + @decorators.action(methods=["post"], detail=True) + def accept(self, request, *args, **kwargs): + try: + follow = self.queryset.get( + target__actor=self.request.user.actor, uuid=kwargs["uuid"] + ) + except models.DomainFollow.DoesNotExist: + return response.Response({}, status=404) + update_follow(follow, approved=True) + return response.Response(status=204) + + @decorators.action(methods=["post"], detail=True) + def reject(self, request, *args, **kwargs): + try: + follow = self.queryset.get( + target__actor=self.request.user.actor, uuid=kwargs["uuid"] + ) + except models.DomainFollow.DoesNotExist: + return response.Response({}, status=404) + + update_follow(follow, approved=False) + return response.Response(status=204) + + @decorators.action(methods=["get"], detail=False) + def all(self, request, *args, **kwargs): + """ + Return all the subscriptions of the current user, with only limited data + to have a performant endpoint and avoid lots of queries just to display + subscription status in the UI + """ + follows = list( + self.get_queryset().values_list("uuid", "target__name", "approved") + ) + + payload = { + "results": [ + {"uuid": str(u[0]), "domain": str(u[1]), "approved": u[2]} + for u in follows + ], + "count": len(follows), + } + return response.Response(payload, status=200) + class ActorViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet): queryset = models.Actor.objects.select_related( diff --git a/api/funkwhale_api/federation/factories.py b/api/funkwhale_api/federation/factories.py index 2d2a2abae64e9602a6ca918127d7097ea8da2660..cc7f576333bd7bb01835592a19a78761c9bc49f2 100644 --- a/api/funkwhale_api/federation/factories.py +++ b/api/funkwhale_api/federation/factories.py @@ -1,6 +1,9 @@ import uuid import factory + +# see https://stackoverflow.com/questions/60914361/python-import-error-module-factory-has-no-attribute-fuzzy +from factory import fuzzy import requests import requests_http_message_signatures from django.conf import settings @@ -238,11 +241,23 @@ class DeliveryFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory): class LibraryFollowFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory): target = factory.SubFactory(MusicLibraryFactory) actor = factory.SubFactory(ActorFactory) + follow_context = fuzzy.FuzzyChoice( + models.LibraryFollow.FOLLOW_CONTEXT_CHOICES, getter=lambda c: c[0] + ) class Meta: model = "federation.LibraryFollow" +@registry.register +class DomainollowFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory): + target = factory.SubFactory(DomainFactory) + actor = factory.SubFactory(ActorFactory) + + class Meta: + model = "federation.DomainFollow" + + class ArtistMetadataFactory(factory.Factory): name = factory.Faker("name") diff --git a/api/funkwhale_api/federation/filters.py b/api/funkwhale_api/federation/filters.py index a57bc40713a03c2c28274c8eaa225cd6a7414b3d..31809604ffb64eda7c3e73047301f4eee81a4886 100644 --- a/api/funkwhale_api/federation/filters.py +++ b/api/funkwhale_api/federation/filters.py @@ -34,6 +34,12 @@ class LibraryFollowFilter(django_filters.FilterSet): fields = ["approved"] +class DomainFollowFilter(django_filters.FilterSet): + class Meta: + model = models.DomainFollow + fields = ["approved"] + + class InboxItemFilter(django_filters.FilterSet): is_read = django_filters.BooleanFilter( "is_read", widget=django_filters.widgets.BooleanWidget() diff --git a/api/funkwhale_api/federation/migrations/0027_auto_20220527_2006.py b/api/funkwhale_api/federation/migrations/0027_auto_20220527_2006.py new file mode 100644 index 0000000000000000000000000000000000000000..32159d17e2c40f8bc89baec6369165eda80b2c5d --- /dev/null +++ b/api/funkwhale_api/federation/migrations/0027_auto_20220527_2006.py @@ -0,0 +1,84 @@ +# Generated by Django 3.2.13 on 2022-05-27 20:06 + +import django.core.serializers.json +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import funkwhale_api.federation.models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('federation', '0026_public_key_format'), + ] + + operations = [ + migrations.AddField( + model_name='libraryfollow', + name='follow_context', + field=models.CharField(choices=[('Domain', 'Domain'), ('Library', 'Library')], default='Library', max_length=100), + ), + migrations.AlterField( + model_name='activity', + name='object_id', + field=models.TextField(blank=True, null=True), + ), + migrations.AlterField( + model_name='activity', + name='payload', + field=models.JSONField(blank=True, default=funkwhale_api.federation.models.empty_dict, encoder=django.core.serializers.json.DjangoJSONEncoder, max_length=50000), + ), + migrations.AlterField( + model_name='activity', + name='related_object_id', + field=models.TextField(blank=True, null=True), + ), + migrations.AlterField( + model_name='actor', + name='manually_approves_followers', + field=models.BooleanField(default=None, null=True), + ), + migrations.AlterField( + model_name='domain', + name='nodeinfo', + field=models.JSONField(blank=True, default=funkwhale_api.federation.models.empty_dict, max_length=50000), + ), + migrations.AlterField( + model_name='fetch', + name='detail', + field=models.JSONField(blank=True, default=funkwhale_api.federation.models.empty_dict, encoder=django.core.serializers.json.DjangoJSONEncoder, max_length=50000), + ), + migrations.AlterField( + model_name='follow', + name='approved', + field=models.BooleanField(default=None, null=True), + ), + migrations.AlterField( + model_name='libraryfollow', + name='approved', + field=models.BooleanField(default=None, null=True), + ), + migrations.AlterField( + model_name='librarytrack', + name='metadata', + field=models.JSONField(blank=True, default=funkwhale_api.federation.models.empty_dict, encoder=django.core.serializers.json.DjangoJSONEncoder, max_length=10000), + ), + migrations.CreateModel( + name='DomainFollow', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('fid', models.URLField(blank=True, max_length=500, null=True, unique=True)), + ('uuid', models.UUIDField(default=uuid.uuid4, unique=True)), + ('creation_date', models.DateTimeField(default=django.utils.timezone.now)), + ('modification_date', models.DateTimeField(auto_now=True)), + ('approved', models.BooleanField(default=None, null=True)), + ('actor', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='domain_follows', to='federation.actor')), + ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='domainfollow_received_follows', to='federation.domain')), + ], + options={ + 'unique_together': {('actor', 'target')}, + }, + ), + ] diff --git a/api/funkwhale_api/federation/migrations/0028_domain_is_funkwhale_instance.py b/api/funkwhale_api/federation/migrations/0028_domain_is_funkwhale_instance.py new file mode 100644 index 0000000000000000000000000000000000000000..a86698c9581162156ebcb687a9aee527f057e5f3 --- /dev/null +++ b/api/funkwhale_api/federation/migrations/0028_domain_is_funkwhale_instance.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2022-07-21 10:46 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("federation", "0027_auto_20220527_2006"), + ] + + operations = [ + migrations.AddField( + model_name="domain", + name="is_funkwhale_instance", + field=models.BooleanField(default=False, null=True), + ), + ] diff --git a/api/funkwhale_api/federation/models.py b/api/funkwhale_api/federation/models.py index d9ea2018074f22332d46d7e1bac102aa9c037c55..ac9c1e6e81fbeb84c67e568e762460174363547d 100644 --- a/api/funkwhale_api/federation/models.py +++ b/api/funkwhale_api/federation/models.py @@ -21,6 +21,10 @@ from funkwhale_api.music import utils as music_utils from . import utils as federation_utils +import logging + +logger = logging.getLogger(__name__) + TYPE_CHOICES = [ ("Person", "Person"), ("Tombstone", "Tombstone"), @@ -123,7 +127,7 @@ class Domain(models.Model): ) # are interactions with this domain allowed (only applies when allow-listing is on) allowed = models.BooleanField(default=None, null=True) - + is_funkwhale_instance = models.BooleanField(default=False, null=True) objects = DomainQuerySet.as_manager() def __str__(self): @@ -151,6 +155,12 @@ class Domain(models.Model): emitted_library_follows=models.Count( "actors__library_follows", distinct=True ), + received_domain_follows=models.Count( + "domainfollow_received_follows", distinct=True + ), + emitted_domain_follows=models.Count( + "actors__domain_follows", distinct=True + ), actors=models.Count("actors", distinct=True), ) data["artists"] = music_models.Artist.objects.filter( @@ -171,10 +181,23 @@ class Domain(models.Model): ) return data + def request_libraries_follows(self, actor): + from funkwhale_api.federation import tasks + + common_utils.on_commit( + tasks.domain_bulk_follow_libraries.delay, self.name, actor_id=actor.pk + ) + @property def is_local(self): return self.name == settings.FEDERATION_HOSTNAME + def should_autoapprove_follow(self, actor): + return settings.GLOBAL_FEDERATION + + def is_following(self, actor): + return self.domainfollow_received_follows__actor == actor + class Actor(models.Model): ap_type = "Actor" @@ -451,7 +474,8 @@ class Activity(models.Model): type = models.CharField(db_index=True, null=True, max_length=100) # generic relations - object_id = models.IntegerField(null=True, blank=True) + # This change is needed because the primary key is a text for domain and an integer for libraries + object_id = models.TextField(null=True, blank=True) object_content_type = models.ForeignKey( ContentType, null=True, @@ -469,7 +493,8 @@ class Activity(models.Model): related_name="targeting_activities", ) target = GenericForeignKey("target_content_type", "target_id") - related_object_id = models.IntegerField(null=True, blank=True) + # This change is needed because object_id is a text for domain and an integer for libraries + related_object_id = models.TextField(null=True, blank=True) related_object_content_type = models.ForeignKey( ContentType, null=True, @@ -519,6 +544,17 @@ class LibraryFollow(AbstractFollow): "music.Library", related_name="received_follows", on_delete=models.CASCADE ) + # This allow to know what libraries follows to delete. Exemple : if the delete comes from unfollow domain, only the libraries with contexte = [domain] will be deleted + # if the library is not followed whyle domain follow : context = domain + # if the library was followed before the domain follow : context = library (excluded from domain bulk lib follow) + # if the library is followed after the domain : can't do, to implement in the front ? + FOLLOW_CONTEXT_CHOICES = [("Domain", "Domain"), ("Library", "Library")] + follow_context = models.CharField( + max_length=100, + choices=FOLLOW_CONTEXT_CHOICES, + default="Library", + ) + class Meta: unique_together = ["actor", "target"] @@ -636,3 +672,19 @@ def update_denormalization_follow_deleted(sender, instance, **kwargs): music_models.TrackActor.objects.filter( actor=instance.actor, upload__in=instance.target.uploads.all() ).delete() + + +class DomainFollow(AbstractFollow): + actor = models.ForeignKey( + Actor, related_name="domain_follows", on_delete=models.CASCADE + ) + + target = models.ForeignKey( + "federation.Domain", + # domainfollow_received_follows + related_name="%(class)s_received_follows", + on_delete=models.CASCADE, + ) + + class Meta: + unique_together = ["actor", "target"] diff --git a/api/funkwhale_api/federation/routes.py b/api/funkwhale_api/federation/routes.py index 0a45f1d64a09743be0e818d548dc40f6cc86f5f6..1a0bbd9b83dc0afda958a7c35217cada727e1ec3 100644 --- a/api/funkwhale_api/federation/routes.py +++ b/api/funkwhale_api/federation/routes.py @@ -1,5 +1,8 @@ import logging import uuid +from urllib.parse import urlparse + +from django.conf import settings from django.db.models import Q @@ -9,6 +12,7 @@ from . import activity from . import actors from . import models from . import serializers +from rest_framework import serializers as django_serializers logger = logging.getLogger(__name__) inbox = activity.InboxRouter() @@ -70,6 +74,8 @@ def outbox_accept(context): follow = context["follow"] if follow._meta.label == "federation.LibraryFollow": actor = follow.target.actor + elif follow._meta.label == "federation.DomainFollow": + actor = actors.get_service_actor() else: actor = follow.target payload = serializers.AcceptFollowSerializer(follow, context={"actor": actor}).data @@ -133,6 +139,8 @@ def outbox_undo_follow(context): actor = follow.actor if follow._meta.label == "federation.LibraryFollow": recipient = follow.target.actor + elif follow._meta.label == "federation.DomainFollow": + recipient = follow.target.service_actor else: recipient = follow.target payload = serializers.UndoFollowSerializer(follow, context={"actor": actor}).data @@ -148,11 +156,26 @@ def outbox_undo_follow(context): @outbox.register({"type": "Follow"}) def outbox_follow(context): follow = context["follow"] + # context["follow"] keys are dict_keys(['_state', 'id', 'fid', 'uuid', 'creation_date', 'modification_date', 'approved', 'actor_id', 'target_id']) + # context["follow"].target keys are ['_state', 'name', 'creation_date', 'nodeinfo_fetch_date', 'nodeinfo', 'service_actor_id', 'allowed']) + if follow._meta.label == "federation.LibraryFollow": target = follow.target.actor + elif follow._meta.label == "federation.DomainFollow": + # to do : sould be target = follow.target.service_actor but this doesn't work if the db isn't updated when the queries is made -> + # we should trigger the db update before the view is called ? Actually the db update is in the domainfollowserializer but this do not make the trick + # is it a cache problem ? + # temporary solution : + domain = models.Domain.objects.get(name=context["follow"].target.name) + # target = domain.nodeinfo["payload"]["metadata"]["actorId"] + target = domain.service_actor else: target = follow.target + if target is None: + raise Exception(f"Can't follow without target. Context is : {context}") + payload = serializers.FollowSerializer(follow, context={"actor": follow.actor}).data + # This generate an activity and send it to activity.OutboxRouter yield { "type": "Follow", "actor": follow.actor, @@ -249,6 +272,7 @@ def outbox_delete_library(context): @outbox.register({"type": "Update", "object.type": "Library"}) def outbox_update_library(context): + domain = models.Domain.objects.get(name=settings.FEDERATION_HOSTNAME) library = context["library"] serializer = serializers.ActivitySerializer( {"type": "Update", "object": serializers.LibrarySerializer(library).data} @@ -258,7 +282,11 @@ def outbox_update_library(context): "type": "Update", "actor": library.actor, "payload": with_recipients( - serializer.data, to=[{"type": "followers", "target": library}] + serializer.data, + to=[ + {"type": "followers", "target": library}, + {"type": "domain_followers", "target": domain}, + ], ), } @@ -286,6 +314,67 @@ def inbox_update_library(payload, context): ) +# if a library is created we inform the followers of the domain +@outbox.register({"type": "Update", "object.type": "Domain"}) +def outbox_update_domain_on_library_creation(context): + domain = context["domain"] + library = context["library"] + + serializer = serializers.ActivitySerializer( + {"type": "Update", "object": serializers.LibrarySerializer(library).data} + ) + + yield { + "type": "Update", + "actor": library.actor, + "payload": with_recipients( + serializer.data, + to=[ + {"type": "domain_followers", "target": domain}, + ], + ), + } + + +@inbox.register({"type": "Update", "object.type": "Domain"}) +def inbox_update_library_from_domain_follow(payload, context): + from . import api_serializers + + payload["object"]["type"] = "Library" + lib_serializer = serializers.LibrarySerializer(data=payload["object"]) + + if lib_serializer.is_valid(): + library = lib_serializer.save() + else: + raise django_serializers.ValidationError( + f"Discarding creation of the library because of error: {lib_serializer.errors}" + ) + + domain = urlparse(payload["object"]["id"]).netloc + domain_follows = [ + domain_follows + for domain_follows in models.DomainFollow.objects.filter(target=domain) + ] + + for domain_follow in domain_follows: + library_follow = {} + actor = domain_follow.actor + library_follow["target"] = library.uuid + library_follow["follow_context"] = "Domain" + + f_serializer = api_serializers.LibraryFollowSerializer( + data=library_follow, context={"actor": actor} + ) + if f_serializer.is_valid(): + follow = f_serializer.save(actor=actor) + else: + logger.info( + "Discarding follow of library %s because of payload errors: %s", + domain, + f_serializer.errors, + ) + + @inbox.register({"type": "Delete", "object.type": "Audio"}) def inbox_delete_audio(payload, context): actor = context["actor"] @@ -507,6 +596,7 @@ def outbox_delete_actor(context): "Organization", "Service", "Group", + "Domain", ], } ) diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py index d3263507baaed8dfde3197cb567be95783886413..9e34f865aac9c8fe91d641f02034abfd93e9e11a 100644 --- a/api/funkwhale_api/federation/serializers.py +++ b/api/funkwhale_api/federation/serializers.py @@ -599,9 +599,11 @@ class BaseActivitySerializer(serializers.Serializer): ) +# TODO class FollowSerializer(serializers.Serializer): id = serializers.URLField(max_length=500) - object = serializers.URLField(max_length=500) + # if obj is a domain its not a valid url + object = serializers.CharField(max_length=500) actor = serializers.URLField(max_length=500) type = serializers.ChoiceField(choices=["Follow"]) @@ -630,6 +632,14 @@ class FollowSerializer(serializers.Serializer): except music_models.Library.DoesNotExist: pass + try: + qs = models.Domain.objects.filter(name=v) + # if recipient: + # qs = qs.filter(actor=recipient) + return qs.get() + except models.Domain.DoesNotExist: + pass + raise serializers.ValidationError("Target not found") def validate_actor(self, v): @@ -642,12 +652,16 @@ class FollowSerializer(serializers.Serializer): raise serializers.ValidationError("Actor not found") def save(self, **kwargs): + logger.info("saving") target = self.validated_data["object"] if target._meta.label == "music.Library": follow_class = models.LibraryFollow + elif target._meta.label == "federation.Domain": + follow_class = models.DomainFollow else: follow_class = models.Follow + logger.info("followclass is " + str(follow_class) + str(target._meta.label)) defaults = kwargs defaults["fid"] = self.validated_data["id"] approved = kwargs.pop("approved", None) @@ -675,13 +689,22 @@ class FollowSerializer(serializers.Serializer): return follow def to_representation(self, instance): - return { - "@context": jsonld.get_default_context(), - "actor": instance.actor.fid, - "id": instance.get_federation_id(), - "object": instance.target.fid, - "type": "Follow", - } + try: + return { + "@context": jsonld.get_default_context(), + "actor": instance.actor.fid, + "id": instance.get_federation_id(), + "object": instance.target.fid, + "type": "Follow", + } + except: + return { + "@context": jsonld.get_default_context(), + "actor": instance.actor.fid, + "id": instance.get_federation_id(), + "object": instance.target.name, + "type": "Follow", + } class APIFollowSerializer(serializers.ModelSerializer): @@ -722,6 +745,9 @@ class FollowActionSerializer(serializers.Serializer): if target._meta.label == "music.Library": expected = target.actor follow_class = models.LibraryFollow + elif target._meta.label == "federation.Domain": + expected = target.service_actor + follow_class = models.DomainFollow else: expected = target follow_class = models.Follow @@ -744,7 +770,10 @@ class FollowActionSerializer(serializers.Serializer): def to_representation(self, instance): if instance.target._meta.label == "music.Library": actor = instance.target.actor + elif instance.target._meta.label == "federation.Domain": + actor = actors.get_service_actor() else: + actor = instance.target return { @@ -767,6 +796,8 @@ class AcceptFollowSerializer(FollowActionSerializer): follow.save() if follow.target._meta.label == "music.Library": follow.target.schedule_scan(actor=follow.actor) + if follow.target._meta.label == "federation.Domain": + follow.target.request_libraries_follows(actor=follow.actor) return follow diff --git a/api/funkwhale_api/federation/tasks.py b/api/funkwhale_api/federation/tasks.py index c040082aa44812603e36fb61495eb165019c729c..a6f537bc68f710b45a7a727013787aa0550a7a92 100644 --- a/api/funkwhale_api/federation/tasks.py +++ b/api/funkwhale_api/federation/tasks.py @@ -3,12 +3,14 @@ import json import logging import os import requests - +import time +from django.db.utils import IntegrityError from django.conf import settings from django.db import transaction from django.db.models import Q, F from django.db.models.deletion import Collector from django.utils import timezone +from django.urls import reverse from dynamic_preferences.registries import global_preferences_registry from requests.exceptions import RequestException @@ -31,6 +33,8 @@ from . import serializers from . import routes from . import utils from . import webfinger +from . import api_serializers +from . import models logger = logging.getLogger(__name__) @@ -90,7 +94,9 @@ def dispatch_inbox(activity, call_handlers=True): Given an activity instance, triggers our internal delivery logic (follow creation, etc.) """ - + logger.info( + "[federation] Dispatch inbox with activity payload : {activity.payload}" + ) routes.inbox.dispatch( activity.payload, context={ @@ -108,6 +114,8 @@ def dispatch_outbox(activity): """ Deliver a local activity to its recipients, both locally and remotely """ + logger.info(f"[federation] Dispatch outbox activity payload : {activity.payload}") + inbox_items = activity.inbox_items.filter(is_read=False).select_related() if inbox_items.exists(): @@ -206,7 +214,7 @@ def update_domain_nodeinfo(domain): domain.service_actor = ( utils.retrieve_ap_object( service_actor_id, - actor=actors.get_service_actor(), + actor=None, queryset=models.Actor, serializer_class=serializers.ActorSerializer, ) @@ -223,7 +231,23 @@ def update_domain_nodeinfo(domain): ) domain.nodeinfo_fetch_date = now domain.nodeinfo = nodeinfo - domain.save(update_fields=["nodeinfo", "nodeinfo_fetch_date", "service_actor"]) + if ( + "payload" in domain.nodeinfo + and "software" in domain.nodeinfo["payload"] + and domain.nodeinfo["payload"]["software"]["name"] == "funkwhale" + ): + domain.is_funkwhale_instance = True + else: + domain.is_funkwhale_instance = False + + domain.save( + update_fields=[ + "nodeinfo", + "nodeinfo_fetch_date", + "service_actor", + "is_funkwhale_instance", + ] + ) @celery.app.task(name="federation.refresh_nodeinfo_known_nodes") @@ -625,3 +649,162 @@ def fetch_collection(url, max_pages, channel, is_page=False): results["errored"], ) return results + + +@celery.app.task(name="federation.domain_bulk_follow_libraries") +@celery.require_instance( + models.Actor.objects.all(), + "actor", +) +def domain_bulk_follow_libraries(name, actor): + if name.startswith("https://"): + pass + else: + url = utils.get_remote_url( + name, reverse("api:v1:libraries-list"), f"domain={name}" + ) + response = utils.get_remote_libraries(url, actor) + data, updated = mrf.inbox.apply(response) + if not data: + raise exceptions.BlockedActorOrDomain() + + serializer = api_serializers.LibrarySerializer(data=response["results"], many=True) + # we do not raise exeption since we know some libs will not be valid (already exist) + serializer.is_valid() + serializer.save() + + fids = [] + for lib in response["results"]: + fids.append(lib["fid"]) + + if "next" in response.keys() and response["next"] is not None: + follow_libraries_locally(fids, actor) + url = response["next"] + domain_bulk_follow_libraries(url, actor_id=actor.id) + else: + return follow_libraries_locally(fids, actor) + + +def follow_libraries_locally(fids, actor): + """ + This will create a LibraryFollow entry only localy, + without sending activity to the remote library host. + """ + libs = music_models.Library.objects.in_bulk(fids, field_name="fid") + libraries_follow = [] + for lib in libs.values(): + library_follow = {} + library_follow["target"] = lib.uuid + library_follow["follow_context"] = "Domain" + libraries_follow.append(library_follow) + + serializer = api_serializers.LibraryFollowSerializer( + data=libraries_follow, context={"actor": actor}, many=True + ) + serializer.is_valid() + return serializer.save(actor=actor) + + +@celery.app.task(name="federation.domain_bulk_unfollow_libraries") +@celery.require_instance( + models.Actor.objects.all(), + "actor", +) +def domain_bulk_unfollow_libraries(name, actor): + lib_follows = models.LibraryFollow.objects.filter( + actor=actor, + target__followers_url__startswith=f"https://{name}", + follow_context="Domain", + ) + + return lib_follows.delete() + + +@celery.app.task(name="federation.follow_all_domains") +def follow_all_domains(): + if settings.GLOBAL_FEDERATION == False: + logger.info("Global federation desabled in env variables. Skipping task.") + return + + domains = ( + models.Domain.objects.all() + .exclude(name=settings.FEDERATION_HOSTNAME) + .exclude(is_funkwhale_instance=False) + ) + actor = actors.get_service_actor() + + for domain in domains: + if not models.Domain.objects.filter(name=domain.name).exists(): + try: + data = {"target": domain.name} + serializer = api_serializers.DomainFollowSerializer( + data=data, context={"actor": actor} + ) + + serializer.is_valid(raise_exception=True) + follow = serializer.save(actor=actor) + routes.outbox.dispatch({"type": "Follow"}, context={"follow": follow}) + except Exception as e: + logger.error(f"Unhandeled error while creating domain follow : {e} ") + + +@celery.app.task(name="federation.discover_domains") +def discover_domains(): + if settings.GLOBAL_FEDERATION == False: + logger.info("Global federation disabled in env variables. Skipping task.") + return + # to do : add exclude(reachable=False) + local_fw_domains = ( + models.Domain.objects.all() + .exclude(is_funkwhale_instance=False) + .exclude(name=settings.FEDERATION_HOSTNAME) + .values_list("name", flat=True) + ) + + local_domains = ( + models.Domain.objects.all() + .exclude(name=settings.FEDERATION_HOSTNAME) + .values_list("name", flat=True) + ) + + for local_fw_domain in local_fw_domains: + url = utils.get_remote_url( + local_fw_domain, reverse("api:v1:federation:domains-list") + ) + discover_domains_from_url(url, local_domains) + + +@celery.app.task(name="federation.discover_domains_from_funkwhale.audio") +def discover_domains_from_funkwhale_audio(): + if settings.GLOBAL_FEDERATION == False: + logger.info("Global federation disabled in env variables. Skipping task.") + return + url = "https://network.funkwhale.audio/api/domains" + discover_domains_from_url(url) + + +def discover_domains_from_url(url, local_domains=[]): + try: + response = session.get_session().get(url) + except Exception: + return + + if response.status_code != 200: + logger.info(f"No domain found from {url}.") + return + + domain_list = response.json() + + for remote_domain in domain_list["results"]: + + if remote_domain["name"] in local_domains: + logger.info("Domain already in db. Skipping") + continue + + else: + logger.info(f"Adding discovered domain {remote_domain} to db") + data = {"name": remote_domain["name"], "creation_date": timezone.now} + serializer = api_serializers.DomainSerializer(data=data) + serializer.is_valid(raise_exception=True) + domain = serializer.save() + update_domain_nodeinfo.delay(domain_name=domain.name) diff --git a/api/funkwhale_api/federation/utils.py b/api/funkwhale_api/federation/utils.py index 2bac8daf8fb8dd5e64fc9711e1367360f0ce2651..f679fb0fad344ed19e0b01e37278af62b7bc3eb9 100644 --- a/api/funkwhale_api/federation/utils.py +++ b/api/funkwhale_api/federation/utils.py @@ -10,10 +10,13 @@ from django.db.models import CharField, Q, Value from funkwhale_api.common import session from funkwhale_api.moderation import mrf - from . import exceptions from . import signing +import logging + +logger = logging.getLogger(__name__) + def full_url(path): """ @@ -82,7 +85,6 @@ def retrieve_ap_object( existing = queryset.objects.filter(fid=fid).first() if existing: return existing - auth = ( None if not actor else signing.get_auth(actor.private_key, actor.private_key_id) ) @@ -291,3 +293,19 @@ def can_manage(obj_owner, actor): return True return False + + +def get_remote_libraries(url, actor): + auth = signing.get_auth(actor.private_key, actor.private_key_id) + response = session.get_session().get( + auth=auth, + url=url, + headers={"Content-Type": "application/activity+json"}, + ) + response.raise_for_status() + response = response.json() + return response + + +def get_remote_url(hostname, reverse, scope=None): + return f"https://{hostname}{reverse}?scope:{scope}" diff --git a/api/funkwhale_api/music/serializers.py b/api/funkwhale_api/music/serializers.py index df4e701335e4ab7fedcdda4ffe5274ea6f7d61db..12ed4f7969f6009863b42822ec26cb31817b77c8 100644 --- a/api/funkwhale_api/music/serializers.py +++ b/api/funkwhale_api/music/serializers.py @@ -11,6 +11,7 @@ from funkwhale_api.common import serializers as common_serializers from funkwhale_api.common import utils as common_utils from funkwhale_api.federation import routes from funkwhale_api.federation import utils as federation_utils +from funkwhale_api.federation import models as federation_models from funkwhale_api.playlists import models as playlists_models from funkwhale_api.tags import models as tag_models from funkwhale_api.tags import serializers as tags_serializers @@ -370,6 +371,19 @@ class LibraryForOwnerSerializer(serializers.ModelSerializer): {"type": "Update", "object": {"type": "Library"}}, context={"library": obj} ) + # If the library is updated after the domain was followed. We need to inform our domain + # followers that the library is now available for everyone + if after["privacy_level"] == "everyone": + routes.outbox.dispatch( + {"type": "Update", "object": {"type": "Domain"}}, + context={ + "library": obj, + "domain": federation_models.Domain.objects.get( + name=settings.FUNKWHALE_HOSTNAME + ), + }, + ) + def get_actor(self, o): # Import at runtime to avoid a circular import issue from funkwhale_api.federation import serializers as federation_serializers diff --git a/api/funkwhale_api/music/views.py b/api/funkwhale_api/music/views.py index d7eba93cfc8d98bd497d2e7c59b1b9b432c49b8a..bf231005426a73455bec1e0226afa606bd2d0d96 100644 --- a/api/funkwhale_api/music/views.py +++ b/api/funkwhale_api/music/views.py @@ -290,7 +290,18 @@ class LibraryViewSet( return qs def perform_create(self, serializer): - serializer.save(actor=self.request.user.actor) + lib = serializer.save(actor=self.request.user.actor) + if lib.privacy_level == "everyone": + lib.actor = self.request.user.actor + routes.outbox.dispatch( + {"type": "Update", "object": {"type": "Domain"}}, + context={ + "domain": federation_models.Domain.objects.get( + name=settings.FUNKWHALE_HOSTNAME + ), + "library": lib, + }, + ) @transaction.atomic def perform_destroy(self, instance): @@ -550,6 +561,7 @@ def record_downloads(f): @record_downloads +# to do : tests fail here ? def handle_serve( upload, user, format=None, max_bitrate=None, proxy_media=True, download=True ): diff --git a/api/tests/federation/test_api_views.py b/api/tests/federation/test_api_views.py index e0fcc6edf46849382c15c033db40752ed92a6953..b3a2bc45db122205901464af2b95f69945abb43e 100644 --- a/api/tests/federation/test_api_views.py +++ b/api/tests/federation/test_api_views.py @@ -8,6 +8,7 @@ from funkwhale_api.federation import api_serializers from funkwhale_api.federation import serializers from funkwhale_api.federation import tasks from funkwhale_api.federation import views +from funkwhale_api.federation import models def test_user_can_list_their_library_follows(factories, logged_in_api_client): @@ -320,3 +321,68 @@ def test_library_follow_get_all(factories, logged_in_api_client): ], "count": 1, } + + +def test_domain_follow_get_all(factories, logged_in_api_client): + actor = logged_in_api_client.user.create_actor() + domain = factories["federation.Domain"]() + follow = factories["federation.DomainFollow"](target=domain, actor=actor) + factories["federation.DomainFollow"]() + factories["federation.Domain"]() + url = reverse("api:v1:federation:domain-follows-all") + response = logged_in_api_client.get(url) + + assert response.status_code == 200 + assert response.data == { + "results": [ + { + "uuid": str(follow.uuid), + "domain": str(domain.name), + "approved": follow.approved, + } + ], + "count": 1, + } + + +def test_can_follow_domain(factories, logged_in_api_client, mocker): + dispatch = mocker.patch("funkwhale_api.federation.routes.outbox.dispatch") + actor = logged_in_api_client.user.create_actor() + domain = factories["federation.Domain"]() + url = reverse("api:v1:federation:domain-follows-list") + response = logged_in_api_client.post(url, {"target": domain.name}) + + assert response.status_code == 201 + + follow = domain.domainfollow_received_follows.latest("id") + + assert follow.approved is None + assert follow.actor == actor + + dispatch.assert_called_once_with({"type": "Follow"}, context={"follow": follow}) + + +def test_can_undo_domain_follow(factories, logged_in_api_client, mocker): + dispatch = mocker.patch("funkwhale_api.federation.routes.outbox.dispatch") + unfollow_libs_delayed = mocker.patch.object( + tasks.domain_bulk_unfollow_libraries, "delay" + ) + actor = ( + logged_in_api_client.user.create_actor() + ) # Object of type Actor is not JSON serializable + domain = factories["federation.Domain"]() + follow = factories["federation.DomainFollow"](target=domain, actor=actor) + delete = mocker.patch.object(follow.__class__, "delete") + url = reverse( + "api:v1:federation:domain-follows-detail", kwargs={"uuid": follow.uuid} + ) + # actor = factories["federation.Actor"]() + response = logged_in_api_client.delete(url, actor=actor) + + assert response.status_code == 204 + + delete.assert_called_once_with() + dispatch.assert_called_once_with( + {"type": "Undo", "object": {"type": "Follow"}}, context={"follow": follow} + ) + unfollow_libs_delayed.assert_called_once_with(domain.name, actor_id=actor.pk) diff --git a/api/tests/federation/test_models.py b/api/tests/federation/test_models.py index 6f722c6e8a3947d11e7171f32de164ba9ac4c2e5..4c89f7f321f7a48433f15d7d9e9d22fbb1661e26 100644 --- a/api/tests/federation/test_models.py +++ b/api/tests/federation/test_models.py @@ -134,6 +134,8 @@ def test_domain_stats(factories): "emitted_library_follows": 0, "media_total_size": 0, "media_downloaded_size": 0, + "emitted_domain_follows": 0, + "received_domain_follows": 0, } domain = factories["federation.Domain"]() diff --git a/api/tests/federation/test_routes.py b/api/tests/federation/test_routes.py index 108ca41b8fa265df3387a656b943928920cd3ee3..d087eae086aa7f60662a81cbde3aa2fe3b084571 100644 --- a/api/tests/federation/test_routes.py +++ b/api/tests/federation/test_routes.py @@ -1,5 +1,7 @@ import pytest - +from django.urls import reverse +from django.conf import settings +import uuid from funkwhale_api.federation import ( activity, actors, @@ -7,8 +9,10 @@ from funkwhale_api.federation import ( jsonld, routes, serializers, + models, ) from funkwhale_api.moderation import serializers as moderation_serializers +from funkwhale_api.federation import routes @pytest.mark.parametrize( @@ -501,12 +505,16 @@ def test_outbox_delete_library(factories): def test_outbox_update_library(factories): library = factories["music.Library"]() + domain = factories["federation.Domain"](name=settings.FEDERATION_HOSTNAME) activity = list(routes.outbox_update_library({"library": library}))[0] expected = serializers.ActivitySerializer( {"type": "Update", "object": serializers.LibrarySerializer(library).data} ).data - expected["to"] = [{"type": "followers", "target": library}] + expected["to"] = [ + {"type": "followers", "target": library}, + {"type": "domain_followers", "target": domain}, + ] assert dict(activity["payload"]) == dict(expected) assert activity["actor"] == library.actor @@ -989,3 +997,81 @@ def test_outbox_flag(factory_name, factory_kwargs, factories, mocker): expected["to"] = [{"type": "actor_inbox", "actor": report.target_owner}] assert activity["payload"] == expected assert activity["actor"] == actors.get_service_actor() + + +def test_dispatch_outbox_update_domain_on_library_creation( + factories, mocker, logged_in_api_client +): + local_domain = factories["federation.Domain"](name=settings.FUNKWHALE_HOSTNAME) + actor = logged_in_api_client.user.create_actor() + url = reverse("api:v1:libraries-list") + dispatch = mocker.patch("funkwhale_api.federation.routes.outbox.dispatch") + logged_in_api_client.post( + url, {"name": "hello", "description": "world", "privacy_level": "everyone"} + ) + library = actor.libraries.first() + + dispatch.assert_called_once_with( + {"type": "Update", "object": {"type": "Domain"}}, + context={"domain": local_domain, "library": library}, + ) + + +def test_inbox_update_library_from_domain_follow( + factories, mocker, logged_in_api_client +): + + domain = factories["federation.Domain"]() + local_domain = factories["federation.Domain"](name=settings.FUNKWHALE_HOSTNAME) + domain_follow = factories["federation.DomainFollow"].create_batch(10, target=domain) + lib_uuid = uuid.uuid4() + fid = "https://{}/federation/music/libraries/{}".format(domain.name, lib_uuid) + library = factories["music.Library"](privacy_level="everyone", fid=fid) + serializer = serializers.ActivitySerializer( + {"type": "Update", "object": serializers.LibrarySerializer(library).data} + ) + serializer.data["to"] = {"type": "domain_followers", "target": domain} + payload = serializer.data + + routes.inbox_update_library_from_domain_follow( + payload, context={"domain": local_domain, "library": library} + ) + + assert models.LibraryFollow.objects.filter(target=library).count() == 10 + + +# from funkwhale_api.music import models as music_models + + +# def test_inbox_update_library_from_domain_follow( +# factories, mocker, logged_in_api_client +# ): +# from funkwhale_api.music import models as music_models + +# domain = factories["federation.Domain"]() +# actor = factories["federation.Actor"](domain=domain) +# local_domain = factories["federation.Domain"](name=settings.FUNKWHALE_HOSTNAME) +# lib_uuid = uuid.uuid4() +# fid = "https://{}/federation/music/libraries/{}".format(domain.name, lib_uuid) +# followers_url = "https://{}/federation/music/libraries/{}/followers".format( +# domain.name, lib_uuid +# ) +# library = music_models.Library( +# privacy_level="everyone", +# fid=fid, +# name="blabla", +# actor=actor, +# followers_url=followers_url, +# ) + +# serializer = serializers.ActivitySerializer( +# {"type": "Update", "object": serializers.LibrarySerializer(library).data} +# ) +# serializer.data["to"] = {"type": "domain_followers", "target": domain} +# payload = serializer.data + +# routes.inbox_update_library_from_domain_follow( +# payload, context={"domain": local_domain, "library": library} +# ) + +# assert models.LibraryFollow.objects.filter(target=library).count() == 10 diff --git a/api/tests/federation/test_tasks.py b/api/tests/federation/test_tasks.py index 82192cef01486b82965487348dc629076f4d46e2..40ed13b70d047248753fbbca520117da30461f59 100644 --- a/api/tests/federation/test_tasks.py +++ b/api/tests/federation/test_tasks.py @@ -1,7 +1,12 @@ import datetime +from funkwhale_api.music import models as music_models +from funkwhale_api.music import serializers as music_serializers + +import factory import os import pathlib import pytest +import uuid from django.utils import timezone @@ -10,6 +15,10 @@ from funkwhale_api.federation import models from funkwhale_api.federation import serializers from funkwhale_api.federation import tasks from funkwhale_api.federation import utils +from funkwhale_api.common import utils as common_utils + + +from funkwhale_api.federation.factories import MusicLibraryFactory, ActorFactory def test_clean_federation_music_cache_if_no_listen(preferences, factories): @@ -210,7 +219,7 @@ def test_update_domain_nodeinfo(factories, mocker, now, service_actor): retrieve_ap_object.assert_called_once_with( "https://actor.id", - actor=service_actor, + actor=None, queryset=models.Actor, serializer_class=serializers.ActorSerializer, ) @@ -670,3 +679,52 @@ def test_fetch_collection(mocker, r_mock): assert result["seen"] == 7 assert result["total"] == 27094 assert result["next_page"] == payloads["page2"]["next"] + + +def test_domain_bulk_follow_libraries(factories, mocker, logged_in_api_client): + domain = factories["federation.Domain"]() + actor = factories["federation.Actor"]() + i = 0 + lib1 = factories["music.Library"](privacy_level="everyone") + lib2 = factories["music.Library"](privacy_level="everyone") + lib3 = factories["music.Library"](privacy_level="everyone") + + libs = [lib1, lib2, lib3] + libs_remote = {} + libraries_dic = [] + + for lib in libs: + lib_json = music_serializers.LibraryForOwnerSerializer(lib) + libraries_dic.append(lib_json.data) + libs_remote["results"] = libraries_dic + mocker.patch( + "funkwhale_api.federation.utils.get_remote_libraries", return_value=libs_remote + ) + tasks.domain_bulk_follow_libraries(domain.name, actor_id=actor.pk) + assert music_models.Library.objects.filter(uuid=lib1.uuid).exists() + assert models.LibraryFollow.objects.filter(target__fid=lib1.fid).exists() + assert models.LibraryFollow.objects.filter(target__fid=lib2.fid).exists() + assert models.LibraryFollow.objects.filter(target__fid=lib3.fid).exists() + + +def test_domain_can_bulk_unfollow_libraries(factories, mocker, logged_in_api_client): + domain = factories["federation.Domain"]() + actor = logged_in_api_client.user.create_actor() + dispacth_unfollow = mocker.patch("funkwhale_api.federation.routes.outbox.dispatch") + i = 0 + calls = [] + follows = [] + while i < 10: + i = i + 1 + lib_uuid = uuid.uuid4() + followers_url = "https://{}/federation/music/libraries/{}/followers".format( + domain.name, lib_uuid + ) + lib = factories["music.Library"](followers_url=followers_url) + follow = factories["federation.LibraryFollow"]( + actor=actor, target=lib, follow_context="Domain" + ) + follows.append(follow) + tasks.domain_bulk_unfollow_libraries(domain.name, actor_id=actor.pk) + for follow in follows: + assert models.LibraryFollow.objects.filter(uuid=follow.uuid).exists() == False diff --git a/api/tests/music/test_serializers.py b/api/tests/music/test_serializers.py index beb30279d3d787558b8901df7cd2bb07078c7f77..ce0f948f0a832a08042989b361de9b9855e25a94 100644 --- a/api/tests/music/test_serializers.py +++ b/api/tests/music/test_serializers.py @@ -1,6 +1,6 @@ import pytest import uuid - +from django.conf import settings from funkwhale_api.common import serializers as common_serializers from funkwhale_api.federation import serializers as federation_serializers from funkwhale_api.music import licenses @@ -406,16 +406,24 @@ def test_update_library_privacy_level_broadcasts_to_followers( ): dispatch = mocker.patch("funkwhale_api.federation.routes.outbox.dispatch") library = factories["music.Library"](**{field: before}) + domain = factories["federation.Domain"](name=settings.FUNKWHALE_HOSTNAME) serializer = serializers.LibraryForOwnerSerializer( library, data={field: after}, partial=True ) assert serializer.is_valid(raise_exception=True) serializer.save() - - dispatch.assert_called_once_with( - {"type": "Update", "object": {"type": "Library"}}, context={"library": library} - ) + if field == "privacy_level": + dispatch.assert_called_with( + {"type": "Update", "object": {"type": "Domain"}}, + context={"library": library, "domain": domain}, + ) + + else: + dispatch.assert_called_once_with( + {"type": "Update", "object": {"type": "Library"}}, + context={"library": library}, + ) def test_upload_with_channel(factories, uploaded_audio_file): diff --git a/changes/changelog.d/762.newfeature b/changes/changelog.d/762.newfeature new file mode 100644 index 0000000000000000000000000000000000000000..b61b4fdeb76e43a2c6c4b1db04cd42f6eb9ceea8 --- /dev/null +++ b/changes/changelog.d/762.newfeature @@ -0,0 +1,2 @@ +Alow actor to follow entire instances (#762) +New env variable GLOBAL_FEDERATION enable should_autoapprove_follow for the domain. diff --git a/front/src/components/federation/DomainFollowButton.vue b/front/src/components/federation/DomainFollowButton.vue new file mode 100644 index 0000000000000000000000000000000000000000..338f968f77c110548ae18890f1a12dc80d5d41e9 --- /dev/null +++ b/front/src/components/federation/DomainFollowButton.vue @@ -0,0 +1,57 @@ + + + diff --git a/front/src/store/domains.js b/front/src/store/domains.js new file mode 100644 index 0000000000000000000000000000000000000000..163c4b17c3815e6fa7f83bb8b7449e49dd9bd995 --- /dev/null +++ b/front/src/store/domains.js @@ -0,0 +1,74 @@ +import axios from 'axios' +import logger from '@/logging' + +export default { + namespaced: true, + state: { + followedDomains: [], + followsByDomain: {}, + count: 0 + }, + mutations: { + follows: (state, { domain, follow }) => { + const replacement = { ...state.followsByDomain } + if (follow) { + if (state.followedDomains.indexOf(domain) === -1) { + state.followedDomains.push(domain) + replacement[domain] = follow + } + } else { + const i = state.followedDomains.indexOf(domain) + if (i > -1) { + state.followedDomains.splice(i, 1) + replacement[domain] = null + } + } + state.followsByDomain = replacement + state.count = state.followedDomains.length + }, + reset (state) { + state.followedDomains = [] + state.followsByDomain = {} + state.count = 0 + } + }, + getters: { + follow: (state) => (domain) => { + return state.followsByDomain[domain] + } + }, + actions: { + set ({ commit, state }, { name, value }) { + if (value) { + return axios.post('federation/follows/domain/', { target: name }).then((response) => { + logger.default.info('Successfully subscribed to domain') + commit('follows', { domain: name, follow: response.data } + ) + }, (response) => { + logger.default.info('Error while subscribing to domain') + commit('follows', { domain: name, follow: null }) + }) + } else { + const follow = state.followsByDomain[name] + return axios.delete(`federation/follows/domain/${follow.uuid}/`).then((response) => { + logger.default.info('Successfully unsubscribed from domain') + commit('follows', { domain: name, follow: null }) + }, (response) => { + logger.default.info('Error while unsubscribing from domain') + commit('follows', { domain: name, follow: follow }) + }) + } + }, + toggle ({ getters, dispatch }, name) { + dispatch('set', { name, value: !getters.follow(name) }) + }, + fetchFollows ({ dispatch, state, commit, rootState }, url) { + const promise = axios.get('federation/follows/domain/all/') + return promise.then((response) => { + response.data.results.forEach(result => { + commit('follows', { domain: result.domain, follow: result }) + }) + }) + } + } +} diff --git a/front/src/store/index.js b/front/src/store/index.js index a7954420346599c335856b82980522dc6239730c..7ad2b90a38a97ebb19bd00a73a564397cd3e13b0 100644 --- a/front/src/store/index.js +++ b/front/src/store/index.js @@ -4,6 +4,7 @@ import createPersistedState from 'vuex-persistedstate' import favorites from './favorites' import channels from './channels' +import domains from './domains' import libraries from './libraries' import auth from './auth' import instance from './instance' @@ -21,6 +22,7 @@ export default new Vuex.Store({ ui, auth, channels, + domains, libraries, favorites, instance, diff --git a/front/src/views/admin/moderation/DomainsDetail.vue b/front/src/views/admin/moderation/DomainsDetail.vue index cfccac97d4846dc0ead602e5788719be50b48a11..2915e09a091cbada6d7b835ca036e7d0027d30c5 100644 --- a/front/src/views/admin/moderation/DomainsDetail.vue +++ b/front/src/views/admin/moderation/DomainsDetail.vue @@ -11,6 +11,9 @@ v-title="object.name" :class="['ui', 'head', 'vertical', 'stripe', 'segment']" > +
@@ -462,11 +465,13 @@ import lodash from 'lodash' import InstancePolicyForm from '@/components/manage/moderation/InstancePolicyForm.vue' import InstancePolicyCard from '@/components/manage/moderation/InstancePolicyCard.vue' +import DomainFollowButton from '@/components/federation/DomainFollowButton.vue' export default { components: { InstancePolicyForm, - InstancePolicyCard + InstancePolicyCard, + DomainFollowButton }, props: { id: { type: String, required: true }, allowListEnabled: { type: Boolean, required: true } }, data () { @@ -477,7 +482,7 @@ export default { isLoadingPolicy: false, isLoadingAllowList: false, policy: null, - object: null, + object: {}, stats: null, showPolicyForm: false, permissions: [] diff --git a/front/vite.config.js b/front/vite.config.js index 549890e8919d16a2c3ab0eb3a050a1111fd31b6b..d1a306ade3efc00508909347a9288c32b6079ec6 100644 --- a/front/vite.config.js +++ b/front/vite.config.js @@ -1,7 +1,7 @@ // vite.config.js import { defineConfig } from 'vite' -import { createVuePlugin as vue } from "vite-plugin-vue2"; +import { createVuePlugin as vue } from 'vite-plugin-vue2' import path from 'path' @@ -16,18 +16,18 @@ export default defineConfig({ return `import jQuery from 'jquery';${src}` } } - }, + } ], server: { port: process.env.VUE_PORT || '8080', hmr: { port: process.env.FUNKWHALE_PROTOCOL === 'https' ? 443 : 8000, - protocol: process.env.FUNKWHALE_PROTOCOL === 'https' ? 'wss' : 'ws', + protocol: process.env.FUNKWHALE_PROTOCOL === 'https' ? 'wss' : 'ws' } }, resolve: { alias: { - "@": path.resolve(__dirname, "./src"), - }, - }, + '@': path.resolve(__dirname, './src') + } + } })