From bd93515f3a8ed7a8375b1a484e98fdef8c7a5b8b Mon Sep 17 00:00:00 2001 From: Petitminion Date: Fri, 27 May 2022 22:23:09 +0200 Subject: [PATCH 01/13] Allowing to follow entire instances / domains --- .env.dev | 21 --- api/Dockerfile | 111 ++++++------- api/config/settings/common.py | 1 + api/config/settings/local.py | 2 +- api/funkwhale_api/federation/activity.py | 27 ++- api/funkwhale_api/federation/admin.py | 8 + .../federation/api_serializers.py | 106 +++++++++++- api/funkwhale_api/federation/api_urls.py | 1 + api/funkwhale_api/federation/api_views.py | 94 +++++++++++ api/funkwhale_api/federation/factories.py | 15 ++ api/funkwhale_api/federation/filters.py | 6 + .../migrations/0027_auto_20220527_2006.py | 84 ++++++++++ api/funkwhale_api/federation/models.py | 66 +++++++- api/funkwhale_api/federation/routes.py | 74 +++++++++ api/funkwhale_api/federation/serializers.py | 51 +++++- api/funkwhale_api/federation/tasks.py | 122 +++++++++++++- api/funkwhale_api/federation/utils.py | 18 +- api/funkwhale_api/music/models.py | 1 + api/funkwhale_api/music/views.py | 14 +- api/tests/federation/test_api_views.py | 155 ++++++++++++++++++ api/tests/federation/test_models.py | 2 + api/tests/federation/test_routes.py | 44 ++++- .../federation/DomainFollowButton.vue | 57 +++++++ front/src/store/domains.js | 74 +++++++++ front/src/store/index.js | 2 + .../views/admin/moderation/DomainsDetail.vue | 9 +- front/vite.config.js | 33 ---- 27 files changed, 1063 insertions(+), 135 deletions(-) delete mode 100644 .env.dev create mode 100644 api/funkwhale_api/federation/migrations/0027_auto_20220527_2006.py create mode 100644 front/src/components/federation/DomainFollowButton.vue create mode 100644 front/src/store/domains.js delete mode 100644 front/vite.config.js diff --git a/.env.dev b/.env.dev deleted file mode 100644 index 66638d77f..000000000 --- a/.env.dev +++ /dev/null @@ -1,21 +0,0 @@ -DJANGO_ALLOWED_HOSTS=.funkwhale.test,localhost,nginx,0.0.0.0,127.0.0.1 -DJANGO_SETTINGS_MODULE=config.settings.local -DJANGO_SECRET_KEY=dev -C_FORCE_ROOT=true -FUNKWHALE_HOSTNAME=localhost -FUNKWHALE_PROTOCOL=http -PYTHONDONTWRITEBYTECODE=true -VUE_PORT=8080 -MUSIC_DIRECTORY_PATH=/music -BROWSABLE_API_ENABLED=True -FORWARDED_PROTO=http -LDAP_ENABLED=False -FUNKWHALE_SPA_HTML_ROOT=http://nginx/front/ -PYTHONTRACEMALLOC=0 - -# Uncomment this if you're using traefik/https -# FORCE_HTTPS_URLS=True - -# Customize to your needs -POSTGRES_VERSION=11 -DEBUG=true diff --git a/api/Dockerfile b/api/Dockerfile index 2b1e831a2..1be009f6d 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -4,37 +4,38 @@ FROM alpine:3.14 as pre-build # dependencies. This is only required until alpine 3.16 is released, since this # allows us to install poetry as package. -RUN apk add --no-cache python3 py3-cryptography py3-pip && \ - pip3 install poetry +RUN --mount=type=cache,target=/root/.cache/pip \ + apk add --no-cache python3 py3-cryptography py3-pip && \ + pip3 install poetry COPY pyproject.toml poetry.lock / RUN poetry export --without-hashes > requirements.txt RUN poetry export --dev --without-hashes > dev-requirements.txt FROM alpine:3.14 as builder -RUN \ - echo 'installing dependencies' && \ - apk add --no-cache \ - git \ - musl-dev \ - gcc \ - postgresql-dev \ - python3-dev \ - py3-psycopg2 \ - py3-cryptography \ - libldap \ - libffi-dev \ - make \ - zlib-dev \ - jpeg-dev \ - openldap-dev \ - openssl-dev \ - cargo \ - libxml2-dev \ - libxslt-dev \ - curl \ - && \ - ln -s /usr/bin/python3 /usr/bin/python +RUN --mount=type=cache,target=/root/.cache/pip \ + echo 'installing dependencies' && \ + apk add --no-cache \ + git \ + musl-dev \ + gcc \ + postgresql-dev \ + python3-dev \ + py3-psycopg2 \ + py3-cryptography \ + libldap \ + libffi-dev \ + make \ + zlib-dev \ + jpeg-dev \ + openldap-dev \ + openssl-dev \ + cargo \ + libxml2-dev \ + libxslt-dev \ + curl \ + && \ + ln -s /usr/bin/python3 /usr/bin/python # create virtual env for next stage RUN python -m venv --system-site-packages /venv @@ -44,26 +45,26 @@ ENV PATH="/venv/bin:/root/.local/bin:$PATH" VIRTUAL_ENV=/venv COPY --from=0 /requirements.txt /requirements.txt COPY --from=0 /dev-requirements.txt /dev-requirements.txt # hack around https://github.com/pypa/pip/issues/6158#issuecomment-456619072 -ENV PIP_DOWNLOAD_CACHE=/noop/ -RUN \ - echo 'installing pip requirements' && \ - pip3 install --upgrade pip && \ - pip3 install setuptools wheel && \ - # Currently we are unable to relieably build cryptography on armv7. This - # is why we need to use the package shipped by Alpine Linux, which is currently - # version 3.3.2. Since poetry does not allow in-place dependency pinning, we need - # to install the deps using pip. - cat /requirements.txt | grep -Ev 'cryptography|autobahn' | pip3 install -r /dev/stdin cryptography==3.3.2 autobahn==21.2.1 && \ - rm -rf $PIP_DOWNLOAD_CACHE +#ENV PIP_DOWNLOAD_CACHE=/noop/ +RUN --mount=type=cache,target=/root/.cache/pip \ + echo 'installing pip requirements' && \ + pip3 install --upgrade pip && \ + pip3 install setuptools wheel && \ + # Currently we are unable to relieably build cryptography on armv7. This + # is why we need to use the package shipped by Alpine Linux, which is currently + # version 3.3.2. Since poetry does not allow in-place dependency pinning, we need + # to install the deps using pip. + cat /requirements.txt | grep -Ev 'cryptography|autobahn' | pip3 install -r /dev/stdin cryptography==3.3.2 autobahn==21.2.1 && \ + #rm -rf $PIP_DOWNLOAD_CACHE -ARG install_dev_deps=0 + ARG install_dev_deps=0 RUN \ - if [ "$install_dev_deps" = "1" ] ; then \ - echo "Installing dev dependencies" && \ - cat /dev-requirements.txt | grep -Ev 'cryptography|autobahn' | pip3 install -r /dev/stdin cryptography==3.3.2 autobahn==21.2.1 \ - ; else \ - echo "Skipping dev deps installation" \ - ; fi + if [ "$install_dev_deps" = "1" ] ; then \ + echo "Installing dev dependencies" && \ + cat /dev-requirements.txt | grep -Ev 'cryptography|autobahn' | pip3 install -r /dev/stdin cryptography==3.3.2 autobahn==21.2.1 \ + ; else \ + echo "Skipping dev deps installation" \ + ; fi FROM alpine:3.14 as build-image @@ -73,18 +74,18 @@ COPY --from=builder /venv /venv ENV PATH="/venv/bin:$PATH" RUN apk add --no-cache \ - libmagic \ - bash \ - gettext \ - python3 \ - jpeg-dev \ - ffmpeg \ - libpq \ - libxml2 \ - libxslt \ - py3-cryptography \ - && \ - ln -s /usr/bin/python3 /usr/bin/python + libmagic \ + bash \ + gettext \ + python3 \ + jpeg-dev \ + ffmpeg \ + libpq \ + libxml2 \ + libxslt \ + py3-cryptography \ + && \ + ln -s /usr/bin/python3 /usr/bin/python COPY . /app WORKDIR /app diff --git a/api/config/settings/common.py b/api/config/settings/common.py index 179c01be4..bbb60b2fd 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -190,6 +190,7 @@ ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=[]) + [FUNKWHALE_HOSTNA """ List of allowed hostnames for which the Funkwhale server will answer. """ +GLOBAL_FEDERATION = env.bool("GLOBAL_FEDERATION", default=True) # APP CONFIGURATION # ------------------------------------------------------------------------------ diff --git a/api/config/settings/local.py b/api/config/settings/local.py index f0a64f0fa..176c7912a 100644 --- a/api/config/settings/local.py +++ b/api/config/settings/local.py @@ -92,7 +92,7 @@ if DEBUG_TOOLBAR_ENABLED: # ------------------------------------------------------------------------------ TEST_RUNNER = "django.test.runner.DiscoverRunner" -# CELERY +# CELERY Set to true to debug CELERY_TASK_ALWAYS_EAGER = False # END CELERY diff --git a/api/funkwhale_api/federation/activity.py b/api/funkwhale_api/federation/activity.py index 7725953e6..959abb47d 100644 --- a/api/funkwhale_api/federation/activity.py +++ b/api/funkwhale_api/federation/activity.py @@ -12,6 +12,11 @@ from funkwhale_api.common import utils as funkwhale_utils from . import contexts +import logging + +logger = logging.getLogger(__name__) + + recursive_getattr = funkwhale_utils.recursive_getattr @@ -119,13 +124,16 @@ def should_reject(fid, actor_id=None, payload={}): @transaction.atomic def receive(activity, on_behalf_of, inbox_actor=None): + """ + Receive an activity, find his recipients and save it to the database before dispatching it + """ from . import models from . import serializers from . import tasks from .routes import inbox from funkwhale_api.moderation import mrf - logger.debug( + logger.info( "[federation] Received activity from %s : %s", on_behalf_of.fid, activity ) # we ensure the activity has the bare minimum structure before storing @@ -339,6 +347,10 @@ class OutboxRouter(Router): deliveries_by_activity_uuid = {} prepared_activities = [] for activity_data in activities_data: + # If its a domain update we change the object type of the library to "Domain" to allow specific routing + for recipient in activity_data["payload"]["to"]: + if recipient is dict and "domain_followers" in recipient["type"]: + activity_data["payload"]["object"]["type"] = "Domain" activity_data["payload"]["actor"] = activity_data["actor"].fid to = activity_data["payload"].pop("to", []) cc = activity_data["payload"].pop("cc", []) @@ -461,6 +473,19 @@ def prepare_deliveries_and_inbox_items(recipient_list, type, allowed_domains=Non else: remote_inbox_urls.add(actor.shared_inbox_url or actor.inbox_url) urls.append(r["target"].followers_url) + + elif isinstance(r, dict) and r["type"] == "domain_followers": + received_follows = ( + r["target"] + .domainfollow_received_follows.filter(approved=True) + .select_related("actor__user") + ) + logger.info("received_follows" + str(received_follows)) + for follow in received_follows: + actor = follow.actor + remote_inbox_urls.add(actor.shared_inbox_url or actor.inbox_url) + urls.append(actor.shared_inbox_url) + elif isinstance(r, dict) and r["type"] == "actor_inbox": actor = r["actor"] urls.append(actor.fid) diff --git a/api/funkwhale_api/federation/admin.py b/api/funkwhale_api/federation/admin.py index 8e66708cf..0ef2e760e 100644 --- a/api/funkwhale_api/federation/admin.py +++ b/api/funkwhale_api/federation/admin.py @@ -78,6 +78,14 @@ class LibraryFollowAdmin(admin.ModelAdmin): list_select_related = True +@admin.register(models.DomainFollow) +class DomainFollowAdmin(admin.ModelAdmin): + list_display = ["actor", "target", "approved", "creation_date"] + list_filter = ["approved"] + search_fields = ["actor__fid", "target__name"] + list_select_related = True + + @admin.register(models.InboxItem) class InboxItemAdmin(admin.ModelAdmin): list_display = ["actor", "activity", "type", "is_read"] diff --git a/api/funkwhale_api/federation/api_serializers.py b/api/funkwhale_api/federation/api_serializers.py index 6ff1e1b2b..24fe52222 100644 --- a/api/funkwhale_api/federation/api_serializers.py +++ b/api/funkwhale_api/federation/api_serializers.py @@ -12,10 +12,15 @@ from funkwhale_api.common import fields as common_fields from funkwhale_api.common import serializers as common_serializers from funkwhale_api.music import models as music_models from funkwhale_api.users import serializers as users_serializers - +from funkwhale_api.federation import models as federation_models from . import filters from . import models from . import serializers as federation_serializers +from . import tasks + +import logging + +logger = logging.getLogger(__name__) class NestedLibraryFollowSerializer(serializers.ModelSerializer): @@ -24,6 +29,17 @@ class NestedLibraryFollowSerializer(serializers.ModelSerializer): fields = ["creation_date", "uuid", "fid", "approved", "modification_date"] +class NestedDomainFollowSerializer(serializers.ModelSerializer): + class Meta: + model = models.DomainFollow + fields = [ + "creation_date", + "fid", + "approved", + "modification_date", + ] + + class LibraryScanSerializer(serializers.ModelSerializer): class Meta: model = music_models.LibraryScan @@ -39,6 +55,33 @@ class LibraryScanSerializer(serializers.ModelSerializer): class DomainSerializer(serializers.Serializer): name = serializers.CharField() + follow = serializers.SerializerMethodField() + + class Meta: + model = federation_models.Domain + fields = [ + "fid", + "name", + "nodeinfo", + ] + + def get_follow(self, o): + try: + return NestedDomainFollowSerializer(o._follows[0]).data + except (AttributeError, IndexError): + return None + + +class LibraryFollowListSerializer(serializers.ListSerializer): + def create(self, validated_data): + libs = [models.LibraryFollow(**lib) for lib in validated_data] + return models.LibraryFollow.objects.bulk_create(libs, batch_size=1000) + + +class LibraryListSerializer(serializers.ListSerializer): + def create(self, validated_data): + libs = [models.Library(**lib) for lib in validated_data] + return models.Library.objects.bulk_create(libs, batch_size=1000) class LibrarySerializer(serializers.ModelSerializer): @@ -61,6 +104,7 @@ class LibrarySerializer(serializers.ModelSerializer): "follow", "latest_scan", ] + list_serializer_class = LibraryListSerializer def get_uploads_count(self, o): return max(getattr(o, "_uploads_count", 0), o.uploads_count) @@ -80,14 +124,28 @@ class LibrarySerializer(serializers.ModelSerializer): class LibraryFollowSerializer(serializers.ModelSerializer): target = common_serializers.RelatedField("uuid", LibrarySerializer(), required=True) actor = serializers.SerializerMethodField() + follow_context = serializers.ChoiceField( + choices=["Domain", "Library"], required=False + ) class Meta: model = models.LibraryFollow - fields = ["creation_date", "actor", "uuid", "target", "approved"] + fields = [ + "creation_date", + "actor", + "uuid", + "target", + "approved", + "follow_context", + ] read_only_fields = ["uuid", "actor", "approved", "creation_date"] + list_serializer_class = LibraryFollowListSerializer def validate_target(self, v): - actor = self.context["actor"] + try: + actor = self.context["actor"] + except KeyError: + raise KeyError if v.actor == actor: raise serializers.ValidationError("You cannot follow your own library") @@ -99,16 +157,54 @@ class LibraryFollowSerializer(serializers.ModelSerializer): return federation_serializers.APIActorSerializer(o.actor).data +class DomainFollowSerializer(serializers.ModelSerializer): + target = common_serializers.RelatedField("name", DomainSerializer(), required=True) + actor = serializers.SerializerMethodField() + + class Meta: + model = models.DomainFollow + fields = ["uuid", "creation_date", "actor", "target", "approved"] + read_only_fields = ["uuid", "actor", "approved", "creation_date"] + + def validate_target(self, v): + actor = self.context["actor"] + if v.domainfollow_received_follows.filter(actor=actor).exists(): + raise serializers.ValidationError("You are already following this Domain") + domain = models.Domain.objects.get(name=v.name) + now = datetime.datetime.now(timezone.utc) + if ( + domain.nodeinfo_fetch_date - now + ).total_seconds() > 86400 or "error" in domain.nodeinfo: + tasks.refresh_actor_data() + tasks.update_domain_nodeinfo(domain_name=domain) + domain = models.Domain.objects.get(name=v.name) + if "error" in domain.nodeinfo: + e = domain.nodeinfo["error"] + logger.error(f"Could not get domain information because of : {e!r}") + raise serializers.ValidationError(domain.nodeinfo["error"]) + return domain + + def get_actor(self, o): + logger.info("o is :" + str(o)) + return federation_serializers.APIActorSerializer(o.actor).data + + def serialize_generic_relation(activity, obj): data = {"type": obj._meta.label} + logger.info("data[type]" + str(data["type"])) if data["type"] == "federation.Actor": data["full_username"] = obj.full_username + elif data["type"] == "federation.Domain": + data["name"] = obj.name else: data["uuid"] = obj.uuid - if data["type"] == "music.Library": data["name"] = obj.name - if data["type"] == "federation.LibraryFollow": + # to do : test if this is works + if ( + data["type"] == "federation.LibraryFollow" + or data["type"] == "federation.DomainFollow" + ): data["approved"] = obj.approved return data diff --git a/api/funkwhale_api/federation/api_urls.py b/api/funkwhale_api/federation/api_urls.py index df5bfb2f0..1e9799521 100644 --- a/api/funkwhale_api/federation/api_urls.py +++ b/api/funkwhale_api/federation/api_urls.py @@ -5,6 +5,7 @@ from . import api_views router = routers.OptionalSlashRouter() router.register(r"fetches", api_views.FetchViewSet, "fetches") router.register(r"follows/library", api_views.LibraryFollowViewSet, "library-follows") +router.register(r"follows/domain", api_views.DomainFollowViewSet, "domain-follows") router.register(r"inbox", api_views.InboxItemViewSet, "inbox") router.register(r"libraries", api_views.LibraryViewSet, "libraries") router.register(r"domains", api_views.DomainViewSet, "domains") diff --git a/api/funkwhale_api/federation/api_views.py b/api/funkwhale_api/federation/api_views.py index f49c51c35..d9e4d5bfe 100644 --- a/api/funkwhale_api/federation/api_views.py +++ b/api/funkwhale_api/federation/api_views.py @@ -1,4 +1,6 @@ +from urllib import request import requests.exceptions +import requests from django.conf import settings from django.db import transaction @@ -11,6 +13,7 @@ from rest_framework import response from rest_framework import viewsets from funkwhale_api.common import preferences +from funkwhale_api.common import permissions as common_permissions from funkwhale_api.common import utils as common_utils from funkwhale_api.common.permissions import ConditionalAuthentication from funkwhale_api.music import models as music_models @@ -27,6 +30,10 @@ from . import serializers from . import tasks from . import utils +import logging + +logger = logging.getLogger(__name__) + @transaction.atomic def update_follow(follow, approved): @@ -263,6 +270,93 @@ class DomainViewSet( return qs +class DomainFollowViewSet( + mixins.CreateModelMixin, + mixins.ListModelMixin, + mixins.RetrieveModelMixin, + mixins.DestroyModelMixin, + viewsets.GenericViewSet, +): + lookup_field = "uuid" + queryset = ( + models.DomainFollow.objects.all() + .order_by("-creation_date") + .select_related("actor") + ) + serializer_class = api_serializers.DomainFollowSerializer + permission_classes = [oauth_permissions.ScopePermission] + required_scope = "follows" + filterset_class = filters.DomainFollowFilter + ordering_fields = ("creation_date",) + + def get_queryset(self): + qs = super().get_queryset() + return qs.filter(actor=self.request.user.actor).exclude(approved=False) + + def perform_create(self, serializer): + follow = serializer.save(actor=self.request.user.actor) + routes.outbox.dispatch({"type": "Follow"}, context={"follow": follow}) + + @transaction.atomic + def perform_destroy(self, instance): + routes.outbox.dispatch( + {"type": "Undo", "object": {"type": "Follow"}}, + context={"follow": instance}, + ) + tasks.domain_bulk_unfollow_libraries.delay( + instance.target.name, actor_id=self.request.user.actor.pk + ) + instance.delete() + + def get_serializer_context(self): + context = super().get_serializer_context() + context["actor"] = self.request.user.actor + return context + + @decorators.action(methods=["post"], detail=True) + def accept(self, request, *args, **kwargs): + try: + follow = self.queryset.get( + target__actor=self.request.user.actor, uuid=kwargs["uuid"] + ) + except models.DomainFollow.DoesNotExist: + return response.Response({}, status=404) + update_follow(follow, approved=True) + return response.Response(status=204) + + @decorators.action(methods=["post"], detail=True) + def reject(self, request, *args, **kwargs): + try: + follow = self.queryset.get( + target__actor=self.request.user.actor, uuid=kwargs["uuid"] + ) + except models.DomainFollow.DoesNotExist: + return response.Response({}, status=404) + + update_follow(follow, approved=False) + return response.Response(status=204) + + @decorators.action(methods=["get"], detail=False) + def all(self, request, *args, **kwargs): + """ + Return all the subscriptions of the current user, with only limited data + to have a performant endpoint and avoid lots of queries just to display + subscription status in the UI + """ + follows = list( + self.get_queryset().values_list("uuid", "target__name", "approved") + ) + + payload = { + "results": [ + {"uuid": str(u[0]), "domain": str(u[1]), "approved": u[2]} + for u in follows + ], + "count": len(follows), + } + return response.Response(payload, status=200) + + class ActorViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet): queryset = models.Actor.objects.select_related( "user", "channel", "summary_obj", "attachment_icon" diff --git a/api/funkwhale_api/federation/factories.py b/api/funkwhale_api/federation/factories.py index 2d2a2abae..cc7f57633 100644 --- a/api/funkwhale_api/federation/factories.py +++ b/api/funkwhale_api/federation/factories.py @@ -1,6 +1,9 @@ import uuid import factory + +# see https://stackoverflow.com/questions/60914361/python-import-error-module-factory-has-no-attribute-fuzzy +from factory import fuzzy import requests import requests_http_message_signatures from django.conf import settings @@ -238,11 +241,23 @@ class DeliveryFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory): class LibraryFollowFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory): target = factory.SubFactory(MusicLibraryFactory) actor = factory.SubFactory(ActorFactory) + follow_context = fuzzy.FuzzyChoice( + models.LibraryFollow.FOLLOW_CONTEXT_CHOICES, getter=lambda c: c[0] + ) class Meta: model = "federation.LibraryFollow" +@registry.register +class DomainollowFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory): + target = factory.SubFactory(DomainFactory) + actor = factory.SubFactory(ActorFactory) + + class Meta: + model = "federation.DomainFollow" + + class ArtistMetadataFactory(factory.Factory): name = factory.Faker("name") diff --git a/api/funkwhale_api/federation/filters.py b/api/funkwhale_api/federation/filters.py index a57bc4071..31809604f 100644 --- a/api/funkwhale_api/federation/filters.py +++ b/api/funkwhale_api/federation/filters.py @@ -34,6 +34,12 @@ class LibraryFollowFilter(django_filters.FilterSet): fields = ["approved"] +class DomainFollowFilter(django_filters.FilterSet): + class Meta: + model = models.DomainFollow + fields = ["approved"] + + class InboxItemFilter(django_filters.FilterSet): is_read = django_filters.BooleanFilter( "is_read", widget=django_filters.widgets.BooleanWidget() diff --git a/api/funkwhale_api/federation/migrations/0027_auto_20220527_2006.py b/api/funkwhale_api/federation/migrations/0027_auto_20220527_2006.py new file mode 100644 index 000000000..32159d17e --- /dev/null +++ b/api/funkwhale_api/federation/migrations/0027_auto_20220527_2006.py @@ -0,0 +1,84 @@ +# Generated by Django 3.2.13 on 2022-05-27 20:06 + +import django.core.serializers.json +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import funkwhale_api.federation.models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('federation', '0026_public_key_format'), + ] + + operations = [ + migrations.AddField( + model_name='libraryfollow', + name='follow_context', + field=models.CharField(choices=[('Domain', 'Domain'), ('Library', 'Library')], default='Library', max_length=100), + ), + migrations.AlterField( + model_name='activity', + name='object_id', + field=models.TextField(blank=True, null=True), + ), + migrations.AlterField( + model_name='activity', + name='payload', + field=models.JSONField(blank=True, default=funkwhale_api.federation.models.empty_dict, encoder=django.core.serializers.json.DjangoJSONEncoder, max_length=50000), + ), + migrations.AlterField( + model_name='activity', + name='related_object_id', + field=models.TextField(blank=True, null=True), + ), + migrations.AlterField( + model_name='actor', + name='manually_approves_followers', + field=models.BooleanField(default=None, null=True), + ), + migrations.AlterField( + model_name='domain', + name='nodeinfo', + field=models.JSONField(blank=True, default=funkwhale_api.federation.models.empty_dict, max_length=50000), + ), + migrations.AlterField( + model_name='fetch', + name='detail', + field=models.JSONField(blank=True, default=funkwhale_api.federation.models.empty_dict, encoder=django.core.serializers.json.DjangoJSONEncoder, max_length=50000), + ), + migrations.AlterField( + model_name='follow', + name='approved', + field=models.BooleanField(default=None, null=True), + ), + migrations.AlterField( + model_name='libraryfollow', + name='approved', + field=models.BooleanField(default=None, null=True), + ), + migrations.AlterField( + model_name='librarytrack', + name='metadata', + field=models.JSONField(blank=True, default=funkwhale_api.federation.models.empty_dict, encoder=django.core.serializers.json.DjangoJSONEncoder, max_length=10000), + ), + migrations.CreateModel( + name='DomainFollow', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('fid', models.URLField(blank=True, max_length=500, null=True, unique=True)), + ('uuid', models.UUIDField(default=uuid.uuid4, unique=True)), + ('creation_date', models.DateTimeField(default=django.utils.timezone.now)), + ('modification_date', models.DateTimeField(auto_now=True)), + ('approved', models.BooleanField(default=None, null=True)), + ('actor', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='domain_follows', to='federation.actor')), + ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='domainfollow_received_follows', to='federation.domain')), + ], + options={ + 'unique_together': {('actor', 'target')}, + }, + ), + ] diff --git a/api/funkwhale_api/federation/models.py b/api/funkwhale_api/federation/models.py index d9ea20180..835cede49 100644 --- a/api/funkwhale_api/federation/models.py +++ b/api/funkwhale_api/federation/models.py @@ -21,6 +21,10 @@ from funkwhale_api.music import utils as music_utils from . import utils as federation_utils +import logging + +logger = logging.getLogger(__name__) + TYPE_CHOICES = [ ("Person", "Person"), ("Tombstone", "Tombstone"), @@ -151,6 +155,12 @@ class Domain(models.Model): emitted_library_follows=models.Count( "actors__library_follows", distinct=True ), + received_domain_follows=models.Count( + "domainfollow_received_follows", distinct=True + ), + emitted_domain_follows=models.Count( + "actors__domain_follows", distinct=True + ), actors=models.Count("actors", distinct=True), ) data["artists"] = music_models.Artist.objects.filter( @@ -171,10 +181,24 @@ class Domain(models.Model): ) return data + def request_libraries_follows(self, actor): + from funkwhale_api.federation import tasks + + common_utils.on_commit( + tasks.domain_bulk_follow_libraries.delay, self.name, actor_id=actor.pk + ) + @property def is_local(self): return self.name == settings.FEDERATION_HOSTNAME + # to do : if false the follow will we saved as approved=none in the db. + # This should be handled by the fronted : when we approved the follow a accept activity is sent ? + # post to DomainFollow -> update_follow ? + + def should_autoapprove_follow(self, actor): + return settings.GLOBAL_FEDERATION + class Actor(models.Model): ap_type = "Actor" @@ -451,7 +475,8 @@ class Activity(models.Model): type = models.CharField(db_index=True, null=True, max_length=100) # generic relations - object_id = models.IntegerField(null=True, blank=True) + # This change is needed because the primary key is a text for domain and an integer for libraries + object_id = models.TextField(null=True, blank=True) object_content_type = models.ForeignKey( ContentType, null=True, @@ -469,7 +494,8 @@ class Activity(models.Model): related_name="targeting_activities", ) target = GenericForeignKey("target_content_type", "target_id") - related_object_id = models.IntegerField(null=True, blank=True) + # This change is needed because object_id is a text for domain and an integer for libraries + related_object_id = models.TextField(null=True, blank=True) related_object_content_type = models.ForeignKey( ContentType, null=True, @@ -519,9 +545,29 @@ class LibraryFollow(AbstractFollow): "music.Library", related_name="received_follows", on_delete=models.CASCADE ) + # This allow to know what libraries follows to delete. Exemple : if the delete comes from unfollow domain, only the libraries with contexte = [domain] will be deleted + # if the library is not followed whyle domain follow : context = domain + # if the library was followed before the domain follow : context = library (excluded from domain bulk lib follow) + # if the library is followed after the domain : can't do, to implement in the front ? + FOLLOW_CONTEXT_CHOICES = [("Domain", "Domain"), ("Library", "Library")] + follow_context = models.CharField( + max_length=100, + choices=FOLLOW_CONTEXT_CHOICES, + default="Library", + ) + class Meta: unique_together = ["actor", "target"] + # to do : test. Is libraryfollow deletion done in fronted ? If yes this need to be implemented to unfollow domain frontend button + # def delete(self): + # from . import routes + + # routes.outbox.dispatch( + # {"type": "Undo", "object": {"type": "Follow"}}, context={"follow": instance} + # ) + # super(LibraryFollow, self).delete() + class Library(models.Model): creation_date = models.DateTimeField(default=timezone.now) @@ -636,3 +682,19 @@ def update_denormalization_follow_deleted(sender, instance, **kwargs): music_models.TrackActor.objects.filter( actor=instance.actor, upload__in=instance.target.uploads.all() ).delete() + + +class DomainFollow(AbstractFollow): + actor = models.ForeignKey( + Actor, related_name="domain_follows", on_delete=models.CASCADE + ) + + target = models.ForeignKey( + "federation.Domain", + # domainfollow_received_follows + related_name="%(class)s_received_follows", + on_delete=models.CASCADE, + ) + + class Meta: + unique_together = ["actor", "target"] diff --git a/api/funkwhale_api/federation/routes.py b/api/funkwhale_api/federation/routes.py index 0a45f1d64..7b300861b 100644 --- a/api/funkwhale_api/federation/routes.py +++ b/api/funkwhale_api/federation/routes.py @@ -1,5 +1,7 @@ import logging import uuid +from urllib.parse import urlparse + from django.db.models import Q @@ -70,6 +72,8 @@ def outbox_accept(context): follow = context["follow"] if follow._meta.label == "federation.LibraryFollow": actor = follow.target.actor + elif follow._meta.label == "federation.DomainFollow": + actor = follow.target.service_actor else: actor = follow.target payload = serializers.AcceptFollowSerializer(follow, context={"actor": actor}).data @@ -133,6 +137,8 @@ def outbox_undo_follow(context): actor = follow.actor if follow._meta.label == "federation.LibraryFollow": recipient = follow.target.actor + elif follow._meta.label == "federation.DomainFollow": + recipient = follow.target.service_actor else: recipient = follow.target payload = serializers.UndoFollowSerializer(follow, context={"actor": actor}).data @@ -148,11 +154,22 @@ def outbox_undo_follow(context): @outbox.register({"type": "Follow"}) def outbox_follow(context): follow = context["follow"] + # context["follow"] keys are dict_keys(['_state', 'id', 'fid', 'uuid', 'creation_date', 'modification_date', 'approved', 'actor_id', 'target_id']) + # context["follow"].target keys are ['_state', 'name', 'creation_date', 'nodeinfo_fetch_date', 'nodeinfo', 'service_actor_id', 'allowed']) + if follow._meta.label == "federation.LibraryFollow": target = follow.target.actor + elif follow._meta.label == "federation.DomainFollow": + # sould be target = follow.target.service_actor but this doesn't work if the db isn't updated when the queries is made -> + # we should trigger the db update before the view is called ? Actually the db update is in the domainfollowserializer but this do not make the trick + # is it a cache problem ? + # temporary solution : + domain = models.Domain.objects.get(name=context["follow"].target.name) + target = domain.service_actor else: target = follow.target payload = serializers.FollowSerializer(follow, context={"actor": follow.actor}).data + # This generate an activity and send it to activity.OutboxRouter yield { "type": "Follow", "actor": follow.actor, @@ -286,6 +303,62 @@ def inbox_update_library(payload, context): ) +# if a library is created we inform the followers of the domain +@outbox.register({"type": "Update", "object.type": "Domain"}) +def outbox_update_domain_on_library_creation(context): + logger.info("OUTBOX DOMAIN UPDATE") + domain = context["domain"] + library = context["library"] + + serializer = serializers.ActivitySerializer( + {"type": "Update", "object": serializers.LibrarySerializer(library).data} + ) + + yield { + "type": "Update", + "actor": library.actor, + "payload": with_recipients( + serializer.data, + to=[ + {"type": "domain_followers", "target": domain}, + ], + ), + } + + +@inbox.register({"type": "Update", "object.type": "Domain"}) +def inbox_update_library_domain(payload, context): + logger.info( + "INBOX UPDATE DOMAIN context and payload " + str(context) + str(payload) + ) + domain = urlparse(payload["actor"]).netloc + library = payload["object"] + actors = [actor for actor in models.DomainFollow.objects.filter(target=domain)] + serializer = serializers.LibrarySerializer(data=library) + if serializer.is_valid(): + serializer.save() + else: + logger.debug( + "Discarding creation of library %s because of payload errors: %s", + library["id"], + serializer.errors, + ) + from . import api_serializers + + for actor in actors: + serializer = api_serializers.LibraryFollowSerializer( + target=library, follow_context="Domain", context={"actor": actor} + ) + if serializer.is_valid(): + serializer.save() + else: + logger.debug( + "Discarding follow of library %s because of payload errors: %s", + domain, + serializer.errors, + ) + + @inbox.register({"type": "Delete", "object.type": "Audio"}) def inbox_delete_audio(payload, context): actor = context["actor"] @@ -507,6 +580,7 @@ def outbox_delete_actor(context): "Organization", "Service", "Group", + "Domain", ], } ) diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py index d3263507b..4df1c3170 100644 --- a/api/funkwhale_api/federation/serializers.py +++ b/api/funkwhale_api/federation/serializers.py @@ -599,9 +599,11 @@ class BaseActivitySerializer(serializers.Serializer): ) +# TODO class FollowSerializer(serializers.Serializer): id = serializers.URLField(max_length=500) - object = serializers.URLField(max_length=500) + # if obj is a domain its not a valid url + object = serializers.CharField(max_length=500) actor = serializers.URLField(max_length=500) type = serializers.ChoiceField(choices=["Follow"]) @@ -630,6 +632,14 @@ class FollowSerializer(serializers.Serializer): except music_models.Library.DoesNotExist: pass + try: + qs = models.Domain.objects.filter(name=v) + # if recipient: + # qs = qs.filter(actor=recipient) + return qs.get() + except models.Domain.DoesNotExist: + pass + raise serializers.ValidationError("Target not found") def validate_actor(self, v): @@ -642,12 +652,16 @@ class FollowSerializer(serializers.Serializer): raise serializers.ValidationError("Actor not found") def save(self, **kwargs): + logger.info("saving") target = self.validated_data["object"] if target._meta.label == "music.Library": follow_class = models.LibraryFollow + elif target._meta.label == "federation.Domain": + follow_class = models.DomainFollow else: follow_class = models.Follow + logger.info("followclass is " + str(follow_class) + str(target._meta.label)) defaults = kwargs defaults["fid"] = self.validated_data["id"] approved = kwargs.pop("approved", None) @@ -675,13 +689,22 @@ class FollowSerializer(serializers.Serializer): return follow def to_representation(self, instance): - return { - "@context": jsonld.get_default_context(), - "actor": instance.actor.fid, - "id": instance.get_federation_id(), - "object": instance.target.fid, - "type": "Follow", - } + try: + return { + "@context": jsonld.get_default_context(), + "actor": instance.actor.fid, + "id": instance.get_federation_id(), + "object": instance.target.fid, + "type": "Follow", + } + except: + return { + "@context": jsonld.get_default_context(), + "actor": instance.actor.fid, + "id": instance.get_federation_id(), + "object": instance.target.name, + "type": "Follow", + } class APIFollowSerializer(serializers.ModelSerializer): @@ -722,6 +745,9 @@ class FollowActionSerializer(serializers.Serializer): if target._meta.label == "music.Library": expected = target.actor follow_class = models.LibraryFollow + elif target._meta.label == "federation.Domain": + expected = target.service_actor + follow_class = models.DomainFollow else: expected = target follow_class = models.Follow @@ -744,9 +770,14 @@ class FollowActionSerializer(serializers.Serializer): def to_representation(self, instance): if instance.target._meta.label == "music.Library": actor = instance.target.actor + elif instance.target._meta.label == "federation.Domain": + actor = actors.get_service_actor() else: - actor = instance.target + actor = instance.target + logger.info( + "actor is caus metalabe is " + str(actor) + str(instance.target.name) + ) return { "@context": jsonld.get_default_context(), "id": instance.get_federation_id() + "/{}".format(self.action_type), @@ -767,6 +798,8 @@ class AcceptFollowSerializer(FollowActionSerializer): follow.save() if follow.target._meta.label == "music.Library": follow.target.schedule_scan(actor=follow.actor) + if follow.target._meta.label == "federation.Domain": + follow.target.request_libraries_follows(actor=follow.actor) return follow diff --git a/api/funkwhale_api/federation/tasks.py b/api/funkwhale_api/federation/tasks.py index c040082aa..d2a7e4318 100644 --- a/api/funkwhale_api/federation/tasks.py +++ b/api/funkwhale_api/federation/tasks.py @@ -3,7 +3,8 @@ import json import logging import os import requests - +import time +from django.db.utils import IntegrityError from django.conf import settings from django.db import transaction from django.db.models import Q, F @@ -31,6 +32,8 @@ from . import serializers from . import routes from . import utils from . import webfinger +from . import api_serializers +from . import models logger = logging.getLogger(__name__) @@ -90,7 +93,7 @@ def dispatch_inbox(activity, call_handlers=True): Given an activity instance, triggers our internal delivery logic (follow creation, etc.) """ - + logger.info("dispatch inbox with activity " + str(activity)) routes.inbox.dispatch( activity.payload, context={ @@ -625,3 +628,118 @@ def fetch_collection(url, max_pages, channel, is_page=False): results["errored"], ) return results + + +@celery.app.task(name="federation.domain_bulk_follow_libraries") +@celery.require_instance( + models.Actor.objects.all(), + "actor", +) +def domain_bulk_follow_libraries(name, actor): + # to do : check utils.retrieve_ap_object for more thing to validate ? + libs = [] + if name.startswith("https://"): + pass + else: + url = f"https://{name}/api/v1/libraries?page_size=1000&scope:domain={name}" + response = utils.get_remote_libraries(url, actor) + data, updated = mrf.inbox.apply(response) + if not data: + raise exceptions.BlockedActorOrDomain() + + logger.info("response[result]" + str(response["results"])) + serializer = api_serializers.LibrarySerializer(data=response["results"], many=True) + # we do not raise exeption since we know some libs will not be valid (already followed) + serializer.is_valid() + libs = serializer.save() + if "next" in response.keys(): + follow_libraries(libs, actor) + url = response["next"] + domain_bulk_follow_libraries(url, actor) + else: + return follow_libraries(response["results"], actor) + + +def follow_libraries(libs, actor): + fids = [] + for lib in libs: + fids.append(lib["fid"]) + + libs = music_models.Library.objects.in_bulk(fids, field_name="fid") + library_follow = {} + libraries_follow = [] + for lib in libs.values(): + library_follow["target"] = lib.uuid + library_follow["follow_context"] = "Domain" + libraries_follow.append(library_follow) + + serializer = api_serializers.LibraryFollowSerializer( + data=libraries_follow, context={"actor": actor}, many=True + ) + serializer.is_valid(raise_exception=True) + follows = serializer.save(actor=actor) + # To do : optimise ! bulk dispatch follows + for follow in follows: + routes.outbox.dispatch({"type": "Follow"}, context={"follow": follow}) + return follows + + +@celery.app.task(name="federation.domain_bulk_unfollow_libraries") +@celery.require_instance( + models.Actor.objects.all(), + "actor", +) +def domain_bulk_unfollow_libraries(name, actor): + lib_follows = models.LibraryFollow.objects.filter( + actor=actor, + target__followers_url__startswith=f"https://{name}", + follow_context="Domain", + ) + # to do : optimize + for follow in lib_follows: + routes.outbox.dispatch( + {"type": "Undo", "object": {"type": "Follow"}}, context={"follow": follow} + ) + return lib_follows.delete() + + +# to do ? +# usefull when a new lib is created and we need to dispatch the activity to all the actors following the domain +@celery.app.task(name="federation.domain_bulk_unfollow_libraries") +def actors_bulk_follow_library(actors, library): + # how to pass many actor and one lib ? + serializer = api_serializers.LibraryFollowSerializer( + data=libraries_follow, context={"actor": actor}, many=True + ) + + +# To do : to delete + +from requests import HTTPError +from funkwhale_api.federation import actors as actors_utils + + +@celery.app.task(name="federation.refresh_actor_data") +def refresh_actor_data(): + actors = models.Actor.objects.all().prefetch_related() + for actor in actors: + try: + data = actors_utils.get_actor_data(actor.fid) + except HTTPError as e: + logger.info( + f"Actor couldn't be fetch because of the following exeption : {e!r}" + ) + if e.response.status_code == 410: + logger.info("Purging actor : {actor.fid!r}") + purge_actors([actor.id], [actor.domain]) + continue + continue + except Exception as e: + logger.info( + f"Actor couldn't be fetch because of the following exeption : {e!r}" + ) + continue + serializer = serializers.ActorSerializer(data=data) + serializer.is_valid(raise_exception=True) + serializer.save(last_fetch_date=timezone.now()) + return diff --git a/api/funkwhale_api/federation/utils.py b/api/funkwhale_api/federation/utils.py index 2bac8daf8..020332bb8 100644 --- a/api/funkwhale_api/federation/utils.py +++ b/api/funkwhale_api/federation/utils.py @@ -10,10 +10,13 @@ from django.db.models import CharField, Q, Value from funkwhale_api.common import session from funkwhale_api.moderation import mrf - from . import exceptions from . import signing +import logging + +logger = logging.getLogger(__name__) + def full_url(path): """ @@ -82,7 +85,6 @@ def retrieve_ap_object( existing = queryset.objects.filter(fid=fid).first() if existing: return existing - auth = ( None if not actor else signing.get_auth(actor.private_key, actor.private_key_id) ) @@ -291,3 +293,15 @@ def can_manage(obj_owner, actor): return True return False + + +def get_remote_libraries(url, actor): + auth = signing.get_auth(actor.private_key, actor.private_key_id) + response = session.get_session().get( + auth=auth, + url=url, + headers={"Content-Type": "application/activity+json"}, + ) + response.raise_for_status() + response = response.json() + return response diff --git a/api/funkwhale_api/music/models.py b/api/funkwhale_api/music/models.py index 82464e40a..07aa0c4f6 100644 --- a/api/funkwhale_api/music/models.py +++ b/api/funkwhale_api/music/models.py @@ -1270,6 +1270,7 @@ SCAN_STATUS = [ ] +# to do domain scan ? class LibraryScan(models.Model): actor = models.ForeignKey( "federation.Actor", null=True, blank=True, on_delete=models.CASCADE diff --git a/api/funkwhale_api/music/views.py b/api/funkwhale_api/music/views.py index d7eba93cf..bf2310054 100644 --- a/api/funkwhale_api/music/views.py +++ b/api/funkwhale_api/music/views.py @@ -290,7 +290,18 @@ class LibraryViewSet( return qs def perform_create(self, serializer): - serializer.save(actor=self.request.user.actor) + lib = serializer.save(actor=self.request.user.actor) + if lib.privacy_level == "everyone": + lib.actor = self.request.user.actor + routes.outbox.dispatch( + {"type": "Update", "object": {"type": "Domain"}}, + context={ + "domain": federation_models.Domain.objects.get( + name=settings.FUNKWHALE_HOSTNAME + ), + "library": lib, + }, + ) @transaction.atomic def perform_destroy(self, instance): @@ -550,6 +561,7 @@ def record_downloads(f): @record_downloads +# to do : tests fail here ? def handle_serve( upload, user, format=None, max_bitrate=None, proxy_media=True, download=True ): diff --git a/api/tests/federation/test_api_views.py b/api/tests/federation/test_api_views.py index e0fcc6edf..14316bd84 100644 --- a/api/tests/federation/test_api_views.py +++ b/api/tests/federation/test_api_views.py @@ -8,6 +8,7 @@ from funkwhale_api.federation import api_serializers from funkwhale_api.federation import serializers from funkwhale_api.federation import tasks from funkwhale_api.federation import views +from funkwhale_api.federation import models def test_user_can_list_their_library_follows(factories, logged_in_api_client): @@ -320,3 +321,157 @@ def test_library_follow_get_all(factories, logged_in_api_client): ], "count": 1, } + + +def test_domain_follow_get_all(factories, logged_in_api_client): + actor = logged_in_api_client.user.create_actor() + domain = factories["federation.Domain"]() + follow = factories["federation.DomainFollow"](target=domain, actor=actor) + factories["federation.DomainFollow"]() + factories["federation.Domain"]() + url = reverse("api:v1:federation:domain-follows-all") + response = logged_in_api_client.get(url) + + assert response.status_code == 200 + assert response.data == { + "results": [ + { + "uuid": str(follow.uuid), + "domain": str(domain.name), + "approved": follow.approved, + } + ], + "count": 1, + } + + +def test_can_follow_domain(factories, logged_in_api_client, mocker): + dispatch = mocker.patch("funkwhale_api.federation.routes.outbox.dispatch") + actor = logged_in_api_client.user.create_actor() + domain = factories["federation.Domain"]() + url = reverse("api:v1:federation:domain-follows-list") + response = logged_in_api_client.post(url, {"target": domain.name}) + + assert response.status_code == 201 + + follow = domain.domainfollow_received_follows.latest("id") + + assert follow.approved is None + assert follow.actor == actor + + dispatch.assert_called_once_with({"type": "Follow"}, context={"follow": follow}) + + +def test_can_undo_domain_follow(factories, logged_in_api_client, mocker): + dispatch = mocker.patch("funkwhale_api.federation.routes.outbox.dispatch") + unfollow_libs_delayed = mocker.patch.object( + tasks.domain_bulk_unfollow_libraries, "delay" + ) + actor = ( + logged_in_api_client.user.create_actor() + ) # Object of type Actor is not JSON serializable + domain = factories["federation.Domain"]() + follow = factories["federation.DomainFollow"](target=domain, actor=actor) + delete = mocker.patch.object(follow.__class__, "delete") + url = reverse( + "api:v1:federation:domain-follows-detail", kwargs={"uuid": follow.uuid} + ) + # actor = factories["federation.Actor"]() + response = logged_in_api_client.delete(url, actor=actor) + + assert response.status_code == 204 + + delete.assert_called_once_with() + dispatch.assert_called_once_with( + {"type": "Undo", "object": {"type": "Follow"}}, context={"follow": follow} + ) + unfollow_libs_delayed.assert_called_once_with(domain.name, actor_id=actor.pk) + + +# to do : deplacer dans task. +import factory +import uuid +from funkwhale_api.federation.factories import MusicLibraryFactory, ActorFactory +from funkwhale_api.common import utils as common_utils + + +def test_domain_bulk_follow_libraries(factories, mocker, logged_in_api_client): + domain = factories["federation.Domain"]() + actor = factories["federation.Actor"]() + i = 0 + lib_remote = {} + libraries_dic = [] + while i < 10: + # this only works if there is no subfactories within the factory class + # So we are manually converting subfactories obj in dic format see https://github.com/FactoryBoy/factory_boy/issues/68 + lib_dic = factory.build(dict, FACTORY_CLASS=MusicLibraryFactory) + actor_dic = vars(lib_dic["actor"]) + lib_dic["actor"] = actor_dic + libraries_dic.append(lib_dic) + i = i + 1 + lib_remote["results"] = libraries_dic + dispatch = mocker.patch("funkwhale_api.federation.routes.outbox.dispatch") + mocker.patch( + "funkwhale_api.federation.utils.get_remote_libraries", return_value=lib_remote + ) + calls = [] + follows = tasks.domain_bulk_follow_libraries(domain.name, actor_id=actor.pk) + import time + + time.sleep(1) + for follow in follows: + call = {} + call["follow_context"] = "Follow" + call["context"] = {"follow": follow} + calls.append(call) + dispatch.assert_has_calls(calls) + + +def test_domain_can_bulk_unfollow_libraries(factories, mocker, logged_in_api_client): + domain = factories["federation.Domain"]() + actor = logged_in_api_client.user.create_actor() + dispacth_unfollow = mocker.patch("funkwhale_api.federation.routes.outbox.dispatch") + i = 0 + calls = [] + follows = [] + while i < 10: + i = i + 1 + lib_uuid = uuid.uuid4() + followers_url = "https://{}/federation/music/libraries/{}/followers".format( + domain.name, lib_uuid + ) + lib = factories["music.Library"](followers_url=followers_url) + follow = factories["federation.LibraryFollow"]( + actor=actor, target=lib, follow_context="Domain" + ) + follows.append(follow) + tasks.domain_bulk_unfollow_libraries(domain.name, actor_id=actor.pk) + # to do : find a way to create calls + # print(str(dispacth_unfollow.call_args_list())) + # dispacth_unfollow.assert_has_calls( + # [ + # call( + # {"type": "Undo", "object": {"type": "Follow"}}, + # context={"follow": follow}, + # ) + # ] + # ) + for follow in follows: + assert models.LibraryFollow.objects.filter(uuid=follow.uuid).exists() == False + + +# to do : a deplacer dans test_routes +from funkwhale_api.federation import routes + + +def outbox_update_domain(factories): + domain = factories["federation.Domain"]() + library = factories["music.Library"]() + context = {"domain": domain.name, "library": library} + expected = { + "type": "Update", + "payload": with_recipients( + serializer.data, to=[{"type": "followers", "target": domain}] + ), + } + assert routes.outbox_update_domain(context) == expected diff --git a/api/tests/federation/test_models.py b/api/tests/federation/test_models.py index 6f722c6e8..4c89f7f32 100644 --- a/api/tests/federation/test_models.py +++ b/api/tests/federation/test_models.py @@ -134,6 +134,8 @@ def test_domain_stats(factories): "emitted_library_follows": 0, "media_total_size": 0, "media_downloaded_size": 0, + "emitted_domain_follows": 0, + "received_domain_follows": 0, } domain = factories["federation.Domain"]() diff --git a/api/tests/federation/test_routes.py b/api/tests/federation/test_routes.py index 108ca41b8..b6b0eac26 100644 --- a/api/tests/federation/test_routes.py +++ b/api/tests/federation/test_routes.py @@ -1,5 +1,7 @@ import pytest - +from django.urls import reverse +from django.conf import settings +import uuid from funkwhale_api.federation import ( activity, actors, @@ -7,6 +9,7 @@ from funkwhale_api.federation import ( jsonld, routes, serializers, + models, ) from funkwhale_api.moderation import serializers as moderation_serializers @@ -989,3 +992,42 @@ def test_outbox_flag(factory_name, factory_kwargs, factories, mocker): expected["to"] = [{"type": "actor_inbox", "actor": report.target_owner}] assert activity["payload"] == expected assert activity["actor"] == actors.get_service_actor() + + +def test_dispatch_outbox_update_domain_on_library_creation( + factories, mocker, logged_in_api_client +): + local_domain = factories["federation.Domain"](name=settings.FUNKWHALE_HOSTNAME) + actor = logged_in_api_client.user.create_actor() + url = reverse("api:v1:libraries-list") + dispatch = mocker.patch("funkwhale_api.federation.routes.outbox.dispatch") + logged_in_api_client.post( + url, {"name": "hello", "description": "world", "privacy_level": "everyone"} + ) + library = actor.libraries.first() + + dispatch.assert_called_once_with( + {"type": "Update", "object": {"type": "Domain"}}, + context={"domain": local_domain, "library": library}, + ) + + +def test_dispatch_inbox_update_library_domain(factories, mocker, logged_in_api_client): + + domain = factories["federation.Domain"]() + local_domain = factories["federation.Domain"](name=settings.FUNKWHALE_HOSTNAME) + domain_follow = factories["federation.DomainFollow"].create_batch(10, target=domain) + lib_uuid = uuid.uuid4() + fid = "https://{}/federation/music/libraries/{}".format(domain.name, lib_uuid) + library = factories["music.Library"](privacy_level="everyone", fid=fid) + payload = {"object": library, "to": {"type": "followers", "target": domain}} + dispatch = mocker.patch("funkwhale_api.federation.routes.inbox.dispatch") + routes.inbox_update_library_domain( + payload, context={"domain": local_domain, "library": library} + ) + + assert models.LibraryFollow.objects.filter(target=library).count() == 10 + + +# to do ? +# def test_library_follow_on_remote_library_creation() diff --git a/front/src/components/federation/DomainFollowButton.vue b/front/src/components/federation/DomainFollowButton.vue new file mode 100644 index 000000000..338f968f7 --- /dev/null +++ b/front/src/components/federation/DomainFollowButton.vue @@ -0,0 +1,57 @@ + + + diff --git a/front/src/store/domains.js b/front/src/store/domains.js new file mode 100644 index 000000000..163c4b17c --- /dev/null +++ b/front/src/store/domains.js @@ -0,0 +1,74 @@ +import axios from 'axios' +import logger from '@/logging' + +export default { + namespaced: true, + state: { + followedDomains: [], + followsByDomain: {}, + count: 0 + }, + mutations: { + follows: (state, { domain, follow }) => { + const replacement = { ...state.followsByDomain } + if (follow) { + if (state.followedDomains.indexOf(domain) === -1) { + state.followedDomains.push(domain) + replacement[domain] = follow + } + } else { + const i = state.followedDomains.indexOf(domain) + if (i > -1) { + state.followedDomains.splice(i, 1) + replacement[domain] = null + } + } + state.followsByDomain = replacement + state.count = state.followedDomains.length + }, + reset (state) { + state.followedDomains = [] + state.followsByDomain = {} + state.count = 0 + } + }, + getters: { + follow: (state) => (domain) => { + return state.followsByDomain[domain] + } + }, + actions: { + set ({ commit, state }, { name, value }) { + if (value) { + return axios.post('federation/follows/domain/', { target: name }).then((response) => { + logger.default.info('Successfully subscribed to domain') + commit('follows', { domain: name, follow: response.data } + ) + }, (response) => { + logger.default.info('Error while subscribing to domain') + commit('follows', { domain: name, follow: null }) + }) + } else { + const follow = state.followsByDomain[name] + return axios.delete(`federation/follows/domain/${follow.uuid}/`).then((response) => { + logger.default.info('Successfully unsubscribed from domain') + commit('follows', { domain: name, follow: null }) + }, (response) => { + logger.default.info('Error while unsubscribing from domain') + commit('follows', { domain: name, follow: follow }) + }) + } + }, + toggle ({ getters, dispatch }, name) { + dispatch('set', { name, value: !getters.follow(name) }) + }, + fetchFollows ({ dispatch, state, commit, rootState }, url) { + const promise = axios.get('federation/follows/domain/all/') + return promise.then((response) => { + response.data.results.forEach(result => { + commit('follows', { domain: result.domain, follow: result }) + }) + }) + } + } +} diff --git a/front/src/store/index.js b/front/src/store/index.js index a79544203..7ad2b90a3 100644 --- a/front/src/store/index.js +++ b/front/src/store/index.js @@ -4,6 +4,7 @@ import createPersistedState from 'vuex-persistedstate' import favorites from './favorites' import channels from './channels' +import domains from './domains' import libraries from './libraries' import auth from './auth' import instance from './instance' @@ -21,6 +22,7 @@ export default new Vuex.Store({ ui, auth, channels, + domains, libraries, favorites, instance, diff --git a/front/src/views/admin/moderation/DomainsDetail.vue b/front/src/views/admin/moderation/DomainsDetail.vue index cfccac97d..2915e09a0 100644 --- a/front/src/views/admin/moderation/DomainsDetail.vue +++ b/front/src/views/admin/moderation/DomainsDetail.vue @@ -11,6 +11,9 @@ v-title="object.name" :class="['ui', 'head', 'vertical', 'stripe', 'segment']" > +
@@ -462,11 +465,13 @@ import lodash from 'lodash' import InstancePolicyForm from '@/components/manage/moderation/InstancePolicyForm.vue' import InstancePolicyCard from '@/components/manage/moderation/InstancePolicyCard.vue' +import DomainFollowButton from '@/components/federation/DomainFollowButton.vue' export default { components: { InstancePolicyForm, - InstancePolicyCard + InstancePolicyCard, + DomainFollowButton }, props: { id: { type: String, required: true }, allowListEnabled: { type: Boolean, required: true } }, data () { @@ -477,7 +482,7 @@ export default { isLoadingPolicy: false, isLoadingAllowList: false, policy: null, - object: null, + object: {}, stats: null, showPolicyForm: false, permissions: [] diff --git a/front/vite.config.js b/front/vite.config.js deleted file mode 100644 index 549890e89..000000000 --- a/front/vite.config.js +++ /dev/null @@ -1,33 +0,0 @@ -// vite.config.js - -import { defineConfig } from 'vite' -import { createVuePlugin as vue } from "vite-plugin-vue2"; - -import path from 'path' - -// https://vitejs.dev/config/ -export default defineConfig({ - plugins: [ - vue(), - { - name: 'fix-fomantic-ui-css', - transform (src, id) { - if (id.includes('fomantic-ui-css') && id.endsWith('.min.js')) { - return `import jQuery from 'jquery';${src}` - } - } - }, - ], - server: { - port: process.env.VUE_PORT || '8080', - hmr: { - port: process.env.FUNKWHALE_PROTOCOL === 'https' ? 443 : 8000, - protocol: process.env.FUNKWHALE_PROTOCOL === 'https' ? 'wss' : 'ws', - } - }, - resolve: { - alias: { - "@": path.resolve(__dirname, "./src"), - }, - }, -}) -- GitLab From b38e38388f120a9e2f2d9fa40932a086c03dbcae Mon Sep 17 00:00:00 2001 From: Petitminion Date: Wed, 1 Jun 2022 11:28:13 +0200 Subject: [PATCH 02/13] Cleaning previous commit, adding changelog and resolving issues --- api/funkwhale_api/federation/activity.py | 14 ++- .../federation/api_serializers.py | 16 ++-- api/funkwhale_api/federation/api_views.py | 4 + api/funkwhale_api/federation/models.py | 13 --- api/funkwhale_api/federation/routes.py | 61 +++++++------ api/funkwhale_api/federation/serializers.py | 4 +- api/funkwhale_api/federation/tasks.py | 42 ++++++--- api/funkwhale_api/music/models.py | 1 - api/tests/federation/test_api_views.py | 89 ------------------- api/tests/federation/test_routes.py | 17 ++-- api/tests/federation/test_tasks.py | 58 ++++++++++++ changes/changelog.d/762.newfeature | 2 + 12 files changed, 163 insertions(+), 158 deletions(-) create mode 100644 changes/changelog.d/762.newfeature diff --git a/api/funkwhale_api/federation/activity.py b/api/funkwhale_api/federation/activity.py index 959abb47d..fc8ce5508 100644 --- a/api/funkwhale_api/federation/activity.py +++ b/api/funkwhale_api/federation/activity.py @@ -234,6 +234,9 @@ class InboxRouter(Router): from . import api_serializers from . import models + logger.info( + f"[federation] Inbox dispatch payload : {payload} with context : {context}" + ) handlers = self.get_matching_handlers(payload) for handler in handlers: if call_handlers: @@ -316,6 +319,8 @@ class OutboxRouter(Router): from . import models from . import tasks + logger.info(f"[federation] Outbox dispatch context : {context}") + allow_list_enabled = preferences.get("moderation__allow_list_enabled") allowed_domains = None if allow_list_enabled: @@ -349,7 +354,11 @@ class OutboxRouter(Router): for activity_data in activities_data: # If its a domain update we change the object type of the library to "Domain" to allow specific routing for recipient in activity_data["payload"]["to"]: - if recipient is dict and "domain_followers" in recipient["type"]: + if ( + type(recipient) is dict + and "domain_followers" in recipient["type"] + ): + activity_data["payload"]["object"]["type"] = "Domain" activity_data["payload"]["actor"] = activity_data["actor"].fid to = activity_data["payload"].pop("to", []) @@ -480,11 +489,10 @@ def prepare_deliveries_and_inbox_items(recipient_list, type, allowed_domains=Non .domainfollow_received_follows.filter(approved=True) .select_related("actor__user") ) - logger.info("received_follows" + str(received_follows)) for follow in received_follows: actor = follow.actor remote_inbox_urls.add(actor.shared_inbox_url or actor.inbox_url) - urls.append(actor.shared_inbox_url) + urls.append(actor.inbox_url) elif isinstance(r, dict) and r["type"] == "actor_inbox": actor = r["actor"] diff --git a/api/funkwhale_api/federation/api_serializers.py b/api/funkwhale_api/federation/api_serializers.py index 24fe52222..4a914fad2 100644 --- a/api/funkwhale_api/federation/api_serializers.py +++ b/api/funkwhale_api/federation/api_serializers.py @@ -173,15 +173,20 @@ class DomainFollowSerializer(serializers.ModelSerializer): domain = models.Domain.objects.get(name=v.name) now = datetime.datetime.now(timezone.utc) if ( - domain.nodeinfo_fetch_date - now - ).total_seconds() > 86400 or "error" in domain.nodeinfo: - tasks.refresh_actor_data() - tasks.update_domain_nodeinfo(domain_name=domain) + (domain.nodeinfo_fetch_date - now).total_seconds() > 86400 + or "error" in domain.nodeinfo + or not domain.service_actor + ): + # to do : timout error still happenig on initial fetch. Can't find when it works. + + tasks.update_domain_nodeinfo.delay(domain_name=domain.name) domain = models.Domain.objects.get(name=v.name) + if "error" in domain.nodeinfo: e = domain.nodeinfo["error"] logger.error(f"Could not get domain information because of : {e!r}") raise serializers.ValidationError(domain.nodeinfo["error"]) + return domain def get_actor(self, o): @@ -191,7 +196,6 @@ class DomainFollowSerializer(serializers.ModelSerializer): def serialize_generic_relation(activity, obj): data = {"type": obj._meta.label} - logger.info("data[type]" + str(data["type"])) if data["type"] == "federation.Actor": data["full_username"] = obj.full_username elif data["type"] == "federation.Domain": @@ -200,7 +204,7 @@ def serialize_generic_relation(activity, obj): data["uuid"] = obj.uuid if data["type"] == "music.Library": data["name"] = obj.name - # to do : test if this is works + if ( data["type"] == "federation.LibraryFollow" or data["type"] == "federation.DomainFollow" diff --git a/api/funkwhale_api/federation/api_views.py b/api/funkwhale_api/federation/api_views.py index d9e4d5bfe..7eed50e3f 100644 --- a/api/funkwhale_api/federation/api_views.py +++ b/api/funkwhale_api/federation/api_views.py @@ -269,6 +269,10 @@ class DomainViewSet( qs = qs.filter(allowed=True) return qs + def perform_create(self, serializer): + domain = serializer.save(actor=self.request.user.actor) + tasks.save_remote_service_actor.delay(domain) + class DomainFollowViewSet( mixins.CreateModelMixin, diff --git a/api/funkwhale_api/federation/models.py b/api/funkwhale_api/federation/models.py index 835cede49..8b584ff78 100644 --- a/api/funkwhale_api/federation/models.py +++ b/api/funkwhale_api/federation/models.py @@ -192,10 +192,6 @@ class Domain(models.Model): def is_local(self): return self.name == settings.FEDERATION_HOSTNAME - # to do : if false the follow will we saved as approved=none in the db. - # This should be handled by the fronted : when we approved the follow a accept activity is sent ? - # post to DomainFollow -> update_follow ? - def should_autoapprove_follow(self, actor): return settings.GLOBAL_FEDERATION @@ -559,15 +555,6 @@ class LibraryFollow(AbstractFollow): class Meta: unique_together = ["actor", "target"] - # to do : test. Is libraryfollow deletion done in fronted ? If yes this need to be implemented to unfollow domain frontend button - # def delete(self): - # from . import routes - - # routes.outbox.dispatch( - # {"type": "Undo", "object": {"type": "Follow"}}, context={"follow": instance} - # ) - # super(LibraryFollow, self).delete() - class Library(models.Model): creation_date = models.DateTimeField(default=timezone.now) diff --git a/api/funkwhale_api/federation/routes.py b/api/funkwhale_api/federation/routes.py index 7b300861b..4a87e9411 100644 --- a/api/funkwhale_api/federation/routes.py +++ b/api/funkwhale_api/federation/routes.py @@ -11,6 +11,7 @@ from . import activity from . import actors from . import models from . import serializers +from rest_framework import serializers as django_serializers logger = logging.getLogger(__name__) inbox = activity.InboxRouter() @@ -73,7 +74,7 @@ def outbox_accept(context): if follow._meta.label == "federation.LibraryFollow": actor = follow.target.actor elif follow._meta.label == "federation.DomainFollow": - actor = follow.target.service_actor + actor = actors.get_service_actor() else: actor = follow.target payload = serializers.AcceptFollowSerializer(follow, context={"actor": actor}).data @@ -160,14 +161,18 @@ def outbox_follow(context): if follow._meta.label == "federation.LibraryFollow": target = follow.target.actor elif follow._meta.label == "federation.DomainFollow": - # sould be target = follow.target.service_actor but this doesn't work if the db isn't updated when the queries is made -> + # to do : sould be target = follow.target.service_actor but this doesn't work if the db isn't updated when the queries is made -> # we should trigger the db update before the view is called ? Actually the db update is in the domainfollowserializer but this do not make the trick # is it a cache problem ? # temporary solution : domain = models.Domain.objects.get(name=context["follow"].target.name) + # target = domain.nodeinfo["payload"]["metadata"]["actorId"] target = domain.service_actor else: target = follow.target + if target is None: + raise Exception(f"Can't follow without target. Context is : {context}") + payload = serializers.FollowSerializer(follow, context={"actor": follow.actor}).data # This generate an activity and send it to activity.OutboxRouter yield { @@ -306,7 +311,6 @@ def inbox_update_library(payload, context): # if a library is created we inform the followers of the domain @outbox.register({"type": "Update", "object.type": "Domain"}) def outbox_update_domain_on_library_creation(context): - logger.info("OUTBOX DOMAIN UPDATE") domain = context["domain"] library = context["library"] @@ -327,35 +331,42 @@ def outbox_update_domain_on_library_creation(context): @inbox.register({"type": "Update", "object.type": "Domain"}) -def inbox_update_library_domain(payload, context): - logger.info( - "INBOX UPDATE DOMAIN context and payload " + str(context) + str(payload) - ) - domain = urlparse(payload["actor"]).netloc - library = payload["object"] - actors = [actor for actor in models.DomainFollow.objects.filter(target=domain)] - serializer = serializers.LibrarySerializer(data=library) - if serializer.is_valid(): - serializer.save() +def inbox_update_library_from_domain_follow(payload, context): + from . import api_serializers + + payload["object"]["type"] = "Library" + lib_serializer = serializers.LibrarySerializer(data=payload["object"]) + + if lib_serializer.is_valid(): + library = lib_serializer.save() else: - logger.debug( - "Discarding creation of library %s because of payload errors: %s", - library["id"], - serializer.errors, + raise django_serializers.ValidationError( + f"Discarding creation of the library {library_data} because of error: {lib_serializer.errors}" ) - from . import api_serializers - for actor in actors: - serializer = api_serializers.LibraryFollowSerializer( - target=library, follow_context="Domain", context={"actor": actor} + domain = urlparse(payload["object"]["id"]).netloc + domain_follows = [ + domain_follows + for domain_follows in models.DomainFollow.objects.filter(target=domain) + ] + # to do : optimise ? + for domain_follow in domain_follows: + library_follow = {} + actor = domain_follow.actor + library_follow["target"] = library.uuid + library_follow["follow_context"] = "Domain" + + f_serializer = api_serializers.LibraryFollowSerializer( + data=library_follow, context={"actor": actor} ) - if serializer.is_valid(): - serializer.save() + if f_serializer.is_valid(): + follow = f_serializer.save(actor=actor) + outbox.dispatch({"type": "Follow"}, context={"follow": follow}) else: - logger.debug( + logger.info( "Discarding follow of library %s because of payload errors: %s", domain, - serializer.errors, + f_serializer.errors, ) diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py index 4df1c3170..9e34f865a 100644 --- a/api/funkwhale_api/federation/serializers.py +++ b/api/funkwhale_api/federation/serializers.py @@ -775,9 +775,7 @@ class FollowActionSerializer(serializers.Serializer): else: actor = instance.target - logger.info( - "actor is caus metalabe is " + str(actor) + str(instance.target.name) - ) + return { "@context": jsonld.get_default_context(), "id": instance.get_federation_id() + "/{}".format(self.action_type), diff --git a/api/funkwhale_api/federation/tasks.py b/api/funkwhale_api/federation/tasks.py index d2a7e4318..d78998653 100644 --- a/api/funkwhale_api/federation/tasks.py +++ b/api/funkwhale_api/federation/tasks.py @@ -93,7 +93,9 @@ def dispatch_inbox(activity, call_handlers=True): Given an activity instance, triggers our internal delivery logic (follow creation, etc.) """ - logger.info("dispatch inbox with activity " + str(activity)) + logger.info( + "[federation] Dispatch inbox with activity payload : {activity.payload}" + ) routes.inbox.dispatch( activity.payload, context={ @@ -111,6 +113,8 @@ def dispatch_outbox(activity): """ Deliver a local activity to its recipients, both locally and remotely """ + logger.info(f"[federation] Dispatch outbox activity payload : {activity.payload}") + inbox_items = activity.inbox_items.filter(is_read=False).select_related() if inbox_items.exists(): @@ -636,7 +640,6 @@ def fetch_collection(url, max_pages, channel, is_page=False): "actor", ) def domain_bulk_follow_libraries(name, actor): - # to do : check utils.retrieve_ap_object for more thing to validate ? libs = [] if name.startswith("https://"): pass @@ -647,15 +650,15 @@ def domain_bulk_follow_libraries(name, actor): if not data: raise exceptions.BlockedActorOrDomain() - logger.info("response[result]" + str(response["results"])) serializer = api_serializers.LibrarySerializer(data=response["results"], many=True) # we do not raise exeption since we know some libs will not be valid (already followed) serializer.is_valid() libs = serializer.save() - if "next" in response.keys(): + + if "next" in response.keys() and response["next"] is not None: follow_libraries(libs, actor) url = response["next"] - domain_bulk_follow_libraries(url, actor) + domain_bulk_follow_libraries(url, actor_id=actor.id) else: return follow_libraries(response["results"], actor) @@ -705,12 +708,29 @@ def domain_bulk_unfollow_libraries(name, actor): # to do ? # usefull when a new lib is created and we need to dispatch the activity to all the actors following the domain -@celery.app.task(name="federation.domain_bulk_unfollow_libraries") -def actors_bulk_follow_library(actors, library): - # how to pass many actor and one lib ? - serializer = api_serializers.LibraryFollowSerializer( - data=libraries_follow, context={"actor": actor}, many=True - ) +# @celery.app.task(name="federation.actors_bulk_follow_library") +# def actors_bulk_follow_library(actors, library): +# # how to pass many actor and one lib ? +# serializer = api_serializers.LibraryFollowSerializer( +# data=libraries_follow, context={"actor": actor}, many=True +# ) + + +@celery.app.task(name="federation.save_remote_service_actor") +def save_remote_service_actor(domain): + try: + service_actor_id = common_utils.recursive_getattr( + domain.nodeinfo, "payload.metadata.actorId", permissive=True + ) + domain.service_actor = utils.retrieve_ap_object( + service_actor_id, + actor=None, + queryset=models.Actor, + serializer_class=serializers.ActorSerializer, + ) + domain.save(update_fields=["service_actor"]) + except Exception as e: + logger.info(f"Could not save service actor of {domain} because of error : {e}") # To do : to delete diff --git a/api/funkwhale_api/music/models.py b/api/funkwhale_api/music/models.py index 07aa0c4f6..82464e40a 100644 --- a/api/funkwhale_api/music/models.py +++ b/api/funkwhale_api/music/models.py @@ -1270,7 +1270,6 @@ SCAN_STATUS = [ ] -# to do domain scan ? class LibraryScan(models.Model): actor = models.ForeignKey( "federation.Actor", null=True, blank=True, on_delete=models.CASCADE diff --git a/api/tests/federation/test_api_views.py b/api/tests/federation/test_api_views.py index 14316bd84..b3a2bc45d 100644 --- a/api/tests/federation/test_api_views.py +++ b/api/tests/federation/test_api_views.py @@ -386,92 +386,3 @@ def test_can_undo_domain_follow(factories, logged_in_api_client, mocker): {"type": "Undo", "object": {"type": "Follow"}}, context={"follow": follow} ) unfollow_libs_delayed.assert_called_once_with(domain.name, actor_id=actor.pk) - - -# to do : deplacer dans task. -import factory -import uuid -from funkwhale_api.federation.factories import MusicLibraryFactory, ActorFactory -from funkwhale_api.common import utils as common_utils - - -def test_domain_bulk_follow_libraries(factories, mocker, logged_in_api_client): - domain = factories["federation.Domain"]() - actor = factories["federation.Actor"]() - i = 0 - lib_remote = {} - libraries_dic = [] - while i < 10: - # this only works if there is no subfactories within the factory class - # So we are manually converting subfactories obj in dic format see https://github.com/FactoryBoy/factory_boy/issues/68 - lib_dic = factory.build(dict, FACTORY_CLASS=MusicLibraryFactory) - actor_dic = vars(lib_dic["actor"]) - lib_dic["actor"] = actor_dic - libraries_dic.append(lib_dic) - i = i + 1 - lib_remote["results"] = libraries_dic - dispatch = mocker.patch("funkwhale_api.federation.routes.outbox.dispatch") - mocker.patch( - "funkwhale_api.federation.utils.get_remote_libraries", return_value=lib_remote - ) - calls = [] - follows = tasks.domain_bulk_follow_libraries(domain.name, actor_id=actor.pk) - import time - - time.sleep(1) - for follow in follows: - call = {} - call["follow_context"] = "Follow" - call["context"] = {"follow": follow} - calls.append(call) - dispatch.assert_has_calls(calls) - - -def test_domain_can_bulk_unfollow_libraries(factories, mocker, logged_in_api_client): - domain = factories["federation.Domain"]() - actor = logged_in_api_client.user.create_actor() - dispacth_unfollow = mocker.patch("funkwhale_api.federation.routes.outbox.dispatch") - i = 0 - calls = [] - follows = [] - while i < 10: - i = i + 1 - lib_uuid = uuid.uuid4() - followers_url = "https://{}/federation/music/libraries/{}/followers".format( - domain.name, lib_uuid - ) - lib = factories["music.Library"](followers_url=followers_url) - follow = factories["federation.LibraryFollow"]( - actor=actor, target=lib, follow_context="Domain" - ) - follows.append(follow) - tasks.domain_bulk_unfollow_libraries(domain.name, actor_id=actor.pk) - # to do : find a way to create calls - # print(str(dispacth_unfollow.call_args_list())) - # dispacth_unfollow.assert_has_calls( - # [ - # call( - # {"type": "Undo", "object": {"type": "Follow"}}, - # context={"follow": follow}, - # ) - # ] - # ) - for follow in follows: - assert models.LibraryFollow.objects.filter(uuid=follow.uuid).exists() == False - - -# to do : a deplacer dans test_routes -from funkwhale_api.federation import routes - - -def outbox_update_domain(factories): - domain = factories["federation.Domain"]() - library = factories["music.Library"]() - context = {"domain": domain.name, "library": library} - expected = { - "type": "Update", - "payload": with_recipients( - serializer.data, to=[{"type": "followers", "target": domain}] - ), - } - assert routes.outbox_update_domain(context) == expected diff --git a/api/tests/federation/test_routes.py b/api/tests/federation/test_routes.py index b6b0eac26..6cf5a63bb 100644 --- a/api/tests/federation/test_routes.py +++ b/api/tests/federation/test_routes.py @@ -12,6 +12,7 @@ from funkwhale_api.federation import ( models, ) from funkwhale_api.moderation import serializers as moderation_serializers +from funkwhale_api.federation import routes @pytest.mark.parametrize( @@ -1012,7 +1013,9 @@ def test_dispatch_outbox_update_domain_on_library_creation( ) -def test_dispatch_inbox_update_library_domain(factories, mocker, logged_in_api_client): +def test_inbox_update_library_from_domain_follow( + factories, mocker, logged_in_api_client +): domain = factories["federation.Domain"]() local_domain = factories["federation.Domain"](name=settings.FUNKWHALE_HOSTNAME) @@ -1020,14 +1023,14 @@ def test_dispatch_inbox_update_library_domain(factories, mocker, logged_in_api_c lib_uuid = uuid.uuid4() fid = "https://{}/federation/music/libraries/{}".format(domain.name, lib_uuid) library = factories["music.Library"](privacy_level="everyone", fid=fid) - payload = {"object": library, "to": {"type": "followers", "target": domain}} + serializer = serializers.ActivitySerializer( + {"type": "Update", "object": serializers.LibrarySerializer(library).data} + ) + serializer.data["to"] = {"type": "followers", "target": domain} + payload = serializer.data dispatch = mocker.patch("funkwhale_api.federation.routes.inbox.dispatch") - routes.inbox_update_library_domain( + routes.inbox_update_library_from_domain_follow( payload, context={"domain": local_domain, "library": library} ) assert models.LibraryFollow.objects.filter(target=library).count() == 10 - - -# to do ? -# def test_library_follow_on_remote_library_creation() diff --git a/api/tests/federation/test_tasks.py b/api/tests/federation/test_tasks.py index 82192cef0..cc0593af9 100644 --- a/api/tests/federation/test_tasks.py +++ b/api/tests/federation/test_tasks.py @@ -1,7 +1,9 @@ import datetime +import factory import os import pathlib import pytest +import uuid from django.utils import timezone @@ -10,6 +12,10 @@ from funkwhale_api.federation import models from funkwhale_api.federation import serializers from funkwhale_api.federation import tasks from funkwhale_api.federation import utils +from funkwhale_api.common import utils as common_utils + + +from funkwhale_api.federation.factories import MusicLibraryFactory, ActorFactory def test_clean_federation_music_cache_if_no_listen(preferences, factories): @@ -670,3 +676,55 @@ def test_fetch_collection(mocker, r_mock): assert result["seen"] == 7 assert result["total"] == 27094 assert result["next_page"] == payloads["page2"]["next"] + + +def test_domain_bulk_follow_libraries(factories, mocker, logged_in_api_client): + domain = factories["federation.Domain"]() + actor = factories["federation.Actor"]() + i = 0 + lib_remote = {} + libraries_dic = [] + while i < 10: + # this only works if there is no subfactories within the factory class + # So we are manually converting subfactories obj in dic format see https://github.com/FactoryBoy/factory_boy/issues/68 + lib_dic = factory.build(dict, FACTORY_CLASS=MusicLibraryFactory) + actor_dic = vars(lib_dic["actor"]) + lib_dic["actor"] = actor_dic + libraries_dic.append(lib_dic) + i = i + 1 + lib_remote["results"] = libraries_dic + dispatch = mocker.patch("funkwhale_api.federation.routes.outbox.dispatch") + mocker.patch( + "funkwhale_api.federation.utils.get_remote_libraries", return_value=lib_remote + ) + calls = [] + follows = tasks.domain_bulk_follow_libraries(domain.name, actor_id=actor.pk) + for follow in follows: + call = {} + call["follow_context"] = "Follow" + call["context"] = {"follow": follow} + calls.append(call) + dispatch.assert_has_calls(calls) + + +def test_domain_can_bulk_unfollow_libraries(factories, mocker, logged_in_api_client): + domain = factories["federation.Domain"]() + actor = logged_in_api_client.user.create_actor() + dispacth_unfollow = mocker.patch("funkwhale_api.federation.routes.outbox.dispatch") + i = 0 + calls = [] + follows = [] + while i < 10: + i = i + 1 + lib_uuid = uuid.uuid4() + followers_url = "https://{}/federation/music/libraries/{}/followers".format( + domain.name, lib_uuid + ) + lib = factories["music.Library"](followers_url=followers_url) + follow = factories["federation.LibraryFollow"]( + actor=actor, target=lib, follow_context="Domain" + ) + follows.append(follow) + tasks.domain_bulk_unfollow_libraries(domain.name, actor_id=actor.pk) + for follow in follows: + assert models.LibraryFollow.objects.filter(uuid=follow.uuid).exists() == False diff --git a/changes/changelog.d/762.newfeature b/changes/changelog.d/762.newfeature new file mode 100644 index 000000000..b61b4fdeb --- /dev/null +++ b/changes/changelog.d/762.newfeature @@ -0,0 +1,2 @@ +Alow actor to follow entire instances (#762) +New env variable GLOBAL_FEDERATION enable should_autoapprove_follow for the domain. -- GitLab From 397f459a076d0d1858accd9b7852c4aa5da276cb Mon Sep 17 00:00:00 2001 From: Petitminion Date: Wed, 1 Jun 2022 12:00:48 +0200 Subject: [PATCH 03/13] resolve env dev diff --- .env.dev | 21 +++++++++++++++++++++ api/Dockerfile | 13 ++++++------- 2 files changed, 27 insertions(+), 7 deletions(-) create mode 100644 .env.dev diff --git a/.env.dev b/.env.dev new file mode 100644 index 000000000..66638d77f --- /dev/null +++ b/.env.dev @@ -0,0 +1,21 @@ +DJANGO_ALLOWED_HOSTS=.funkwhale.test,localhost,nginx,0.0.0.0,127.0.0.1 +DJANGO_SETTINGS_MODULE=config.settings.local +DJANGO_SECRET_KEY=dev +C_FORCE_ROOT=true +FUNKWHALE_HOSTNAME=localhost +FUNKWHALE_PROTOCOL=http +PYTHONDONTWRITEBYTECODE=true +VUE_PORT=8080 +MUSIC_DIRECTORY_PATH=/music +BROWSABLE_API_ENABLED=True +FORWARDED_PROTO=http +LDAP_ENABLED=False +FUNKWHALE_SPA_HTML_ROOT=http://nginx/front/ +PYTHONTRACEMALLOC=0 + +# Uncomment this if you're using traefik/https +# FORCE_HTTPS_URLS=True + +# Customize to your needs +POSTGRES_VERSION=11 +DEBUG=true diff --git a/api/Dockerfile b/api/Dockerfile index 1be009f6d..eea83be6a 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -4,8 +4,7 @@ FROM alpine:3.14 as pre-build # dependencies. This is only required until alpine 3.16 is released, since this # allows us to install poetry as package. -RUN --mount=type=cache,target=/root/.cache/pip \ - apk add --no-cache python3 py3-cryptography py3-pip && \ +RUN apk add --no-cache python3 py3-cryptography py3-pip && \ pip3 install poetry COPY pyproject.toml poetry.lock / RUN poetry export --without-hashes > requirements.txt @@ -13,7 +12,7 @@ RUN poetry export --dev --without-hashes > dev-requirements.txt FROM alpine:3.14 as builder -RUN --mount=type=cache,target=/root/.cache/pip \ +RUN \ echo 'installing dependencies' && \ apk add --no-cache \ git \ @@ -45,8 +44,8 @@ ENV PATH="/venv/bin:/root/.local/bin:$PATH" VIRTUAL_ENV=/venv COPY --from=0 /requirements.txt /requirements.txt COPY --from=0 /dev-requirements.txt /dev-requirements.txt # hack around https://github.com/pypa/pip/issues/6158#issuecomment-456619072 -#ENV PIP_DOWNLOAD_CACHE=/noop/ -RUN --mount=type=cache,target=/root/.cache/pip \ +ENV PIP_DOWNLOAD_CACHE=/noop/ +RUN \ echo 'installing pip requirements' && \ pip3 install --upgrade pip && \ pip3 install setuptools wheel && \ @@ -55,9 +54,9 @@ RUN --mount=type=cache,target=/root/.cache/pip \ # version 3.3.2. Since poetry does not allow in-place dependency pinning, we need # to install the deps using pip. cat /requirements.txt | grep -Ev 'cryptography|autobahn' | pip3 install -r /dev/stdin cryptography==3.3.2 autobahn==21.2.1 && \ - #rm -rf $PIP_DOWNLOAD_CACHE + rm -rf $PIP_DOWNLOAD_CACHE - ARG install_dev_deps=0 +ARG install_dev_deps=0 RUN \ if [ "$install_dev_deps" = "1" ] ; then \ echo "Installing dev dependencies" && \ -- GitLab From 6431390af10493c35362b46f92ad91bf42fe2725 Mon Sep 17 00:00:00 2001 From: Petitminion Date: Wed, 1 Jun 2022 12:02:22 +0200 Subject: [PATCH 04/13] resolve env dev diff2 --- front/vite.config.js | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 front/vite.config.js diff --git a/front/vite.config.js b/front/vite.config.js new file mode 100644 index 000000000..d1a306ade --- /dev/null +++ b/front/vite.config.js @@ -0,0 +1,33 @@ +// vite.config.js + +import { defineConfig } from 'vite' +import { createVuePlugin as vue } from 'vite-plugin-vue2' + +import path from 'path' + +// https://vitejs.dev/config/ +export default defineConfig({ + plugins: [ + vue(), + { + name: 'fix-fomantic-ui-css', + transform (src, id) { + if (id.includes('fomantic-ui-css') && id.endsWith('.min.js')) { + return `import jQuery from 'jquery';${src}` + } + } + } + ], + server: { + port: process.env.VUE_PORT || '8080', + hmr: { + port: process.env.FUNKWHALE_PROTOCOL === 'https' ? 443 : 8000, + protocol: process.env.FUNKWHALE_PROTOCOL === 'https' ? 'wss' : 'ws' + } + }, + resolve: { + alias: { + '@': path.resolve(__dirname, './src') + } + } +}) -- GitLab From 25539923907fd99886fe0b68299104a6815230f9 Mon Sep 17 00:00:00 2001 From: Petitminion Date: Fri, 24 Jun 2022 21:52:16 +0200 Subject: [PATCH 05/13] Resolve http timouts on domain discovery. Closes #1714 --- api/funkwhale_api/federation/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/funkwhale_api/federation/tasks.py b/api/funkwhale_api/federation/tasks.py index d78998653..09badc63c 100644 --- a/api/funkwhale_api/federation/tasks.py +++ b/api/funkwhale_api/federation/tasks.py @@ -213,7 +213,7 @@ def update_domain_nodeinfo(domain): domain.service_actor = ( utils.retrieve_ap_object( service_actor_id, - actor=actors.get_service_actor(), + actor=None, queryset=models.Actor, serializer_class=serializers.ActorSerializer, ) -- GitLab From 7bd06b58bb407b36583799119110480eb7ea59c2 Mon Sep 17 00:00:00 2001 From: Petitminion Date: Mon, 4 Jul 2022 13:39:41 +0200 Subject: [PATCH 06/13] add task to follow all known domains --- .../federation/api_serializers.py | 1 - api/funkwhale_api/federation/api_views.py | 1 - api/funkwhale_api/federation/models.py | 3 + api/funkwhale_api/federation/tasks.py | 65 ++++++------------- 4 files changed, 23 insertions(+), 47 deletions(-) diff --git a/api/funkwhale_api/federation/api_serializers.py b/api/funkwhale_api/federation/api_serializers.py index 4a914fad2..cf8630071 100644 --- a/api/funkwhale_api/federation/api_serializers.py +++ b/api/funkwhale_api/federation/api_serializers.py @@ -177,7 +177,6 @@ class DomainFollowSerializer(serializers.ModelSerializer): or "error" in domain.nodeinfo or not domain.service_actor ): - # to do : timout error still happenig on initial fetch. Can't find when it works. tasks.update_domain_nodeinfo.delay(domain_name=domain.name) domain = models.Domain.objects.get(name=v.name) diff --git a/api/funkwhale_api/federation/api_views.py b/api/funkwhale_api/federation/api_views.py index 7eed50e3f..eb35f23be 100644 --- a/api/funkwhale_api/federation/api_views.py +++ b/api/funkwhale_api/federation/api_views.py @@ -271,7 +271,6 @@ class DomainViewSet( def perform_create(self, serializer): domain = serializer.save(actor=self.request.user.actor) - tasks.save_remote_service_actor.delay(domain) class DomainFollowViewSet( diff --git a/api/funkwhale_api/federation/models.py b/api/funkwhale_api/federation/models.py index 8b584ff78..eeeb54691 100644 --- a/api/funkwhale_api/federation/models.py +++ b/api/funkwhale_api/federation/models.py @@ -195,6 +195,9 @@ class Domain(models.Model): def should_autoapprove_follow(self, actor): return settings.GLOBAL_FEDERATION + def is_following(self, actor): + return self.domainfollow_received_follows__actor == actor + class Actor(models.Model): ap_type = "Actor" diff --git a/api/funkwhale_api/federation/tasks.py b/api/funkwhale_api/federation/tasks.py index 09badc63c..861c503de 100644 --- a/api/funkwhale_api/federation/tasks.py +++ b/api/funkwhale_api/federation/tasks.py @@ -716,50 +716,25 @@ def domain_bulk_unfollow_libraries(name, actor): # ) -@celery.app.task(name="federation.save_remote_service_actor") -def save_remote_service_actor(domain): - try: - service_actor_id = common_utils.recursive_getattr( - domain.nodeinfo, "payload.metadata.actorId", permissive=True - ) - domain.service_actor = utils.retrieve_ap_object( - service_actor_id, - actor=None, - queryset=models.Actor, - serializer_class=serializers.ActorSerializer, - ) - domain.save(update_fields=["service_actor"]) - except Exception as e: - logger.info(f"Could not save service actor of {domain} because of error : {e}") - - -# To do : to delete - -from requests import HTTPError -from funkwhale_api.federation import actors as actors_utils +@celery.app.task(name="federation.follow_all_domains") +def follow_all_domains(): + domains = ( + models.Domain.objects.all() + .prefetch_related() + .exclude(name=settings.FEDERATION_HOSTNAME) + ) + actor = actors.get_service_actor() + for domain in domains: + if not models.Domain.objects.filter(name=domain.name).exists(): + try: + data = {"target": domain.name} + serializer = api_serializers.DomainFollowSerializer( + data=data, context={"actor": actor} + ) -@celery.app.task(name="federation.refresh_actor_data") -def refresh_actor_data(): - actors = models.Actor.objects.all().prefetch_related() - for actor in actors: - try: - data = actors_utils.get_actor_data(actor.fid) - except HTTPError as e: - logger.info( - f"Actor couldn't be fetch because of the following exeption : {e!r}" - ) - if e.response.status_code == 410: - logger.info("Purging actor : {actor.fid!r}") - purge_actors([actor.id], [actor.domain]) - continue - continue - except Exception as e: - logger.info( - f"Actor couldn't be fetch because of the following exeption : {e!r}" - ) - continue - serializer = serializers.ActorSerializer(data=data) - serializer.is_valid(raise_exception=True) - serializer.save(last_fetch_date=timezone.now()) - return + serializer.is_valid(raise_exception=True) + follow = serializer.save(actor=actor) + routes.outbox.dispatch({"type": "Follow"}, context={"follow": follow}) + except Exception as e: + logger.error(f"Unhandeled error while creating domain follow : {e} ") -- GitLab From 9d6dc5abf33da2289d9fe05c338621aab650a5e1 Mon Sep 17 00:00:00 2001 From: Petitminion Date: Mon, 4 Jul 2022 19:50:44 +0200 Subject: [PATCH 07/13] a task to discover new domains --- api/config/settings/common.py | 10 +++++ .../federation/api_serializers.py | 4 +- api/funkwhale_api/federation/tasks.py | 43 ++++++++++++++++++- api/funkwhale_api/federation/utils.py | 4 ++ 4 files changed, 58 insertions(+), 3 deletions(-) diff --git a/api/config/settings/common.py b/api/config/settings/common.py index bbb60b2fd..0f63e5d9a 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -825,6 +825,16 @@ CELERY_BEAT_SCHEDULE = { ), "options": {"expires": 60 * 60}, }, + "federation.follow_all_domains": { + "task": "federation.follow_all_domains", + "schedule": crontab(minute="30", hour="*/12"), + "options": {"expires": 60 * 60 * 24}, + }, + "federation.discover_domains": { + "task": "federation.discover_domains", + "schedule": crontab(minute="0", hour="*/12"), + "options": {"expires": 60 * 60 * 24}, + }, } if env.bool("ADD_ALBUM_TAGS_FROM_TRACKS", default=True): diff --git a/api/funkwhale_api/federation/api_serializers.py b/api/funkwhale_api/federation/api_serializers.py index cf8630071..2a3997cf3 100644 --- a/api/funkwhale_api/federation/api_serializers.py +++ b/api/funkwhale_api/federation/api_serializers.py @@ -53,16 +53,16 @@ class LibraryScanSerializer(serializers.ModelSerializer): ] -class DomainSerializer(serializers.Serializer): +class DomainSerializer(serializers.ModelSerializer): name = serializers.CharField() follow = serializers.SerializerMethodField() class Meta: model = federation_models.Domain fields = [ - "fid", "name", "nodeinfo", + "follow", ] def get_follow(self, o): diff --git a/api/funkwhale_api/federation/tasks.py b/api/funkwhale_api/federation/tasks.py index 861c503de..fcdc2da0c 100644 --- a/api/funkwhale_api/federation/tasks.py +++ b/api/funkwhale_api/federation/tasks.py @@ -1,3 +1,4 @@ +from django.urls import reverse import datetime import json import logging @@ -644,7 +645,10 @@ def domain_bulk_follow_libraries(name, actor): if name.startswith("https://"): pass else: - url = f"https://{name}/api/v1/libraries?page_size=1000&scope:domain={name}" + url = utils.get_remote_url( + name, reverse("api:v1:libraries-list", "domain=name") + ) + # url = f"https://{name}/api/v1/libraries?page_size=1000&scope:domain={name}" response = utils.get_remote_libraries(url, actor) data, updated = mrf.inbox.apply(response) if not data: @@ -718,6 +722,10 @@ def domain_bulk_unfollow_libraries(name, actor): @celery.app.task(name="federation.follow_all_domains") def follow_all_domains(): + if settings.GLOBAL_FEDERATION == False: + logger.info("Global federation desabled in env variables. Skipping task.") + return + domains = ( models.Domain.objects.all() .prefetch_related() @@ -738,3 +746,36 @@ def follow_all_domains(): routes.outbox.dispatch({"type": "Follow"}, context={"follow": follow}) except Exception as e: logger.error(f"Unhandeled error while creating domain follow : {e} ") + + +@celery.app.task(name="federation.discover_domains") +def discover_domains(): + if settings.GLOBAL_FEDERATION == False: + logger.info("Global federation desabled in env variables. Skipping task.") + return + # to do : add filter(reachable=True) + local_domains = ( + models.Domain.objects.all() + .prefetch_related() + .exclude(name=settings.FEDERATION_HOSTNAME) + ) + actor = actors.get_service_actor() + + for local_domain in local_domains: + url = utils.get_remote_url( + local_domain.name, reverse("api:v1:federation:domains-list") + ) + logger.info(f"url is {url}") + domain_list = session.get_session().get(url) + domain_list = domain_list.json() + + for remote_domain in domain_list["results"]: + if models.Domain.objects.filter(name=remote_domain["name"]).exists(): + logger.info("Domain already in db. Skipping") + continue + else: + logger.info(f"Adding discovered domain {remote_domain} to db") + data = {"name": remote_domain["name"], "creation_date": timezone.now} + serializer = api_serializers.DomainSerializer(data=data) + serializer.is_valid(raise_exception=True) + serializer.save() diff --git a/api/funkwhale_api/federation/utils.py b/api/funkwhale_api/federation/utils.py index 020332bb8..f679fb0fa 100644 --- a/api/funkwhale_api/federation/utils.py +++ b/api/funkwhale_api/federation/utils.py @@ -305,3 +305,7 @@ def get_remote_libraries(url, actor): response.raise_for_status() response = response.json() return response + + +def get_remote_url(hostname, reverse, scope=None): + return f"https://{hostname}{reverse}?scope:{scope}" -- GitLab From f4b55af3ccaa08c44353c4d55292f731af71c9ff Mon Sep 17 00:00:00 2001 From: Petitminion Date: Tue, 12 Jul 2022 01:23:49 +0200 Subject: [PATCH 08/13] discover domain from network.funkwhale.audio --- api/config/settings/common.py | 7 ++++- api/funkwhale_api/federation/tasks.py | 38 ++++++++++++++++++--------- 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/api/config/settings/common.py b/api/config/settings/common.py index 0f63e5d9a..7c3ce57bd 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -832,7 +832,12 @@ CELERY_BEAT_SCHEDULE = { }, "federation.discover_domains": { "task": "federation.discover_domains", - "schedule": crontab(minute="0", hour="*/12"), + "schedule": crontab(minute="15", hour="*/12"), + "options": {"expires": 60 * 60 * 24}, + }, + "federation.discover_domains_from_funkwhale.audio": { + "task": "federation.discover_domains_from_funkwhale.audio", + "schedule": crontab(minute="0", hour="*/48"), "options": {"expires": 60 * 60 * 24}, }, } diff --git a/api/funkwhale_api/federation/tasks.py b/api/funkwhale_api/federation/tasks.py index fcdc2da0c..74f26297f 100644 --- a/api/funkwhale_api/federation/tasks.py +++ b/api/funkwhale_api/federation/tasks.py @@ -765,17 +765,29 @@ def discover_domains(): url = utils.get_remote_url( local_domain.name, reverse("api:v1:federation:domains-list") ) - logger.info(f"url is {url}") - domain_list = session.get_session().get(url) - domain_list = domain_list.json() + follow_domains_from_url(url) - for remote_domain in domain_list["results"]: - if models.Domain.objects.filter(name=remote_domain["name"]).exists(): - logger.info("Domain already in db. Skipping") - continue - else: - logger.info(f"Adding discovered domain {remote_domain} to db") - data = {"name": remote_domain["name"], "creation_date": timezone.now} - serializer = api_serializers.DomainSerializer(data=data) - serializer.is_valid(raise_exception=True) - serializer.save() + +@celery.app.task(name="federation.discover_domains_from_funkwhale.audio") +def discover_domains_from_funkwhale_audio(): + if settings.GLOBAL_FEDERATION == False: + logger.info("Global federation desabled in env variables. Skipping task.") + return + url = "https://network.funkwhale.audio/api/domains" + follow_domains_from_url(url) + + +def follow_domains_from_url(url): + domain_list = session.get_session().get(url) + domain_list = domain_list.json() + + for remote_domain in domain_list["results"]: + if models.Domain.objects.filter(name=remote_domain["name"]).exists(): + logger.info("Domain already in db. Skipping") + continue + else: + logger.info(f"Adding discovered domain {remote_domain} to db") + data = {"name": remote_domain["name"], "creation_date": timezone.now} + serializer = api_serializers.DomainSerializer(data=data) + serializer.is_valid(raise_exception=True) + serializer.save() -- GitLab From d4c91ee1d1c09c76288f3a10cc28e5d6e35ac99d Mon Sep 17 00:00:00 2001 From: Petitminion Date: Tue, 12 Jul 2022 01:50:13 +0200 Subject: [PATCH 09/13] resolve url from utils --- api/funkwhale_api/federation/tasks.py | 2 +- api/tests/federation/test_tasks.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/funkwhale_api/federation/tasks.py b/api/funkwhale_api/federation/tasks.py index 74f26297f..d66a53f50 100644 --- a/api/funkwhale_api/federation/tasks.py +++ b/api/funkwhale_api/federation/tasks.py @@ -646,7 +646,7 @@ def domain_bulk_follow_libraries(name, actor): pass else: url = utils.get_remote_url( - name, reverse("api:v1:libraries-list", "domain=name") + name, reverse("api:v1:libraries-list"), "domain=name" ) # url = f"https://{name}/api/v1/libraries?page_size=1000&scope:domain={name}" response = utils.get_remote_libraries(url, actor) diff --git a/api/tests/federation/test_tasks.py b/api/tests/federation/test_tasks.py index cc0593af9..2be234d14 100644 --- a/api/tests/federation/test_tasks.py +++ b/api/tests/federation/test_tasks.py @@ -216,7 +216,7 @@ def test_update_domain_nodeinfo(factories, mocker, now, service_actor): retrieve_ap_object.assert_called_once_with( "https://actor.id", - actor=service_actor, + actor=None, queryset=models.Actor, serializer_class=serializers.ActorSerializer, ) -- GitLab From 48b7fd295145cd785fd0d34a0938de7d6a9aa876 Mon Sep 17 00:00:00 2001 From: Petitminion Date: Thu, 14 Jul 2022 15:36:27 +0200 Subject: [PATCH 10/13] if library is updated from private to public, activity need to be send to domain follow --- api/funkwhale_api/federation/tasks.py | 1 - api/funkwhale_api/music/serializers.py | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/api/funkwhale_api/federation/tasks.py b/api/funkwhale_api/federation/tasks.py index d66a53f50..84e05f363 100644 --- a/api/funkwhale_api/federation/tasks.py +++ b/api/funkwhale_api/federation/tasks.py @@ -648,7 +648,6 @@ def domain_bulk_follow_libraries(name, actor): url = utils.get_remote_url( name, reverse("api:v1:libraries-list"), "domain=name" ) - # url = f"https://{name}/api/v1/libraries?page_size=1000&scope:domain={name}" response = utils.get_remote_libraries(url, actor) data, updated = mrf.inbox.apply(response) if not data: diff --git a/api/funkwhale_api/music/serializers.py b/api/funkwhale_api/music/serializers.py index df4e70133..12ed4f796 100644 --- a/api/funkwhale_api/music/serializers.py +++ b/api/funkwhale_api/music/serializers.py @@ -11,6 +11,7 @@ from funkwhale_api.common import serializers as common_serializers from funkwhale_api.common import utils as common_utils from funkwhale_api.federation import routes from funkwhale_api.federation import utils as federation_utils +from funkwhale_api.federation import models as federation_models from funkwhale_api.playlists import models as playlists_models from funkwhale_api.tags import models as tag_models from funkwhale_api.tags import serializers as tags_serializers @@ -370,6 +371,19 @@ class LibraryForOwnerSerializer(serializers.ModelSerializer): {"type": "Update", "object": {"type": "Library"}}, context={"library": obj} ) + # If the library is updated after the domain was followed. We need to inform our domain + # followers that the library is now available for everyone + if after["privacy_level"] == "everyone": + routes.outbox.dispatch( + {"type": "Update", "object": {"type": "Domain"}}, + context={ + "library": obj, + "domain": federation_models.Domain.objects.get( + name=settings.FUNKWHALE_HOSTNAME + ), + }, + ) + def get_actor(self, o): # Import at runtime to avoid a circular import issue from funkwhale_api.federation import serializers as federation_serializers -- GitLab From 8da1dc823cd987efb0a4cdc49bd4e8ea4f11f97c Mon Sep 17 00:00:00 2001 From: Petitminion Date: Thu, 21 Jul 2022 15:49:19 +0200 Subject: [PATCH 11/13] Enhance discovery by filtering domain that are not funkwhale instances --- api/config/settings/common.py | 4 +- .../federation/api_serializers.py | 12 ++++ .../0028_domain_is_funkwhale_instance.py | 18 +++++ api/funkwhale_api/federation/models.py | 2 +- api/funkwhale_api/federation/tasks.py | 67 ++++++++++++++----- 5 files changed, 83 insertions(+), 20 deletions(-) create mode 100644 api/funkwhale_api/federation/migrations/0028_domain_is_funkwhale_instance.py diff --git a/api/config/settings/common.py b/api/config/settings/common.py index 7c3ce57bd..8e9e32593 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -832,12 +832,12 @@ CELERY_BEAT_SCHEDULE = { }, "federation.discover_domains": { "task": "federation.discover_domains", - "schedule": crontab(minute="15", hour="*/12"), + "schedule": crontab(minute="0", hour="2", day_of_week="1"), "options": {"expires": 60 * 60 * 24}, }, "federation.discover_domains_from_funkwhale.audio": { "task": "federation.discover_domains_from_funkwhale.audio", - "schedule": crontab(minute="0", hour="*/48"), + "schedule": crontab(minute="0", hour="1", day_of_week="1"), "options": {"expires": 60 * 60 * 24}, }, } diff --git a/api/funkwhale_api/federation/api_serializers.py b/api/funkwhale_api/federation/api_serializers.py index 2a3997cf3..f5fa429bd 100644 --- a/api/funkwhale_api/federation/api_serializers.py +++ b/api/funkwhale_api/federation/api_serializers.py @@ -56,6 +56,7 @@ class LibraryScanSerializer(serializers.ModelSerializer): class DomainSerializer(serializers.ModelSerializer): name = serializers.CharField() follow = serializers.SerializerMethodField() + is_funkwhale_instance = serializers.SerializerMethodField() class Meta: model = federation_models.Domain @@ -63,6 +64,7 @@ class DomainSerializer(serializers.ModelSerializer): "name", "nodeinfo", "follow", + "is_funkwhale_instance", ] def get_follow(self, o): @@ -71,6 +73,16 @@ class DomainSerializer(serializers.ModelSerializer): except (AttributeError, IndexError): return None + def get_is_funkwhale_instance(self, o): + try: + if o.nodeinfo["payload"]["software"]["name"] == "funkwhale": + return True + else: + return False + except Exception as e: + logger.info(f"Exception {e}") + return False + class LibraryFollowListSerializer(serializers.ListSerializer): def create(self, validated_data): diff --git a/api/funkwhale_api/federation/migrations/0028_domain_is_funkwhale_instance.py b/api/funkwhale_api/federation/migrations/0028_domain_is_funkwhale_instance.py new file mode 100644 index 000000000..a86698c95 --- /dev/null +++ b/api/funkwhale_api/federation/migrations/0028_domain_is_funkwhale_instance.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2022-07-21 10:46 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("federation", "0027_auto_20220527_2006"), + ] + + operations = [ + migrations.AddField( + model_name="domain", + name="is_funkwhale_instance", + field=models.BooleanField(default=False, null=True), + ), + ] diff --git a/api/funkwhale_api/federation/models.py b/api/funkwhale_api/federation/models.py index eeeb54691..ac9c1e6e8 100644 --- a/api/funkwhale_api/federation/models.py +++ b/api/funkwhale_api/federation/models.py @@ -127,7 +127,7 @@ class Domain(models.Model): ) # are interactions with this domain allowed (only applies when allow-listing is on) allowed = models.BooleanField(default=None, null=True) - + is_funkwhale_instance = models.BooleanField(default=False, null=True) objects = DomainQuerySet.as_manager() def __str__(self): diff --git a/api/funkwhale_api/federation/tasks.py b/api/funkwhale_api/federation/tasks.py index 84e05f363..7ade14709 100644 --- a/api/funkwhale_api/federation/tasks.py +++ b/api/funkwhale_api/federation/tasks.py @@ -1,4 +1,3 @@ -from django.urls import reverse import datetime import json import logging @@ -11,6 +10,7 @@ from django.db import transaction from django.db.models import Q, F from django.db.models.deletion import Collector from django.utils import timezone +from django.urls import reverse from dynamic_preferences.registries import global_preferences_registry from requests.exceptions import RequestException @@ -231,7 +231,23 @@ def update_domain_nodeinfo(domain): ) domain.nodeinfo_fetch_date = now domain.nodeinfo = nodeinfo - domain.save(update_fields=["nodeinfo", "nodeinfo_fetch_date", "service_actor"]) + if ( + "payload" in domain.nodeinfo + and "software" in domain.nodeinfo["payload"] + and domain.nodeinfo["payload"]["software"]["name"] == "funkwhale" + ): + domain.is_funkwhale_instance = True + else: + domain.is_funkwhale_instance = False + + domain.save( + update_fields=[ + "nodeinfo", + "nodeinfo_fetch_date", + "service_actor", + "is_funkwhale_instance", + ] + ) @celery.app.task(name="federation.refresh_nodeinfo_known_nodes") @@ -727,8 +743,8 @@ def follow_all_domains(): domains = ( models.Domain.objects.all() - .prefetch_related() .exclude(name=settings.FEDERATION_HOSTNAME) + .exclude(is_funkwhale_instance=False) ) actor = actors.get_service_actor() @@ -750,43 +766,60 @@ def follow_all_domains(): @celery.app.task(name="federation.discover_domains") def discover_domains(): if settings.GLOBAL_FEDERATION == False: - logger.info("Global federation desabled in env variables. Skipping task.") + logger.info("Global federation disabled in env variables. Skipping task.") return - # to do : add filter(reachable=True) + # to do : add exclude(reachable=False) + local_fw_domains = ( + models.Domain.objects.all() + .exclude(is_funkwhale_instance=False) + .exclude(name=settings.FEDERATION_HOSTNAME) + .values_list("name", flat=True) + ) + local_domains = ( models.Domain.objects.all() - .prefetch_related() .exclude(name=settings.FEDERATION_HOSTNAME) + .values_list("name", flat=True) ) - actor = actors.get_service_actor() - for local_domain in local_domains: + for local_fw_domain in local_fw_domains: url = utils.get_remote_url( - local_domain.name, reverse("api:v1:federation:domains-list") + local_fw_domain, reverse("api:v1:federation:domains-list") ) - follow_domains_from_url(url) + discover_domains_from_url(url, local_domains) @celery.app.task(name="federation.discover_domains_from_funkwhale.audio") def discover_domains_from_funkwhale_audio(): if settings.GLOBAL_FEDERATION == False: - logger.info("Global federation desabled in env variables. Skipping task.") + logger.info("Global federation disabled in env variables. Skipping task.") return url = "https://network.funkwhale.audio/api/domains" - follow_domains_from_url(url) + discover_domains_from_url(url) -def follow_domains_from_url(url): - domain_list = session.get_session().get(url) - domain_list = domain_list.json() +def discover_domains_from_url(url, local_domains=[]): + try: + response = session.get_session().get(url) + except Exception: + return + + if response.status_code != 200: + logger.info(f"No domain found from {url}.") + return + + domain_list = response.json() for remote_domain in domain_list["results"]: - if models.Domain.objects.filter(name=remote_domain["name"]).exists(): + + if remote_domain["name"] in local_domains: logger.info("Domain already in db. Skipping") continue + else: logger.info(f"Adding discovered domain {remote_domain} to db") data = {"name": remote_domain["name"], "creation_date": timezone.now} serializer = api_serializers.DomainSerializer(data=data) serializer.is_valid(raise_exception=True) - serializer.save() + domain = serializer.save() + update_domain_nodeinfo.delay(domain_name=domain.name) -- GitLab From 244879c52fe6d94776267d5e55fc7647a106cd16 Mon Sep 17 00:00:00 2001 From: Petitminion Date: Tue, 26 Jul 2022 16:44:19 +0200 Subject: [PATCH 12/13] resolve bulk_follow errors and do not send activity for bulk library follow since remote do not need them --- api/funkwhale_api/federation/admin.py | 2 +- .../federation/api_serializers.py | 17 ++++--- api/funkwhale_api/federation/routes.py | 13 ++++-- api/funkwhale_api/federation/tasks.py | 46 ++++++------------- 4 files changed, 35 insertions(+), 43 deletions(-) diff --git a/api/funkwhale_api/federation/admin.py b/api/funkwhale_api/federation/admin.py index 0ef2e760e..b71fad971 100644 --- a/api/funkwhale_api/federation/admin.py +++ b/api/funkwhale_api/federation/admin.py @@ -27,7 +27,7 @@ redeliver_activities.short_description = "Redeliver" @admin.register(models.Domain) class DomainAdmin(admin.ModelAdmin): list_display = ["name", "allowed", "creation_date"] - list_filter = ["allowed"] + list_filter = ["allowed", "is_funkwhale_instance"] search_fields = ["name"] diff --git a/api/funkwhale_api/federation/api_serializers.py b/api/funkwhale_api/federation/api_serializers.py index f5fa429bd..fbb81e12b 100644 --- a/api/funkwhale_api/federation/api_serializers.py +++ b/api/funkwhale_api/federation/api_serializers.py @@ -92,8 +92,14 @@ class LibraryFollowListSerializer(serializers.ListSerializer): class LibraryListSerializer(serializers.ListSerializer): def create(self, validated_data): - libs = [models.Library(**lib) for lib in validated_data] - return models.Library.objects.bulk_create(libs, batch_size=1000) + for index, lib in enumerate(validated_data): + actor, status = models.Actor.objects.update_or_create(**lib["actor"]) + lib["actor"] = actor + validated_data[index] = lib + logger.info(f"EEEEEH actor is {actor} and libs are {lib}") + + libs = [music_models.Library(**lib) for lib in validated_data] + return music_models.Library.objects.bulk_create(libs, batch_size=1000) class LibrarySerializer(serializers.ModelSerializer): @@ -154,10 +160,8 @@ class LibraryFollowSerializer(serializers.ModelSerializer): list_serializer_class = LibraryFollowListSerializer def validate_target(self, v): - try: - actor = self.context["actor"] - except KeyError: - raise KeyError + actor = self.context["actor"] + if v.actor == actor: raise serializers.ValidationError("You cannot follow your own library") @@ -201,7 +205,6 @@ class DomainFollowSerializer(serializers.ModelSerializer): return domain def get_actor(self, o): - logger.info("o is :" + str(o)) return federation_serializers.APIActorSerializer(o.actor).data diff --git a/api/funkwhale_api/federation/routes.py b/api/funkwhale_api/federation/routes.py index 4a87e9411..1a0bbd9b8 100644 --- a/api/funkwhale_api/federation/routes.py +++ b/api/funkwhale_api/federation/routes.py @@ -2,6 +2,7 @@ import logging import uuid from urllib.parse import urlparse +from django.conf import settings from django.db.models import Q @@ -271,6 +272,7 @@ def outbox_delete_library(context): @outbox.register({"type": "Update", "object.type": "Library"}) def outbox_update_library(context): + domain = models.Domain.objects.get(name=settings.FEDERATION_HOSTNAME) library = context["library"] serializer = serializers.ActivitySerializer( {"type": "Update", "object": serializers.LibrarySerializer(library).data} @@ -280,7 +282,11 @@ def outbox_update_library(context): "type": "Update", "actor": library.actor, "payload": with_recipients( - serializer.data, to=[{"type": "followers", "target": library}] + serializer.data, + to=[ + {"type": "followers", "target": library}, + {"type": "domain_followers", "target": domain}, + ], ), } @@ -341,7 +347,7 @@ def inbox_update_library_from_domain_follow(payload, context): library = lib_serializer.save() else: raise django_serializers.ValidationError( - f"Discarding creation of the library {library_data} because of error: {lib_serializer.errors}" + f"Discarding creation of the library because of error: {lib_serializer.errors}" ) domain = urlparse(payload["object"]["id"]).netloc @@ -349,7 +355,7 @@ def inbox_update_library_from_domain_follow(payload, context): domain_follows for domain_follows in models.DomainFollow.objects.filter(target=domain) ] - # to do : optimise ? + for domain_follow in domain_follows: library_follow = {} actor = domain_follow.actor @@ -361,7 +367,6 @@ def inbox_update_library_from_domain_follow(payload, context): ) if f_serializer.is_valid(): follow = f_serializer.save(actor=actor) - outbox.dispatch({"type": "Follow"}, context={"follow": follow}) else: logger.info( "Discarding follow of library %s because of payload errors: %s", diff --git a/api/funkwhale_api/federation/tasks.py b/api/funkwhale_api/federation/tasks.py index 7ade14709..aa7b09cf8 100644 --- a/api/funkwhale_api/federation/tasks.py +++ b/api/funkwhale_api/federation/tasks.py @@ -657,12 +657,11 @@ def fetch_collection(url, max_pages, channel, is_page=False): "actor", ) def domain_bulk_follow_libraries(name, actor): - libs = [] if name.startswith("https://"): pass else: url = utils.get_remote_url( - name, reverse("api:v1:libraries-list"), "domain=name" + name, reverse("api:v1:libraries-list"), f"domain={name}" ) response = utils.get_remote_libraries(url, actor) data, updated = mrf.inbox.apply(response) @@ -673,24 +672,27 @@ def domain_bulk_follow_libraries(name, actor): # we do not raise exeption since we know some libs will not be valid (already followed) serializer.is_valid() libs = serializer.save() + fids = [] + for lib in libs: + fids.append(lib.fid) if "next" in response.keys() and response["next"] is not None: - follow_libraries(libs, actor) + follow_libraries_locally(fids, actor) url = response["next"] domain_bulk_follow_libraries(url, actor_id=actor.id) else: - return follow_libraries(response["results"], actor) + follow_libraries_locally(fids, actor) -def follow_libraries(libs, actor): - fids = [] - for lib in libs: - fids.append(lib["fid"]) - +def follow_libraries_locally(fids, actor): + """ + This will create a LibraryFollow entry only localy, + without sending activity to the remote library host. + """ libs = music_models.Library.objects.in_bulk(fids, field_name="fid") - library_follow = {} libraries_follow = [] for lib in libs.values(): + library_follow = {} library_follow["target"] = lib.uuid library_follow["follow_context"] = "Domain" libraries_follow.append(library_follow) @@ -698,12 +700,8 @@ def follow_libraries(libs, actor): serializer = api_serializers.LibraryFollowSerializer( data=libraries_follow, context={"actor": actor}, many=True ) - serializer.is_valid(raise_exception=True) - follows = serializer.save(actor=actor) - # To do : optimise ! bulk dispatch follows - for follow in follows: - routes.outbox.dispatch({"type": "Follow"}, context={"follow": follow}) - return follows + serializer.is_valid() + serializer.save(actor=actor) @celery.app.task(name="federation.domain_bulk_unfollow_libraries") @@ -717,22 +715,8 @@ def domain_bulk_unfollow_libraries(name, actor): target__followers_url__startswith=f"https://{name}", follow_context="Domain", ) - # to do : optimize - for follow in lib_follows: - routes.outbox.dispatch( - {"type": "Undo", "object": {"type": "Follow"}}, context={"follow": follow} - ) - return lib_follows.delete() - -# to do ? -# usefull when a new lib is created and we need to dispatch the activity to all the actors following the domain -# @celery.app.task(name="federation.actors_bulk_follow_library") -# def actors_bulk_follow_library(actors, library): -# # how to pass many actor and one lib ? -# serializer = api_serializers.LibraryFollowSerializer( -# data=libraries_follow, context={"actor": actor}, many=True -# ) + return lib_follows.delete() @celery.app.task(name="federation.follow_all_domains") -- GitLab From d11e855379fd8b039c705b581bff95c555bd4eb8 Mon Sep 17 00:00:00 2001 From: Petitminion Date: Sun, 31 Jul 2022 23:25:36 +0200 Subject: [PATCH 13/13] resolves tests errors --- api/funkwhale_api/federation/tasks.py | 13 ++++---- api/tests/federation/test_routes.py | 47 +++++++++++++++++++++++++-- api/tests/federation/test_tasks.py | 40 +++++++++++------------ api/tests/music/test_serializers.py | 18 +++++++--- 4 files changed, 84 insertions(+), 34 deletions(-) diff --git a/api/funkwhale_api/federation/tasks.py b/api/funkwhale_api/federation/tasks.py index aa7b09cf8..a6f537bc6 100644 --- a/api/funkwhale_api/federation/tasks.py +++ b/api/funkwhale_api/federation/tasks.py @@ -669,19 +669,20 @@ def domain_bulk_follow_libraries(name, actor): raise exceptions.BlockedActorOrDomain() serializer = api_serializers.LibrarySerializer(data=response["results"], many=True) - # we do not raise exeption since we know some libs will not be valid (already followed) + # we do not raise exeption since we know some libs will not be valid (already exist) serializer.is_valid() - libs = serializer.save() + serializer.save() + fids = [] - for lib in libs: - fids.append(lib.fid) + for lib in response["results"]: + fids.append(lib["fid"]) if "next" in response.keys() and response["next"] is not None: follow_libraries_locally(fids, actor) url = response["next"] domain_bulk_follow_libraries(url, actor_id=actor.id) else: - follow_libraries_locally(fids, actor) + return follow_libraries_locally(fids, actor) def follow_libraries_locally(fids, actor): @@ -701,7 +702,7 @@ def follow_libraries_locally(fids, actor): data=libraries_follow, context={"actor": actor}, many=True ) serializer.is_valid() - serializer.save(actor=actor) + return serializer.save(actor=actor) @celery.app.task(name="federation.domain_bulk_unfollow_libraries") diff --git a/api/tests/federation/test_routes.py b/api/tests/federation/test_routes.py index 6cf5a63bb..d087eae08 100644 --- a/api/tests/federation/test_routes.py +++ b/api/tests/federation/test_routes.py @@ -505,12 +505,16 @@ def test_outbox_delete_library(factories): def test_outbox_update_library(factories): library = factories["music.Library"]() + domain = factories["federation.Domain"](name=settings.FEDERATION_HOSTNAME) activity = list(routes.outbox_update_library({"library": library}))[0] expected = serializers.ActivitySerializer( {"type": "Update", "object": serializers.LibrarySerializer(library).data} ).data - expected["to"] = [{"type": "followers", "target": library}] + expected["to"] = [ + {"type": "followers", "target": library}, + {"type": "domain_followers", "target": domain}, + ] assert dict(activity["payload"]) == dict(expected) assert activity["actor"] == library.actor @@ -1026,11 +1030,48 @@ def test_inbox_update_library_from_domain_follow( serializer = serializers.ActivitySerializer( {"type": "Update", "object": serializers.LibrarySerializer(library).data} ) - serializer.data["to"] = {"type": "followers", "target": domain} + serializer.data["to"] = {"type": "domain_followers", "target": domain} payload = serializer.data - dispatch = mocker.patch("funkwhale_api.federation.routes.inbox.dispatch") + routes.inbox_update_library_from_domain_follow( payload, context={"domain": local_domain, "library": library} ) assert models.LibraryFollow.objects.filter(target=library).count() == 10 + + +# from funkwhale_api.music import models as music_models + + +# def test_inbox_update_library_from_domain_follow( +# factories, mocker, logged_in_api_client +# ): +# from funkwhale_api.music import models as music_models + +# domain = factories["federation.Domain"]() +# actor = factories["federation.Actor"](domain=domain) +# local_domain = factories["federation.Domain"](name=settings.FUNKWHALE_HOSTNAME) +# lib_uuid = uuid.uuid4() +# fid = "https://{}/federation/music/libraries/{}".format(domain.name, lib_uuid) +# followers_url = "https://{}/federation/music/libraries/{}/followers".format( +# domain.name, lib_uuid +# ) +# library = music_models.Library( +# privacy_level="everyone", +# fid=fid, +# name="blabla", +# actor=actor, +# followers_url=followers_url, +# ) + +# serializer = serializers.ActivitySerializer( +# {"type": "Update", "object": serializers.LibrarySerializer(library).data} +# ) +# serializer.data["to"] = {"type": "domain_followers", "target": domain} +# payload = serializer.data + +# routes.inbox_update_library_from_domain_follow( +# payload, context={"domain": local_domain, "library": library} +# ) + +# assert models.LibraryFollow.objects.filter(target=library).count() == 10 diff --git a/api/tests/federation/test_tasks.py b/api/tests/federation/test_tasks.py index 2be234d14..40ed13b70 100644 --- a/api/tests/federation/test_tasks.py +++ b/api/tests/federation/test_tasks.py @@ -1,4 +1,7 @@ import datetime +from funkwhale_api.music import models as music_models +from funkwhale_api.music import serializers as music_serializers + import factory import os import pathlib @@ -682,29 +685,26 @@ def test_domain_bulk_follow_libraries(factories, mocker, logged_in_api_client): domain = factories["federation.Domain"]() actor = factories["federation.Actor"]() i = 0 - lib_remote = {} + lib1 = factories["music.Library"](privacy_level="everyone") + lib2 = factories["music.Library"](privacy_level="everyone") + lib3 = factories["music.Library"](privacy_level="everyone") + + libs = [lib1, lib2, lib3] + libs_remote = {} libraries_dic = [] - while i < 10: - # this only works if there is no subfactories within the factory class - # So we are manually converting subfactories obj in dic format see https://github.com/FactoryBoy/factory_boy/issues/68 - lib_dic = factory.build(dict, FACTORY_CLASS=MusicLibraryFactory) - actor_dic = vars(lib_dic["actor"]) - lib_dic["actor"] = actor_dic - libraries_dic.append(lib_dic) - i = i + 1 - lib_remote["results"] = libraries_dic - dispatch = mocker.patch("funkwhale_api.federation.routes.outbox.dispatch") + + for lib in libs: + lib_json = music_serializers.LibraryForOwnerSerializer(lib) + libraries_dic.append(lib_json.data) + libs_remote["results"] = libraries_dic mocker.patch( - "funkwhale_api.federation.utils.get_remote_libraries", return_value=lib_remote + "funkwhale_api.federation.utils.get_remote_libraries", return_value=libs_remote ) - calls = [] - follows = tasks.domain_bulk_follow_libraries(domain.name, actor_id=actor.pk) - for follow in follows: - call = {} - call["follow_context"] = "Follow" - call["context"] = {"follow": follow} - calls.append(call) - dispatch.assert_has_calls(calls) + tasks.domain_bulk_follow_libraries(domain.name, actor_id=actor.pk) + assert music_models.Library.objects.filter(uuid=lib1.uuid).exists() + assert models.LibraryFollow.objects.filter(target__fid=lib1.fid).exists() + assert models.LibraryFollow.objects.filter(target__fid=lib2.fid).exists() + assert models.LibraryFollow.objects.filter(target__fid=lib3.fid).exists() def test_domain_can_bulk_unfollow_libraries(factories, mocker, logged_in_api_client): diff --git a/api/tests/music/test_serializers.py b/api/tests/music/test_serializers.py index beb30279d..ce0f948f0 100644 --- a/api/tests/music/test_serializers.py +++ b/api/tests/music/test_serializers.py @@ -1,6 +1,6 @@ import pytest import uuid - +from django.conf import settings from funkwhale_api.common import serializers as common_serializers from funkwhale_api.federation import serializers as federation_serializers from funkwhale_api.music import licenses @@ -406,16 +406,24 @@ def test_update_library_privacy_level_broadcasts_to_followers( ): dispatch = mocker.patch("funkwhale_api.federation.routes.outbox.dispatch") library = factories["music.Library"](**{field: before}) + domain = factories["federation.Domain"](name=settings.FUNKWHALE_HOSTNAME) serializer = serializers.LibraryForOwnerSerializer( library, data={field: after}, partial=True ) assert serializer.is_valid(raise_exception=True) serializer.save() - - dispatch.assert_called_once_with( - {"type": "Update", "object": {"type": "Library"}}, context={"library": library} - ) + if field == "privacy_level": + dispatch.assert_called_with( + {"type": "Update", "object": {"type": "Domain"}}, + context={"library": library, "domain": domain}, + ) + + else: + dispatch.assert_called_once_with( + {"type": "Update", "object": {"type": "Library"}}, + context={"library": library}, + ) def test_upload_with_channel(factories, uploaded_audio_file): -- GitLab