diff --git a/api/Dockerfile b/api/Dockerfile
index 2b1e831a2e11e6474de950ce5664796e7b6f1c22..eea83be6a4b8f0fc204262d909a459863686d562 100644
--- a/api/Dockerfile
+++ b/api/Dockerfile
@@ -5,7 +5,7 @@ FROM alpine:3.14 as pre-build
# allows us to install poetry as package.
RUN apk add --no-cache python3 py3-cryptography py3-pip && \
- pip3 install poetry
+ pip3 install poetry
COPY pyproject.toml poetry.lock /
RUN poetry export --without-hashes > requirements.txt
RUN poetry export --dev --without-hashes > dev-requirements.txt
@@ -13,28 +13,28 @@ RUN poetry export --dev --without-hashes > dev-requirements.txt
FROM alpine:3.14 as builder
RUN \
- echo 'installing dependencies' && \
- apk add --no-cache \
- git \
- musl-dev \
- gcc \
- postgresql-dev \
- python3-dev \
- py3-psycopg2 \
- py3-cryptography \
- libldap \
- libffi-dev \
- make \
- zlib-dev \
- jpeg-dev \
- openldap-dev \
- openssl-dev \
- cargo \
- libxml2-dev \
- libxslt-dev \
- curl \
- && \
- ln -s /usr/bin/python3 /usr/bin/python
+ echo 'installing dependencies' && \
+ apk add --no-cache \
+ git \
+ musl-dev \
+ gcc \
+ postgresql-dev \
+ python3-dev \
+ py3-psycopg2 \
+ py3-cryptography \
+ libldap \
+ libffi-dev \
+ make \
+ zlib-dev \
+ jpeg-dev \
+ openldap-dev \
+ openssl-dev \
+ cargo \
+ libxml2-dev \
+ libxslt-dev \
+ curl \
+ && \
+ ln -s /usr/bin/python3 /usr/bin/python
# create virtual env for next stage
RUN python -m venv --system-site-packages /venv
@@ -46,24 +46,24 @@ COPY --from=0 /dev-requirements.txt /dev-requirements.txt
# hack around https://github.com/pypa/pip/issues/6158#issuecomment-456619072
ENV PIP_DOWNLOAD_CACHE=/noop/
RUN \
- echo 'installing pip requirements' && \
- pip3 install --upgrade pip && \
- pip3 install setuptools wheel && \
- # Currently we are unable to relieably build cryptography on armv7. This
- # is why we need to use the package shipped by Alpine Linux, which is currently
- # version 3.3.2. Since poetry does not allow in-place dependency pinning, we need
- # to install the deps using pip.
- cat /requirements.txt | grep -Ev 'cryptography|autobahn' | pip3 install -r /dev/stdin cryptography==3.3.2 autobahn==21.2.1 && \
- rm -rf $PIP_DOWNLOAD_CACHE
+ echo 'installing pip requirements' && \
+ pip3 install --upgrade pip && \
+ pip3 install setuptools wheel && \
+ # Currently we are unable to relieably build cryptography on armv7. This
+ # is why we need to use the package shipped by Alpine Linux, which is currently
+ # version 3.3.2. Since poetry does not allow in-place dependency pinning, we need
+ # to install the deps using pip.
+ cat /requirements.txt | grep -Ev 'cryptography|autobahn' | pip3 install -r /dev/stdin cryptography==3.3.2 autobahn==21.2.1 && \
+ rm -rf $PIP_DOWNLOAD_CACHE
ARG install_dev_deps=0
RUN \
- if [ "$install_dev_deps" = "1" ] ; then \
- echo "Installing dev dependencies" && \
- cat /dev-requirements.txt | grep -Ev 'cryptography|autobahn' | pip3 install -r /dev/stdin cryptography==3.3.2 autobahn==21.2.1 \
- ; else \
- echo "Skipping dev deps installation" \
- ; fi
+ if [ "$install_dev_deps" = "1" ] ; then \
+ echo "Installing dev dependencies" && \
+ cat /dev-requirements.txt | grep -Ev 'cryptography|autobahn' | pip3 install -r /dev/stdin cryptography==3.3.2 autobahn==21.2.1 \
+ ; else \
+ echo "Skipping dev deps installation" \
+ ; fi
FROM alpine:3.14 as build-image
@@ -73,18 +73,18 @@ COPY --from=builder /venv /venv
ENV PATH="/venv/bin:$PATH"
RUN apk add --no-cache \
- libmagic \
- bash \
- gettext \
- python3 \
- jpeg-dev \
- ffmpeg \
- libpq \
- libxml2 \
- libxslt \
- py3-cryptography \
- && \
- ln -s /usr/bin/python3 /usr/bin/python
+ libmagic \
+ bash \
+ gettext \
+ python3 \
+ jpeg-dev \
+ ffmpeg \
+ libpq \
+ libxml2 \
+ libxslt \
+ py3-cryptography \
+ && \
+ ln -s /usr/bin/python3 /usr/bin/python
COPY . /app
WORKDIR /app
diff --git a/api/config/settings/common.py b/api/config/settings/common.py
index 179c01be4b466cd62f741569e6a0c9a8dc3ca134..8e9e3259325ed8bdf89f561269dd48ebafeb5c18 100644
--- a/api/config/settings/common.py
+++ b/api/config/settings/common.py
@@ -190,6 +190,7 @@ ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=[]) + [FUNKWHALE_HOSTNA
"""
List of allowed hostnames for which the Funkwhale server will answer.
"""
+GLOBAL_FEDERATION = env.bool("GLOBAL_FEDERATION", default=True)
# APP CONFIGURATION
# ------------------------------------------------------------------------------
@@ -824,6 +825,21 @@ CELERY_BEAT_SCHEDULE = {
),
"options": {"expires": 60 * 60},
},
+ "federation.follow_all_domains": {
+ "task": "federation.follow_all_domains",
+ "schedule": crontab(minute="30", hour="*/12"),
+ "options": {"expires": 60 * 60 * 24},
+ },
+ "federation.discover_domains": {
+ "task": "federation.discover_domains",
+ "schedule": crontab(minute="0", hour="2", day_of_week="1"),
+ "options": {"expires": 60 * 60 * 24},
+ },
+ "federation.discover_domains_from_funkwhale.audio": {
+ "task": "federation.discover_domains_from_funkwhale.audio",
+ "schedule": crontab(minute="0", hour="1", day_of_week="1"),
+ "options": {"expires": 60 * 60 * 24},
+ },
}
if env.bool("ADD_ALBUM_TAGS_FROM_TRACKS", default=True):
diff --git a/api/config/settings/local.py b/api/config/settings/local.py
index f0a64f0fa8192885b73f5a753926c40a796a3457..176c7912a4fec9c5ecd5aad9fa136d1aee924ca2 100644
--- a/api/config/settings/local.py
+++ b/api/config/settings/local.py
@@ -92,7 +92,7 @@ if DEBUG_TOOLBAR_ENABLED:
# ------------------------------------------------------------------------------
TEST_RUNNER = "django.test.runner.DiscoverRunner"
-# CELERY
+# CELERY Set to true to debug
CELERY_TASK_ALWAYS_EAGER = False
# END CELERY
diff --git a/api/funkwhale_api/federation/activity.py b/api/funkwhale_api/federation/activity.py
index 7725953e69706e746247c2f754afcf28e943c618..fc8ce550834ac8b264998e2c36517b53d535863d 100644
--- a/api/funkwhale_api/federation/activity.py
+++ b/api/funkwhale_api/federation/activity.py
@@ -12,6 +12,11 @@ from funkwhale_api.common import utils as funkwhale_utils
from . import contexts
+import logging
+
+logger = logging.getLogger(__name__)
+
+
recursive_getattr = funkwhale_utils.recursive_getattr
@@ -119,13 +124,16 @@ def should_reject(fid, actor_id=None, payload={}):
@transaction.atomic
def receive(activity, on_behalf_of, inbox_actor=None):
+ """
+ Receive an activity, find his recipients and save it to the database before dispatching it
+ """
from . import models
from . import serializers
from . import tasks
from .routes import inbox
from funkwhale_api.moderation import mrf
- logger.debug(
+ logger.info(
"[federation] Received activity from %s : %s", on_behalf_of.fid, activity
)
# we ensure the activity has the bare minimum structure before storing
@@ -226,6 +234,9 @@ class InboxRouter(Router):
from . import api_serializers
from . import models
+ logger.info(
+ f"[federation] Inbox dispatch payload : {payload} with context : {context}"
+ )
handlers = self.get_matching_handlers(payload)
for handler in handlers:
if call_handlers:
@@ -308,6 +319,8 @@ class OutboxRouter(Router):
from . import models
from . import tasks
+ logger.info(f"[federation] Outbox dispatch context : {context}")
+
allow_list_enabled = preferences.get("moderation__allow_list_enabled")
allowed_domains = None
if allow_list_enabled:
@@ -339,6 +352,14 @@ class OutboxRouter(Router):
deliveries_by_activity_uuid = {}
prepared_activities = []
for activity_data in activities_data:
+ # If its a domain update we change the object type of the library to "Domain" to allow specific routing
+ for recipient in activity_data["payload"]["to"]:
+ if (
+ type(recipient) is dict
+ and "domain_followers" in recipient["type"]
+ ):
+
+ activity_data["payload"]["object"]["type"] = "Domain"
activity_data["payload"]["actor"] = activity_data["actor"].fid
to = activity_data["payload"].pop("to", [])
cc = activity_data["payload"].pop("cc", [])
@@ -461,6 +482,18 @@ def prepare_deliveries_and_inbox_items(recipient_list, type, allowed_domains=Non
else:
remote_inbox_urls.add(actor.shared_inbox_url or actor.inbox_url)
urls.append(r["target"].followers_url)
+
+ elif isinstance(r, dict) and r["type"] == "domain_followers":
+ received_follows = (
+ r["target"]
+ .domainfollow_received_follows.filter(approved=True)
+ .select_related("actor__user")
+ )
+ for follow in received_follows:
+ actor = follow.actor
+ remote_inbox_urls.add(actor.shared_inbox_url or actor.inbox_url)
+ urls.append(actor.inbox_url)
+
elif isinstance(r, dict) and r["type"] == "actor_inbox":
actor = r["actor"]
urls.append(actor.fid)
diff --git a/api/funkwhale_api/federation/admin.py b/api/funkwhale_api/federation/admin.py
index 8e66708cfcd13054ff57f5477a3feef9788a446b..b71fad9716db7cccc483f9459bac4082ec1e7892 100644
--- a/api/funkwhale_api/federation/admin.py
+++ b/api/funkwhale_api/federation/admin.py
@@ -27,7 +27,7 @@ redeliver_activities.short_description = "Redeliver"
@admin.register(models.Domain)
class DomainAdmin(admin.ModelAdmin):
list_display = ["name", "allowed", "creation_date"]
- list_filter = ["allowed"]
+ list_filter = ["allowed", "is_funkwhale_instance"]
search_fields = ["name"]
@@ -78,6 +78,14 @@ class LibraryFollowAdmin(admin.ModelAdmin):
list_select_related = True
+@admin.register(models.DomainFollow)
+class DomainFollowAdmin(admin.ModelAdmin):
+ list_display = ["actor", "target", "approved", "creation_date"]
+ list_filter = ["approved"]
+ search_fields = ["actor__fid", "target__name"]
+ list_select_related = True
+
+
@admin.register(models.InboxItem)
class InboxItemAdmin(admin.ModelAdmin):
list_display = ["actor", "activity", "type", "is_read"]
diff --git a/api/funkwhale_api/federation/api_serializers.py b/api/funkwhale_api/federation/api_serializers.py
index 6ff1e1b2bc32978ba3c9441a623496a375b847b7..fbb81e12b02a9c30f87d212bff04e937888143dd 100644
--- a/api/funkwhale_api/federation/api_serializers.py
+++ b/api/funkwhale_api/federation/api_serializers.py
@@ -12,10 +12,15 @@ from funkwhale_api.common import fields as common_fields
from funkwhale_api.common import serializers as common_serializers
from funkwhale_api.music import models as music_models
from funkwhale_api.users import serializers as users_serializers
-
+from funkwhale_api.federation import models as federation_models
from . import filters
from . import models
from . import serializers as federation_serializers
+from . import tasks
+
+import logging
+
+logger = logging.getLogger(__name__)
class NestedLibraryFollowSerializer(serializers.ModelSerializer):
@@ -24,6 +29,17 @@ class NestedLibraryFollowSerializer(serializers.ModelSerializer):
fields = ["creation_date", "uuid", "fid", "approved", "modification_date"]
+class NestedDomainFollowSerializer(serializers.ModelSerializer):
+ class Meta:
+ model = models.DomainFollow
+ fields = [
+ "creation_date",
+ "fid",
+ "approved",
+ "modification_date",
+ ]
+
+
class LibraryScanSerializer(serializers.ModelSerializer):
class Meta:
model = music_models.LibraryScan
@@ -37,8 +53,53 @@ class LibraryScanSerializer(serializers.ModelSerializer):
]
-class DomainSerializer(serializers.Serializer):
+class DomainSerializer(serializers.ModelSerializer):
name = serializers.CharField()
+ follow = serializers.SerializerMethodField()
+ is_funkwhale_instance = serializers.SerializerMethodField()
+
+ class Meta:
+ model = federation_models.Domain
+ fields = [
+ "name",
+ "nodeinfo",
+ "follow",
+ "is_funkwhale_instance",
+ ]
+
+ def get_follow(self, o):
+ try:
+ return NestedDomainFollowSerializer(o._follows[0]).data
+ except (AttributeError, IndexError):
+ return None
+
+ def get_is_funkwhale_instance(self, o):
+ try:
+ if o.nodeinfo["payload"]["software"]["name"] == "funkwhale":
+ return True
+ else:
+ return False
+ except Exception as e:
+ logger.info(f"Exception {e}")
+ return False
+
+
+class LibraryFollowListSerializer(serializers.ListSerializer):
+ def create(self, validated_data):
+ libs = [models.LibraryFollow(**lib) for lib in validated_data]
+ return models.LibraryFollow.objects.bulk_create(libs, batch_size=1000)
+
+
+class LibraryListSerializer(serializers.ListSerializer):
+ def create(self, validated_data):
+ for index, lib in enumerate(validated_data):
+ actor, status = models.Actor.objects.update_or_create(**lib["actor"])
+ lib["actor"] = actor
+ validated_data[index] = lib
+ logger.info(f"EEEEEH actor is {actor} and libs are {lib}")
+
+ libs = [music_models.Library(**lib) for lib in validated_data]
+ return music_models.Library.objects.bulk_create(libs, batch_size=1000)
class LibrarySerializer(serializers.ModelSerializer):
@@ -61,6 +122,7 @@ class LibrarySerializer(serializers.ModelSerializer):
"follow",
"latest_scan",
]
+ list_serializer_class = LibraryListSerializer
def get_uploads_count(self, o):
return max(getattr(o, "_uploads_count", 0), o.uploads_count)
@@ -80,14 +142,26 @@ class LibrarySerializer(serializers.ModelSerializer):
class LibraryFollowSerializer(serializers.ModelSerializer):
target = common_serializers.RelatedField("uuid", LibrarySerializer(), required=True)
actor = serializers.SerializerMethodField()
+ follow_context = serializers.ChoiceField(
+ choices=["Domain", "Library"], required=False
+ )
class Meta:
model = models.LibraryFollow
- fields = ["creation_date", "actor", "uuid", "target", "approved"]
+ fields = [
+ "creation_date",
+ "actor",
+ "uuid",
+ "target",
+ "approved",
+ "follow_context",
+ ]
read_only_fields = ["uuid", "actor", "approved", "creation_date"]
+ list_serializer_class = LibraryFollowListSerializer
def validate_target(self, v):
actor = self.context["actor"]
+
if v.actor == actor:
raise serializers.ValidationError("You cannot follow your own library")
@@ -99,16 +173,56 @@ class LibraryFollowSerializer(serializers.ModelSerializer):
return federation_serializers.APIActorSerializer(o.actor).data
+class DomainFollowSerializer(serializers.ModelSerializer):
+ target = common_serializers.RelatedField("name", DomainSerializer(), required=True)
+ actor = serializers.SerializerMethodField()
+
+ class Meta:
+ model = models.DomainFollow
+ fields = ["uuid", "creation_date", "actor", "target", "approved"]
+ read_only_fields = ["uuid", "actor", "approved", "creation_date"]
+
+ def validate_target(self, v):
+ actor = self.context["actor"]
+ if v.domainfollow_received_follows.filter(actor=actor).exists():
+ raise serializers.ValidationError("You are already following this Domain")
+ domain = models.Domain.objects.get(name=v.name)
+ now = datetime.datetime.now(timezone.utc)
+ if (
+ (domain.nodeinfo_fetch_date - now).total_seconds() > 86400
+ or "error" in domain.nodeinfo
+ or not domain.service_actor
+ ):
+
+ tasks.update_domain_nodeinfo.delay(domain_name=domain.name)
+ domain = models.Domain.objects.get(name=v.name)
+
+ if "error" in domain.nodeinfo:
+ e = domain.nodeinfo["error"]
+ logger.error(f"Could not get domain information because of : {e!r}")
+ raise serializers.ValidationError(domain.nodeinfo["error"])
+
+ return domain
+
+ def get_actor(self, o):
+ return federation_serializers.APIActorSerializer(o.actor).data
+
+
def serialize_generic_relation(activity, obj):
data = {"type": obj._meta.label}
if data["type"] == "federation.Actor":
data["full_username"] = obj.full_username
+ elif data["type"] == "federation.Domain":
+ data["name"] = obj.name
else:
data["uuid"] = obj.uuid
-
if data["type"] == "music.Library":
data["name"] = obj.name
- if data["type"] == "federation.LibraryFollow":
+
+ if (
+ data["type"] == "federation.LibraryFollow"
+ or data["type"] == "federation.DomainFollow"
+ ):
data["approved"] = obj.approved
return data
diff --git a/api/funkwhale_api/federation/api_urls.py b/api/funkwhale_api/federation/api_urls.py
index df5bfb2f0396af3da21357aba967d51d941cad9a..1e9799521f5a11769d9e266ba7dcad457b728424 100644
--- a/api/funkwhale_api/federation/api_urls.py
+++ b/api/funkwhale_api/federation/api_urls.py
@@ -5,6 +5,7 @@ from . import api_views
router = routers.OptionalSlashRouter()
router.register(r"fetches", api_views.FetchViewSet, "fetches")
router.register(r"follows/library", api_views.LibraryFollowViewSet, "library-follows")
+router.register(r"follows/domain", api_views.DomainFollowViewSet, "domain-follows")
router.register(r"inbox", api_views.InboxItemViewSet, "inbox")
router.register(r"libraries", api_views.LibraryViewSet, "libraries")
router.register(r"domains", api_views.DomainViewSet, "domains")
diff --git a/api/funkwhale_api/federation/api_views.py b/api/funkwhale_api/federation/api_views.py
index f49c51c3575869e2417b672ea6adfb1d88c823d1..eb35f23be12875845599fbf2317ba5b68fe25f60 100644
--- a/api/funkwhale_api/federation/api_views.py
+++ b/api/funkwhale_api/federation/api_views.py
@@ -1,4 +1,6 @@
+from urllib import request
import requests.exceptions
+import requests
from django.conf import settings
from django.db import transaction
@@ -11,6 +13,7 @@ from rest_framework import response
from rest_framework import viewsets
from funkwhale_api.common import preferences
+from funkwhale_api.common import permissions as common_permissions
from funkwhale_api.common import utils as common_utils
from funkwhale_api.common.permissions import ConditionalAuthentication
from funkwhale_api.music import models as music_models
@@ -27,6 +30,10 @@ from . import serializers
from . import tasks
from . import utils
+import logging
+
+logger = logging.getLogger(__name__)
+
@transaction.atomic
def update_follow(follow, approved):
@@ -262,6 +269,96 @@ class DomainViewSet(
qs = qs.filter(allowed=True)
return qs
+ def perform_create(self, serializer):
+ domain = serializer.save(actor=self.request.user.actor)
+
+
+class DomainFollowViewSet(
+ mixins.CreateModelMixin,
+ mixins.ListModelMixin,
+ mixins.RetrieveModelMixin,
+ mixins.DestroyModelMixin,
+ viewsets.GenericViewSet,
+):
+ lookup_field = "uuid"
+ queryset = (
+ models.DomainFollow.objects.all()
+ .order_by("-creation_date")
+ .select_related("actor")
+ )
+ serializer_class = api_serializers.DomainFollowSerializer
+ permission_classes = [oauth_permissions.ScopePermission]
+ required_scope = "follows"
+ filterset_class = filters.DomainFollowFilter
+ ordering_fields = ("creation_date",)
+
+ def get_queryset(self):
+ qs = super().get_queryset()
+ return qs.filter(actor=self.request.user.actor).exclude(approved=False)
+
+ def perform_create(self, serializer):
+ follow = serializer.save(actor=self.request.user.actor)
+ routes.outbox.dispatch({"type": "Follow"}, context={"follow": follow})
+
+ @transaction.atomic
+ def perform_destroy(self, instance):
+ routes.outbox.dispatch(
+ {"type": "Undo", "object": {"type": "Follow"}},
+ context={"follow": instance},
+ )
+ tasks.domain_bulk_unfollow_libraries.delay(
+ instance.target.name, actor_id=self.request.user.actor.pk
+ )
+ instance.delete()
+
+ def get_serializer_context(self):
+ context = super().get_serializer_context()
+ context["actor"] = self.request.user.actor
+ return context
+
+ @decorators.action(methods=["post"], detail=True)
+ def accept(self, request, *args, **kwargs):
+ try:
+ follow = self.queryset.get(
+ target__actor=self.request.user.actor, uuid=kwargs["uuid"]
+ )
+ except models.DomainFollow.DoesNotExist:
+ return response.Response({}, status=404)
+ update_follow(follow, approved=True)
+ return response.Response(status=204)
+
+ @decorators.action(methods=["post"], detail=True)
+ def reject(self, request, *args, **kwargs):
+ try:
+ follow = self.queryset.get(
+ target__actor=self.request.user.actor, uuid=kwargs["uuid"]
+ )
+ except models.DomainFollow.DoesNotExist:
+ return response.Response({}, status=404)
+
+ update_follow(follow, approved=False)
+ return response.Response(status=204)
+
+ @decorators.action(methods=["get"], detail=False)
+ def all(self, request, *args, **kwargs):
+ """
+ Return all the subscriptions of the current user, with only limited data
+ to have a performant endpoint and avoid lots of queries just to display
+ subscription status in the UI
+ """
+ follows = list(
+ self.get_queryset().values_list("uuid", "target__name", "approved")
+ )
+
+ payload = {
+ "results": [
+ {"uuid": str(u[0]), "domain": str(u[1]), "approved": u[2]}
+ for u in follows
+ ],
+ "count": len(follows),
+ }
+ return response.Response(payload, status=200)
+
class ActorViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet):
queryset = models.Actor.objects.select_related(
diff --git a/api/funkwhale_api/federation/factories.py b/api/funkwhale_api/federation/factories.py
index 2d2a2abae64e9602a6ca918127d7097ea8da2660..cc7f576333bd7bb01835592a19a78761c9bc49f2 100644
--- a/api/funkwhale_api/federation/factories.py
+++ b/api/funkwhale_api/federation/factories.py
@@ -1,6 +1,9 @@
import uuid
import factory
+
+# see https://stackoverflow.com/questions/60914361/python-import-error-module-factory-has-no-attribute-fuzzy
+from factory import fuzzy
import requests
import requests_http_message_signatures
from django.conf import settings
@@ -238,11 +241,23 @@ class DeliveryFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
class LibraryFollowFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
target = factory.SubFactory(MusicLibraryFactory)
actor = factory.SubFactory(ActorFactory)
+ follow_context = fuzzy.FuzzyChoice(
+ models.LibraryFollow.FOLLOW_CONTEXT_CHOICES, getter=lambda c: c[0]
+ )
class Meta:
model = "federation.LibraryFollow"
+@registry.register
+class DomainollowFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
+ target = factory.SubFactory(DomainFactory)
+ actor = factory.SubFactory(ActorFactory)
+
+ class Meta:
+ model = "federation.DomainFollow"
+
+
class ArtistMetadataFactory(factory.Factory):
name = factory.Faker("name")
diff --git a/api/funkwhale_api/federation/filters.py b/api/funkwhale_api/federation/filters.py
index a57bc40713a03c2c28274c8eaa225cd6a7414b3d..31809604ffb64eda7c3e73047301f4eee81a4886 100644
--- a/api/funkwhale_api/federation/filters.py
+++ b/api/funkwhale_api/federation/filters.py
@@ -34,6 +34,12 @@ class LibraryFollowFilter(django_filters.FilterSet):
fields = ["approved"]
+class DomainFollowFilter(django_filters.FilterSet):
+ class Meta:
+ model = models.DomainFollow
+ fields = ["approved"]
+
+
class InboxItemFilter(django_filters.FilterSet):
is_read = django_filters.BooleanFilter(
"is_read", widget=django_filters.widgets.BooleanWidget()
diff --git a/api/funkwhale_api/federation/migrations/0027_auto_20220527_2006.py b/api/funkwhale_api/federation/migrations/0027_auto_20220527_2006.py
new file mode 100644
index 0000000000000000000000000000000000000000..32159d17e2c40f8bc89baec6369165eda80b2c5d
--- /dev/null
+++ b/api/funkwhale_api/federation/migrations/0027_auto_20220527_2006.py
@@ -0,0 +1,84 @@
+# Generated by Django 3.2.13 on 2022-05-27 20:06
+
+import django.core.serializers.json
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import funkwhale_api.federation.models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('federation', '0026_public_key_format'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='libraryfollow',
+ name='follow_context',
+ field=models.CharField(choices=[('Domain', 'Domain'), ('Library', 'Library')], default='Library', max_length=100),
+ ),
+ migrations.AlterField(
+ model_name='activity',
+ name='object_id',
+ field=models.TextField(blank=True, null=True),
+ ),
+ migrations.AlterField(
+ model_name='activity',
+ name='payload',
+ field=models.JSONField(blank=True, default=funkwhale_api.federation.models.empty_dict, encoder=django.core.serializers.json.DjangoJSONEncoder, max_length=50000),
+ ),
+ migrations.AlterField(
+ model_name='activity',
+ name='related_object_id',
+ field=models.TextField(blank=True, null=True),
+ ),
+ migrations.AlterField(
+ model_name='actor',
+ name='manually_approves_followers',
+ field=models.BooleanField(default=None, null=True),
+ ),
+ migrations.AlterField(
+ model_name='domain',
+ name='nodeinfo',
+ field=models.JSONField(blank=True, default=funkwhale_api.federation.models.empty_dict, max_length=50000),
+ ),
+ migrations.AlterField(
+ model_name='fetch',
+ name='detail',
+ field=models.JSONField(blank=True, default=funkwhale_api.federation.models.empty_dict, encoder=django.core.serializers.json.DjangoJSONEncoder, max_length=50000),
+ ),
+ migrations.AlterField(
+ model_name='follow',
+ name='approved',
+ field=models.BooleanField(default=None, null=True),
+ ),
+ migrations.AlterField(
+ model_name='libraryfollow',
+ name='approved',
+ field=models.BooleanField(default=None, null=True),
+ ),
+ migrations.AlterField(
+ model_name='librarytrack',
+ name='metadata',
+ field=models.JSONField(blank=True, default=funkwhale_api.federation.models.empty_dict, encoder=django.core.serializers.json.DjangoJSONEncoder, max_length=10000),
+ ),
+ migrations.CreateModel(
+ name='DomainFollow',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('fid', models.URLField(blank=True, max_length=500, null=True, unique=True)),
+ ('uuid', models.UUIDField(default=uuid.uuid4, unique=True)),
+ ('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
+ ('modification_date', models.DateTimeField(auto_now=True)),
+ ('approved', models.BooleanField(default=None, null=True)),
+ ('actor', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='domain_follows', to='federation.actor')),
+ ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='domainfollow_received_follows', to='federation.domain')),
+ ],
+ options={
+ 'unique_together': {('actor', 'target')},
+ },
+ ),
+ ]
diff --git a/api/funkwhale_api/federation/migrations/0028_domain_is_funkwhale_instance.py b/api/funkwhale_api/federation/migrations/0028_domain_is_funkwhale_instance.py
new file mode 100644
index 0000000000000000000000000000000000000000..a86698c9581162156ebcb687a9aee527f057e5f3
--- /dev/null
+++ b/api/funkwhale_api/federation/migrations/0028_domain_is_funkwhale_instance.py
@@ -0,0 +1,18 @@
+# Generated by Django 3.2.13 on 2022-07-21 10:46
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("federation", "0027_auto_20220527_2006"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="domain",
+ name="is_funkwhale_instance",
+ field=models.BooleanField(default=False, null=True),
+ ),
+ ]
diff --git a/api/funkwhale_api/federation/models.py b/api/funkwhale_api/federation/models.py
index d9ea2018074f22332d46d7e1bac102aa9c037c55..ac9c1e6e81fbeb84c67e568e762460174363547d 100644
--- a/api/funkwhale_api/federation/models.py
+++ b/api/funkwhale_api/federation/models.py
@@ -21,6 +21,10 @@ from funkwhale_api.music import utils as music_utils
from . import utils as federation_utils
+import logging
+
+logger = logging.getLogger(__name__)
+
TYPE_CHOICES = [
("Person", "Person"),
("Tombstone", "Tombstone"),
@@ -123,7 +127,7 @@ class Domain(models.Model):
)
# are interactions with this domain allowed (only applies when allow-listing is on)
allowed = models.BooleanField(default=None, null=True)
-
+ is_funkwhale_instance = models.BooleanField(default=False, null=True)
objects = DomainQuerySet.as_manager()
def __str__(self):
@@ -151,6 +155,12 @@ class Domain(models.Model):
emitted_library_follows=models.Count(
"actors__library_follows", distinct=True
),
+ received_domain_follows=models.Count(
+ "domainfollow_received_follows", distinct=True
+ ),
+ emitted_domain_follows=models.Count(
+ "actors__domain_follows", distinct=True
+ ),
actors=models.Count("actors", distinct=True),
)
data["artists"] = music_models.Artist.objects.filter(
@@ -171,10 +181,23 @@ class Domain(models.Model):
)
return data
+ def request_libraries_follows(self, actor):
+ from funkwhale_api.federation import tasks
+
+ common_utils.on_commit(
+ tasks.domain_bulk_follow_libraries.delay, self.name, actor_id=actor.pk
+ )
+
@property
def is_local(self):
return self.name == settings.FEDERATION_HOSTNAME
+ def should_autoapprove_follow(self, actor):
+ return settings.GLOBAL_FEDERATION
+
+ def is_following(self, actor):
+ return self.domainfollow_received_follows__actor == actor
+
class Actor(models.Model):
ap_type = "Actor"
@@ -451,7 +474,8 @@ class Activity(models.Model):
type = models.CharField(db_index=True, null=True, max_length=100)
# generic relations
- object_id = models.IntegerField(null=True, blank=True)
+ # This change is needed because the primary key is a text for domain and an integer for libraries
+ object_id = models.TextField(null=True, blank=True)
object_content_type = models.ForeignKey(
ContentType,
null=True,
@@ -469,7 +493,8 @@ class Activity(models.Model):
related_name="targeting_activities",
)
target = GenericForeignKey("target_content_type", "target_id")
- related_object_id = models.IntegerField(null=True, blank=True)
+ # This change is needed because object_id is a text for domain and an integer for libraries
+ related_object_id = models.TextField(null=True, blank=True)
related_object_content_type = models.ForeignKey(
ContentType,
null=True,
@@ -519,6 +544,17 @@ class LibraryFollow(AbstractFollow):
"music.Library", related_name="received_follows", on_delete=models.CASCADE
)
+ # This allow to know what libraries follows to delete. Exemple : if the delete comes from unfollow domain, only the libraries with contexte = [domain] will be deleted
+ # if the library is not followed whyle domain follow : context = domain
+ # if the library was followed before the domain follow : context = library (excluded from domain bulk lib follow)
+ # if the library is followed after the domain : can't do, to implement in the front ?
+ FOLLOW_CONTEXT_CHOICES = [("Domain", "Domain"), ("Library", "Library")]
+ follow_context = models.CharField(
+ max_length=100,
+ choices=FOLLOW_CONTEXT_CHOICES,
+ default="Library",
+ )
+
class Meta:
unique_together = ["actor", "target"]
@@ -636,3 +672,19 @@ def update_denormalization_follow_deleted(sender, instance, **kwargs):
music_models.TrackActor.objects.filter(
actor=instance.actor, upload__in=instance.target.uploads.all()
).delete()
+
+
+class DomainFollow(AbstractFollow):
+ actor = models.ForeignKey(
+ Actor, related_name="domain_follows", on_delete=models.CASCADE
+ )
+
+ target = models.ForeignKey(
+ "federation.Domain",
+ # domainfollow_received_follows
+ related_name="%(class)s_received_follows",
+ on_delete=models.CASCADE,
+ )
+
+ class Meta:
+ unique_together = ["actor", "target"]
diff --git a/api/funkwhale_api/federation/routes.py b/api/funkwhale_api/federation/routes.py
index 0a45f1d64a09743be0e818d548dc40f6cc86f5f6..1a0bbd9b83dc0afda958a7c35217cada727e1ec3 100644
--- a/api/funkwhale_api/federation/routes.py
+++ b/api/funkwhale_api/federation/routes.py
@@ -1,5 +1,8 @@
import logging
import uuid
+from urllib.parse import urlparse
+
+from django.conf import settings
from django.db.models import Q
@@ -9,6 +12,7 @@ from . import activity
from . import actors
from . import models
from . import serializers
+from rest_framework import serializers as django_serializers
logger = logging.getLogger(__name__)
inbox = activity.InboxRouter()
@@ -70,6 +74,8 @@ def outbox_accept(context):
follow = context["follow"]
if follow._meta.label == "federation.LibraryFollow":
actor = follow.target.actor
+ elif follow._meta.label == "federation.DomainFollow":
+ actor = actors.get_service_actor()
else:
actor = follow.target
payload = serializers.AcceptFollowSerializer(follow, context={"actor": actor}).data
@@ -133,6 +139,8 @@ def outbox_undo_follow(context):
actor = follow.actor
if follow._meta.label == "federation.LibraryFollow":
recipient = follow.target.actor
+ elif follow._meta.label == "federation.DomainFollow":
+ recipient = follow.target.service_actor
else:
recipient = follow.target
payload = serializers.UndoFollowSerializer(follow, context={"actor": actor}).data
@@ -148,11 +156,26 @@ def outbox_undo_follow(context):
@outbox.register({"type": "Follow"})
def outbox_follow(context):
follow = context["follow"]
+ # context["follow"] keys are dict_keys(['_state', 'id', 'fid', 'uuid', 'creation_date', 'modification_date', 'approved', 'actor_id', 'target_id'])
+ # context["follow"].target keys are ['_state', 'name', 'creation_date', 'nodeinfo_fetch_date', 'nodeinfo', 'service_actor_id', 'allowed'])
+
if follow._meta.label == "federation.LibraryFollow":
target = follow.target.actor
+ elif follow._meta.label == "federation.DomainFollow":
+ # to do : sould be target = follow.target.service_actor but this doesn't work if the db isn't updated when the queries is made ->
+ # we should trigger the db update before the view is called ? Actually the db update is in the domainfollowserializer but this do not make the trick
+ # is it a cache problem ?
+ # temporary solution :
+ domain = models.Domain.objects.get(name=context["follow"].target.name)
+ # target = domain.nodeinfo["payload"]["metadata"]["actorId"]
+ target = domain.service_actor
else:
target = follow.target
+ if target is None:
+ raise Exception(f"Can't follow without target. Context is : {context}")
+
payload = serializers.FollowSerializer(follow, context={"actor": follow.actor}).data
+ # This generate an activity and send it to activity.OutboxRouter
yield {
"type": "Follow",
"actor": follow.actor,
@@ -249,6 +272,7 @@ def outbox_delete_library(context):
@outbox.register({"type": "Update", "object.type": "Library"})
def outbox_update_library(context):
+ domain = models.Domain.objects.get(name=settings.FEDERATION_HOSTNAME)
library = context["library"]
serializer = serializers.ActivitySerializer(
{"type": "Update", "object": serializers.LibrarySerializer(library).data}
@@ -258,7 +282,11 @@ def outbox_update_library(context):
"type": "Update",
"actor": library.actor,
"payload": with_recipients(
- serializer.data, to=[{"type": "followers", "target": library}]
+ serializer.data,
+ to=[
+ {"type": "followers", "target": library},
+ {"type": "domain_followers", "target": domain},
+ ],
),
}
@@ -286,6 +314,67 @@ def inbox_update_library(payload, context):
)
+# if a library is created we inform the followers of the domain
+@outbox.register({"type": "Update", "object.type": "Domain"})
+def outbox_update_domain_on_library_creation(context):
+ domain = context["domain"]
+ library = context["library"]
+
+ serializer = serializers.ActivitySerializer(
+ {"type": "Update", "object": serializers.LibrarySerializer(library).data}
+ )
+
+ yield {
+ "type": "Update",
+ "actor": library.actor,
+ "payload": with_recipients(
+ serializer.data,
+ to=[
+ {"type": "domain_followers", "target": domain},
+ ],
+ ),
+ }
+
+
+@inbox.register({"type": "Update", "object.type": "Domain"})
+def inbox_update_library_from_domain_follow(payload, context):
+ from . import api_serializers
+
+ payload["object"]["type"] = "Library"
+ lib_serializer = serializers.LibrarySerializer(data=payload["object"])
+
+ if lib_serializer.is_valid():
+ library = lib_serializer.save()
+ else:
+ raise django_serializers.ValidationError(
+ f"Discarding creation of the library because of error: {lib_serializer.errors}"
+ )
+
+ domain = urlparse(payload["object"]["id"]).netloc
+ domain_follows = [
+ domain_follows
+ for domain_follows in models.DomainFollow.objects.filter(target=domain)
+ ]
+
+ for domain_follow in domain_follows:
+ library_follow = {}
+ actor = domain_follow.actor
+ library_follow["target"] = library.uuid
+ library_follow["follow_context"] = "Domain"
+
+ f_serializer = api_serializers.LibraryFollowSerializer(
+ data=library_follow, context={"actor": actor}
+ )
+ if f_serializer.is_valid():
+ follow = f_serializer.save(actor=actor)
+ else:
+ logger.info(
+ "Discarding follow of library %s because of payload errors: %s",
+ domain,
+ f_serializer.errors,
+ )
+
+
@inbox.register({"type": "Delete", "object.type": "Audio"})
def inbox_delete_audio(payload, context):
actor = context["actor"]
@@ -507,6 +596,7 @@ def outbox_delete_actor(context):
"Organization",
"Service",
"Group",
+ "Domain",
],
}
)
diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py
index d3263507baaed8dfde3197cb567be95783886413..9e34f865aac9c8fe91d641f02034abfd93e9e11a 100644
--- a/api/funkwhale_api/federation/serializers.py
+++ b/api/funkwhale_api/federation/serializers.py
@@ -599,9 +599,11 @@ class BaseActivitySerializer(serializers.Serializer):
)
+# TODO
class FollowSerializer(serializers.Serializer):
id = serializers.URLField(max_length=500)
- object = serializers.URLField(max_length=500)
+ # if obj is a domain its not a valid url
+ object = serializers.CharField(max_length=500)
actor = serializers.URLField(max_length=500)
type = serializers.ChoiceField(choices=["Follow"])
@@ -630,6 +632,14 @@ class FollowSerializer(serializers.Serializer):
except music_models.Library.DoesNotExist:
pass
+ try:
+ qs = models.Domain.objects.filter(name=v)
+ # if recipient:
+ # qs = qs.filter(actor=recipient)
+ return qs.get()
+ except models.Domain.DoesNotExist:
+ pass
+
raise serializers.ValidationError("Target not found")
def validate_actor(self, v):
@@ -642,12 +652,16 @@ class FollowSerializer(serializers.Serializer):
raise serializers.ValidationError("Actor not found")
def save(self, **kwargs):
+ logger.info("saving")
target = self.validated_data["object"]
if target._meta.label == "music.Library":
follow_class = models.LibraryFollow
+ elif target._meta.label == "federation.Domain":
+ follow_class = models.DomainFollow
else:
follow_class = models.Follow
+ logger.info("followclass is " + str(follow_class) + str(target._meta.label))
defaults = kwargs
defaults["fid"] = self.validated_data["id"]
approved = kwargs.pop("approved", None)
@@ -675,13 +689,22 @@ class FollowSerializer(serializers.Serializer):
return follow
def to_representation(self, instance):
- return {
- "@context": jsonld.get_default_context(),
- "actor": instance.actor.fid,
- "id": instance.get_federation_id(),
- "object": instance.target.fid,
- "type": "Follow",
- }
+ try:
+ return {
+ "@context": jsonld.get_default_context(),
+ "actor": instance.actor.fid,
+ "id": instance.get_federation_id(),
+ "object": instance.target.fid,
+ "type": "Follow",
+ }
+ except:
+ return {
+ "@context": jsonld.get_default_context(),
+ "actor": instance.actor.fid,
+ "id": instance.get_federation_id(),
+ "object": instance.target.name,
+ "type": "Follow",
+ }
class APIFollowSerializer(serializers.ModelSerializer):
@@ -722,6 +745,9 @@ class FollowActionSerializer(serializers.Serializer):
if target._meta.label == "music.Library":
expected = target.actor
follow_class = models.LibraryFollow
+ elif target._meta.label == "federation.Domain":
+ expected = target.service_actor
+ follow_class = models.DomainFollow
else:
expected = target
follow_class = models.Follow
@@ -744,7 +770,10 @@ class FollowActionSerializer(serializers.Serializer):
def to_representation(self, instance):
if instance.target._meta.label == "music.Library":
actor = instance.target.actor
+ elif instance.target._meta.label == "federation.Domain":
+ actor = actors.get_service_actor()
else:
+
actor = instance.target
return {
@@ -767,6 +796,8 @@ class AcceptFollowSerializer(FollowActionSerializer):
follow.save()
if follow.target._meta.label == "music.Library":
follow.target.schedule_scan(actor=follow.actor)
+ if follow.target._meta.label == "federation.Domain":
+ follow.target.request_libraries_follows(actor=follow.actor)
return follow
diff --git a/api/funkwhale_api/federation/tasks.py b/api/funkwhale_api/federation/tasks.py
index c040082aa44812603e36fb61495eb165019c729c..a6f537bc68f710b45a7a727013787aa0550a7a92 100644
--- a/api/funkwhale_api/federation/tasks.py
+++ b/api/funkwhale_api/federation/tasks.py
@@ -3,12 +3,14 @@ import json
import logging
import os
import requests
-
+import time
+from django.db.utils import IntegrityError
from django.conf import settings
from django.db import transaction
from django.db.models import Q, F
from django.db.models.deletion import Collector
from django.utils import timezone
+from django.urls import reverse
from dynamic_preferences.registries import global_preferences_registry
from requests.exceptions import RequestException
@@ -31,6 +33,8 @@ from . import serializers
from . import routes
from . import utils
from . import webfinger
+from . import api_serializers
+from . import models
logger = logging.getLogger(__name__)
@@ -90,7 +94,9 @@ def dispatch_inbox(activity, call_handlers=True):
Given an activity instance, triggers our internal delivery logic (follow
creation, etc.)
"""
-
+ logger.info(
+ "[federation] Dispatch inbox with activity payload : {activity.payload}"
+ )
routes.inbox.dispatch(
activity.payload,
context={
@@ -108,6 +114,8 @@ def dispatch_outbox(activity):
"""
Deliver a local activity to its recipients, both locally and remotely
"""
+ logger.info(f"[federation] Dispatch outbox activity payload : {activity.payload}")
+
inbox_items = activity.inbox_items.filter(is_read=False).select_related()
if inbox_items.exists():
@@ -206,7 +214,7 @@ def update_domain_nodeinfo(domain):
domain.service_actor = (
utils.retrieve_ap_object(
service_actor_id,
- actor=actors.get_service_actor(),
+ actor=None,
queryset=models.Actor,
serializer_class=serializers.ActorSerializer,
)
@@ -223,7 +231,23 @@ def update_domain_nodeinfo(domain):
)
domain.nodeinfo_fetch_date = now
domain.nodeinfo = nodeinfo
- domain.save(update_fields=["nodeinfo", "nodeinfo_fetch_date", "service_actor"])
+ if (
+ "payload" in domain.nodeinfo
+ and "software" in domain.nodeinfo["payload"]
+ and domain.nodeinfo["payload"]["software"]["name"] == "funkwhale"
+ ):
+ domain.is_funkwhale_instance = True
+ else:
+ domain.is_funkwhale_instance = False
+
+ domain.save(
+ update_fields=[
+ "nodeinfo",
+ "nodeinfo_fetch_date",
+ "service_actor",
+ "is_funkwhale_instance",
+ ]
+ )
@celery.app.task(name="federation.refresh_nodeinfo_known_nodes")
@@ -625,3 +649,162 @@ def fetch_collection(url, max_pages, channel, is_page=False):
results["errored"],
)
return results
+
+
+@celery.app.task(name="federation.domain_bulk_follow_libraries")
+@celery.require_instance(
+ models.Actor.objects.all(),
+ "actor",
+)
+def domain_bulk_follow_libraries(name, actor):
+ if name.startswith("https://"):
+ pass
+ else:
+ url = utils.get_remote_url(
+ name, reverse("api:v1:libraries-list"), f"domain={name}"
+ )
+ response = utils.get_remote_libraries(url, actor)
+ data, updated = mrf.inbox.apply(response)
+ if not data:
+ raise exceptions.BlockedActorOrDomain()
+
+ serializer = api_serializers.LibrarySerializer(data=response["results"], many=True)
+ # we do not raise exeption since we know some libs will not be valid (already exist)
+ serializer.is_valid()
+ serializer.save()
+
+ fids = []
+ for lib in response["results"]:
+ fids.append(lib["fid"])
+
+ if "next" in response.keys() and response["next"] is not None:
+ follow_libraries_locally(fids, actor)
+ url = response["next"]
+ domain_bulk_follow_libraries(url, actor_id=actor.id)
+ else:
+ return follow_libraries_locally(fids, actor)
+
+
+def follow_libraries_locally(fids, actor):
+ """
+ This will create a LibraryFollow entry only localy,
+ without sending activity to the remote library host.
+ """
+ libs = music_models.Library.objects.in_bulk(fids, field_name="fid")
+ libraries_follow = []
+ for lib in libs.values():
+ library_follow = {}
+ library_follow["target"] = lib.uuid
+ library_follow["follow_context"] = "Domain"
+ libraries_follow.append(library_follow)
+
+ serializer = api_serializers.LibraryFollowSerializer(
+ data=libraries_follow, context={"actor": actor}, many=True
+ )
+ serializer.is_valid()
+ return serializer.save(actor=actor)
+
+
+@celery.app.task(name="federation.domain_bulk_unfollow_libraries")
+@celery.require_instance(
+ models.Actor.objects.all(),
+ "actor",
+)
+def domain_bulk_unfollow_libraries(name, actor):
+ lib_follows = models.LibraryFollow.objects.filter(
+ actor=actor,
+ target__followers_url__startswith=f"https://{name}",
+ follow_context="Domain",
+ )
+
+ return lib_follows.delete()
+
+
+@celery.app.task(name="federation.follow_all_domains")
+def follow_all_domains():
+ if settings.GLOBAL_FEDERATION == False:
+ logger.info("Global federation desabled in env variables. Skipping task.")
+ return
+
+ domains = (
+ models.Domain.objects.all()
+ .exclude(name=settings.FEDERATION_HOSTNAME)
+ .exclude(is_funkwhale_instance=False)
+ )
+ actor = actors.get_service_actor()
+
+ for domain in domains:
+ if not models.Domain.objects.filter(name=domain.name).exists():
+ try:
+ data = {"target": domain.name}
+ serializer = api_serializers.DomainFollowSerializer(
+ data=data, context={"actor": actor}
+ )
+
+ serializer.is_valid(raise_exception=True)
+ follow = serializer.save(actor=actor)
+ routes.outbox.dispatch({"type": "Follow"}, context={"follow": follow})
+ except Exception as e:
+ logger.error(f"Unhandeled error while creating domain follow : {e} ")
+
+
+@celery.app.task(name="federation.discover_domains")
+def discover_domains():
+ if settings.GLOBAL_FEDERATION == False:
+ logger.info("Global federation disabled in env variables. Skipping task.")
+ return
+ # to do : add exclude(reachable=False)
+ local_fw_domains = (
+ models.Domain.objects.all()
+ .exclude(is_funkwhale_instance=False)
+ .exclude(name=settings.FEDERATION_HOSTNAME)
+ .values_list("name", flat=True)
+ )
+
+ local_domains = (
+ models.Domain.objects.all()
+ .exclude(name=settings.FEDERATION_HOSTNAME)
+ .values_list("name", flat=True)
+ )
+
+ for local_fw_domain in local_fw_domains:
+ url = utils.get_remote_url(
+ local_fw_domain, reverse("api:v1:federation:domains-list")
+ )
+ discover_domains_from_url(url, local_domains)
+
+
+@celery.app.task(name="federation.discover_domains_from_funkwhale.audio")
+def discover_domains_from_funkwhale_audio():
+ if settings.GLOBAL_FEDERATION == False:
+ logger.info("Global federation disabled in env variables. Skipping task.")
+ return
+ url = "https://network.funkwhale.audio/api/domains"
+ discover_domains_from_url(url)
+
+
+def discover_domains_from_url(url, local_domains=[]):
+ try:
+ response = session.get_session().get(url)
+ except Exception:
+ return
+
+ if response.status_code != 200:
+ logger.info(f"No domain found from {url}.")
+ return
+
+ domain_list = response.json()
+
+ for remote_domain in domain_list["results"]:
+
+ if remote_domain["name"] in local_domains:
+ logger.info("Domain already in db. Skipping")
+ continue
+
+ else:
+ logger.info(f"Adding discovered domain {remote_domain} to db")
+ data = {"name": remote_domain["name"], "creation_date": timezone.now}
+ serializer = api_serializers.DomainSerializer(data=data)
+ serializer.is_valid(raise_exception=True)
+ domain = serializer.save()
+ update_domain_nodeinfo.delay(domain_name=domain.name)
diff --git a/api/funkwhale_api/federation/utils.py b/api/funkwhale_api/federation/utils.py
index 2bac8daf8fb8dd5e64fc9711e1367360f0ce2651..f679fb0fad344ed19e0b01e37278af62b7bc3eb9 100644
--- a/api/funkwhale_api/federation/utils.py
+++ b/api/funkwhale_api/federation/utils.py
@@ -10,10 +10,13 @@ from django.db.models import CharField, Q, Value
from funkwhale_api.common import session
from funkwhale_api.moderation import mrf
-
from . import exceptions
from . import signing
+import logging
+
+logger = logging.getLogger(__name__)
+
def full_url(path):
"""
@@ -82,7 +85,6 @@ def retrieve_ap_object(
existing = queryset.objects.filter(fid=fid).first()
if existing:
return existing
-
auth = (
None if not actor else signing.get_auth(actor.private_key, actor.private_key_id)
)
@@ -291,3 +293,19 @@ def can_manage(obj_owner, actor):
return True
return False
+
+
+def get_remote_libraries(url, actor):
+ auth = signing.get_auth(actor.private_key, actor.private_key_id)
+ response = session.get_session().get(
+ auth=auth,
+ url=url,
+ headers={"Content-Type": "application/activity+json"},
+ )
+ response.raise_for_status()
+ response = response.json()
+ return response
+
+
+def get_remote_url(hostname, reverse, scope=None):
+ return f"https://{hostname}{reverse}?scope:{scope}"
diff --git a/api/funkwhale_api/music/serializers.py b/api/funkwhale_api/music/serializers.py
index df4e701335e4ab7fedcdda4ffe5274ea6f7d61db..12ed4f7969f6009863b42822ec26cb31817b77c8 100644
--- a/api/funkwhale_api/music/serializers.py
+++ b/api/funkwhale_api/music/serializers.py
@@ -11,6 +11,7 @@ from funkwhale_api.common import serializers as common_serializers
from funkwhale_api.common import utils as common_utils
from funkwhale_api.federation import routes
from funkwhale_api.federation import utils as federation_utils
+from funkwhale_api.federation import models as federation_models
from funkwhale_api.playlists import models as playlists_models
from funkwhale_api.tags import models as tag_models
from funkwhale_api.tags import serializers as tags_serializers
@@ -370,6 +371,19 @@ class LibraryForOwnerSerializer(serializers.ModelSerializer):
{"type": "Update", "object": {"type": "Library"}}, context={"library": obj}
)
+ # If the library is updated after the domain was followed. We need to inform our domain
+ # followers that the library is now available for everyone
+ if after["privacy_level"] == "everyone":
+ routes.outbox.dispatch(
+ {"type": "Update", "object": {"type": "Domain"}},
+ context={
+ "library": obj,
+ "domain": federation_models.Domain.objects.get(
+ name=settings.FUNKWHALE_HOSTNAME
+ ),
+ },
+ )
+
def get_actor(self, o):
# Import at runtime to avoid a circular import issue
from funkwhale_api.federation import serializers as federation_serializers
diff --git a/api/funkwhale_api/music/views.py b/api/funkwhale_api/music/views.py
index d7eba93cfc8d98bd497d2e7c59b1b9b432c49b8a..bf231005426a73455bec1e0226afa606bd2d0d96 100644
--- a/api/funkwhale_api/music/views.py
+++ b/api/funkwhale_api/music/views.py
@@ -290,7 +290,18 @@ class LibraryViewSet(
return qs
def perform_create(self, serializer):
- serializer.save(actor=self.request.user.actor)
+ lib = serializer.save(actor=self.request.user.actor)
+ if lib.privacy_level == "everyone":
+ lib.actor = self.request.user.actor
+ routes.outbox.dispatch(
+ {"type": "Update", "object": {"type": "Domain"}},
+ context={
+ "domain": federation_models.Domain.objects.get(
+ name=settings.FUNKWHALE_HOSTNAME
+ ),
+ "library": lib,
+ },
+ )
@transaction.atomic
def perform_destroy(self, instance):
@@ -550,6 +561,7 @@ def record_downloads(f):
@record_downloads
+# to do : tests fail here ?
def handle_serve(
upload, user, format=None, max_bitrate=None, proxy_media=True, download=True
):
diff --git a/api/tests/federation/test_api_views.py b/api/tests/federation/test_api_views.py
index e0fcc6edf46849382c15c033db40752ed92a6953..b3a2bc45db122205901464af2b95f69945abb43e 100644
--- a/api/tests/federation/test_api_views.py
+++ b/api/tests/federation/test_api_views.py
@@ -8,6 +8,7 @@ from funkwhale_api.federation import api_serializers
from funkwhale_api.federation import serializers
from funkwhale_api.federation import tasks
from funkwhale_api.federation import views
+from funkwhale_api.federation import models
def test_user_can_list_their_library_follows(factories, logged_in_api_client):
@@ -320,3 +321,68 @@ def test_library_follow_get_all(factories, logged_in_api_client):
],
"count": 1,
}
+
+
+def test_domain_follow_get_all(factories, logged_in_api_client):
+ actor = logged_in_api_client.user.create_actor()
+ domain = factories["federation.Domain"]()
+ follow = factories["federation.DomainFollow"](target=domain, actor=actor)
+ factories["federation.DomainFollow"]()
+ factories["federation.Domain"]()
+ url = reverse("api:v1:federation:domain-follows-all")
+ response = logged_in_api_client.get(url)
+
+ assert response.status_code == 200
+ assert response.data == {
+ "results": [
+ {
+ "uuid": str(follow.uuid),
+ "domain": str(domain.name),
+ "approved": follow.approved,
+ }
+ ],
+ "count": 1,
+ }
+
+
+def test_can_follow_domain(factories, logged_in_api_client, mocker):
+ dispatch = mocker.patch("funkwhale_api.federation.routes.outbox.dispatch")
+ actor = logged_in_api_client.user.create_actor()
+ domain = factories["federation.Domain"]()
+ url = reverse("api:v1:federation:domain-follows-list")
+ response = logged_in_api_client.post(url, {"target": domain.name})
+
+ assert response.status_code == 201
+
+ follow = domain.domainfollow_received_follows.latest("id")
+
+ assert follow.approved is None
+ assert follow.actor == actor
+
+ dispatch.assert_called_once_with({"type": "Follow"}, context={"follow": follow})
+
+
+def test_can_undo_domain_follow(factories, logged_in_api_client, mocker):
+ dispatch = mocker.patch("funkwhale_api.federation.routes.outbox.dispatch")
+ unfollow_libs_delayed = mocker.patch.object(
+ tasks.domain_bulk_unfollow_libraries, "delay"
+ )
+ actor = (
+ logged_in_api_client.user.create_actor()
+ ) # Object of type Actor is not JSON serializable
+ domain = factories["federation.Domain"]()
+ follow = factories["federation.DomainFollow"](target=domain, actor=actor)
+ delete = mocker.patch.object(follow.__class__, "delete")
+ url = reverse(
+ "api:v1:federation:domain-follows-detail", kwargs={"uuid": follow.uuid}
+ )
+ # actor = factories["federation.Actor"]()
+ response = logged_in_api_client.delete(url, actor=actor)
+
+ assert response.status_code == 204
+
+ delete.assert_called_once_with()
+ dispatch.assert_called_once_with(
+ {"type": "Undo", "object": {"type": "Follow"}}, context={"follow": follow}
+ )
+ unfollow_libs_delayed.assert_called_once_with(domain.name, actor_id=actor.pk)
diff --git a/api/tests/federation/test_models.py b/api/tests/federation/test_models.py
index 6f722c6e8a3947d11e7171f32de164ba9ac4c2e5..4c89f7f321f7a48433f15d7d9e9d22fbb1661e26 100644
--- a/api/tests/federation/test_models.py
+++ b/api/tests/federation/test_models.py
@@ -134,6 +134,8 @@ def test_domain_stats(factories):
"emitted_library_follows": 0,
"media_total_size": 0,
"media_downloaded_size": 0,
+ "emitted_domain_follows": 0,
+ "received_domain_follows": 0,
}
domain = factories["federation.Domain"]()
diff --git a/api/tests/federation/test_routes.py b/api/tests/federation/test_routes.py
index 108ca41b8fa265df3387a656b943928920cd3ee3..d087eae086aa7f60662a81cbde3aa2fe3b084571 100644
--- a/api/tests/federation/test_routes.py
+++ b/api/tests/federation/test_routes.py
@@ -1,5 +1,7 @@
import pytest
-
+from django.urls import reverse
+from django.conf import settings
+import uuid
from funkwhale_api.federation import (
activity,
actors,
@@ -7,8 +9,10 @@ from funkwhale_api.federation import (
jsonld,
routes,
serializers,
+ models,
)
from funkwhale_api.moderation import serializers as moderation_serializers
+from funkwhale_api.federation import routes
@pytest.mark.parametrize(
@@ -501,12 +505,16 @@ def test_outbox_delete_library(factories):
def test_outbox_update_library(factories):
library = factories["music.Library"]()
+ domain = factories["federation.Domain"](name=settings.FEDERATION_HOSTNAME)
activity = list(routes.outbox_update_library({"library": library}))[0]
expected = serializers.ActivitySerializer(
{"type": "Update", "object": serializers.LibrarySerializer(library).data}
).data
- expected["to"] = [{"type": "followers", "target": library}]
+ expected["to"] = [
+ {"type": "followers", "target": library},
+ {"type": "domain_followers", "target": domain},
+ ]
assert dict(activity["payload"]) == dict(expected)
assert activity["actor"] == library.actor
@@ -989,3 +997,81 @@ def test_outbox_flag(factory_name, factory_kwargs, factories, mocker):
expected["to"] = [{"type": "actor_inbox", "actor": report.target_owner}]
assert activity["payload"] == expected
assert activity["actor"] == actors.get_service_actor()
+
+
+def test_dispatch_outbox_update_domain_on_library_creation(
+ factories, mocker, logged_in_api_client
+):
+ local_domain = factories["federation.Domain"](name=settings.FUNKWHALE_HOSTNAME)
+ actor = logged_in_api_client.user.create_actor()
+ url = reverse("api:v1:libraries-list")
+ dispatch = mocker.patch("funkwhale_api.federation.routes.outbox.dispatch")
+ logged_in_api_client.post(
+ url, {"name": "hello", "description": "world", "privacy_level": "everyone"}
+ )
+ library = actor.libraries.first()
+
+ dispatch.assert_called_once_with(
+ {"type": "Update", "object": {"type": "Domain"}},
+ context={"domain": local_domain, "library": library},
+ )
+
+
+def test_inbox_update_library_from_domain_follow(
+ factories, mocker, logged_in_api_client
+):
+
+ domain = factories["federation.Domain"]()
+ local_domain = factories["federation.Domain"](name=settings.FUNKWHALE_HOSTNAME)
+ domain_follow = factories["federation.DomainFollow"].create_batch(10, target=domain)
+ lib_uuid = uuid.uuid4()
+ fid = "https://{}/federation/music/libraries/{}".format(domain.name, lib_uuid)
+ library = factories["music.Library"](privacy_level="everyone", fid=fid)
+ serializer = serializers.ActivitySerializer(
+ {"type": "Update", "object": serializers.LibrarySerializer(library).data}
+ )
+ serializer.data["to"] = {"type": "domain_followers", "target": domain}
+ payload = serializer.data
+
+ routes.inbox_update_library_from_domain_follow(
+ payload, context={"domain": local_domain, "library": library}
+ )
+
+ assert models.LibraryFollow.objects.filter(target=library).count() == 10
+
+
+# from funkwhale_api.music import models as music_models
+
+
+# def test_inbox_update_library_from_domain_follow(
+# factories, mocker, logged_in_api_client
+# ):
+# from funkwhale_api.music import models as music_models
+
+# domain = factories["federation.Domain"]()
+# actor = factories["federation.Actor"](domain=domain)
+# local_domain = factories["federation.Domain"](name=settings.FUNKWHALE_HOSTNAME)
+# lib_uuid = uuid.uuid4()
+# fid = "https://{}/federation/music/libraries/{}".format(domain.name, lib_uuid)
+# followers_url = "https://{}/federation/music/libraries/{}/followers".format(
+# domain.name, lib_uuid
+# )
+# library = music_models.Library(
+# privacy_level="everyone",
+# fid=fid,
+# name="blabla",
+# actor=actor,
+# followers_url=followers_url,
+# )
+
+# serializer = serializers.ActivitySerializer(
+# {"type": "Update", "object": serializers.LibrarySerializer(library).data}
+# )
+# serializer.data["to"] = {"type": "domain_followers", "target": domain}
+# payload = serializer.data
+
+# routes.inbox_update_library_from_domain_follow(
+# payload, context={"domain": local_domain, "library": library}
+# )
+
+# assert models.LibraryFollow.objects.filter(target=library).count() == 10
diff --git a/api/tests/federation/test_tasks.py b/api/tests/federation/test_tasks.py
index 82192cef01486b82965487348dc629076f4d46e2..40ed13b70d047248753fbbca520117da30461f59 100644
--- a/api/tests/federation/test_tasks.py
+++ b/api/tests/federation/test_tasks.py
@@ -1,7 +1,12 @@
import datetime
+from funkwhale_api.music import models as music_models
+from funkwhale_api.music import serializers as music_serializers
+
+import factory
import os
import pathlib
import pytest
+import uuid
from django.utils import timezone
@@ -10,6 +15,10 @@ from funkwhale_api.federation import models
from funkwhale_api.federation import serializers
from funkwhale_api.federation import tasks
from funkwhale_api.federation import utils
+from funkwhale_api.common import utils as common_utils
+
+
+from funkwhale_api.federation.factories import MusicLibraryFactory, ActorFactory
def test_clean_federation_music_cache_if_no_listen(preferences, factories):
@@ -210,7 +219,7 @@ def test_update_domain_nodeinfo(factories, mocker, now, service_actor):
retrieve_ap_object.assert_called_once_with(
"https://actor.id",
- actor=service_actor,
+ actor=None,
queryset=models.Actor,
serializer_class=serializers.ActorSerializer,
)
@@ -670,3 +679,52 @@ def test_fetch_collection(mocker, r_mock):
assert result["seen"] == 7
assert result["total"] == 27094
assert result["next_page"] == payloads["page2"]["next"]
+
+
+def test_domain_bulk_follow_libraries(factories, mocker, logged_in_api_client):
+ domain = factories["federation.Domain"]()
+ actor = factories["federation.Actor"]()
+ i = 0
+ lib1 = factories["music.Library"](privacy_level="everyone")
+ lib2 = factories["music.Library"](privacy_level="everyone")
+ lib3 = factories["music.Library"](privacy_level="everyone")
+
+ libs = [lib1, lib2, lib3]
+ libs_remote = {}
+ libraries_dic = []
+
+ for lib in libs:
+ lib_json = music_serializers.LibraryForOwnerSerializer(lib)
+ libraries_dic.append(lib_json.data)
+ libs_remote["results"] = libraries_dic
+ mocker.patch(
+ "funkwhale_api.federation.utils.get_remote_libraries", return_value=libs_remote
+ )
+ tasks.domain_bulk_follow_libraries(domain.name, actor_id=actor.pk)
+ assert music_models.Library.objects.filter(uuid=lib1.uuid).exists()
+ assert models.LibraryFollow.objects.filter(target__fid=lib1.fid).exists()
+ assert models.LibraryFollow.objects.filter(target__fid=lib2.fid).exists()
+ assert models.LibraryFollow.objects.filter(target__fid=lib3.fid).exists()
+
+
+def test_domain_can_bulk_unfollow_libraries(factories, mocker, logged_in_api_client):
+ domain = factories["federation.Domain"]()
+ actor = logged_in_api_client.user.create_actor()
+ dispacth_unfollow = mocker.patch("funkwhale_api.federation.routes.outbox.dispatch")
+ i = 0
+ calls = []
+ follows = []
+ while i < 10:
+ i = i + 1
+ lib_uuid = uuid.uuid4()
+ followers_url = "https://{}/federation/music/libraries/{}/followers".format(
+ domain.name, lib_uuid
+ )
+ lib = factories["music.Library"](followers_url=followers_url)
+ follow = factories["federation.LibraryFollow"](
+ actor=actor, target=lib, follow_context="Domain"
+ )
+ follows.append(follow)
+ tasks.domain_bulk_unfollow_libraries(domain.name, actor_id=actor.pk)
+ for follow in follows:
+ assert models.LibraryFollow.objects.filter(uuid=follow.uuid).exists() == False
diff --git a/api/tests/music/test_serializers.py b/api/tests/music/test_serializers.py
index beb30279d3d787558b8901df7cd2bb07078c7f77..ce0f948f0a832a08042989b361de9b9855e25a94 100644
--- a/api/tests/music/test_serializers.py
+++ b/api/tests/music/test_serializers.py
@@ -1,6 +1,6 @@
import pytest
import uuid
-
+from django.conf import settings
from funkwhale_api.common import serializers as common_serializers
from funkwhale_api.federation import serializers as federation_serializers
from funkwhale_api.music import licenses
@@ -406,16 +406,24 @@ def test_update_library_privacy_level_broadcasts_to_followers(
):
dispatch = mocker.patch("funkwhale_api.federation.routes.outbox.dispatch")
library = factories["music.Library"](**{field: before})
+ domain = factories["federation.Domain"](name=settings.FUNKWHALE_HOSTNAME)
serializer = serializers.LibraryForOwnerSerializer(
library, data={field: after}, partial=True
)
assert serializer.is_valid(raise_exception=True)
serializer.save()
-
- dispatch.assert_called_once_with(
- {"type": "Update", "object": {"type": "Library"}}, context={"library": library}
- )
+ if field == "privacy_level":
+ dispatch.assert_called_with(
+ {"type": "Update", "object": {"type": "Domain"}},
+ context={"library": library, "domain": domain},
+ )
+
+ else:
+ dispatch.assert_called_once_with(
+ {"type": "Update", "object": {"type": "Library"}},
+ context={"library": library},
+ )
def test_upload_with_channel(factories, uploaded_audio_file):
diff --git a/changes/changelog.d/762.newfeature b/changes/changelog.d/762.newfeature
new file mode 100644
index 0000000000000000000000000000000000000000..b61b4fdeb76e43a2c6c4b1db04cd42f6eb9ceea8
--- /dev/null
+++ b/changes/changelog.d/762.newfeature
@@ -0,0 +1,2 @@
+Alow actor to follow entire instances (#762)
+New env variable GLOBAL_FEDERATION enable should_autoapprove_follow for the domain.
diff --git a/front/src/components/federation/DomainFollowButton.vue b/front/src/components/federation/DomainFollowButton.vue
new file mode 100644
index 0000000000000000000000000000000000000000..338f968f77c110548ae18890f1a12dc80d5d41e9
--- /dev/null
+++ b/front/src/components/federation/DomainFollowButton.vue
@@ -0,0 +1,57 @@
+
+
+
+
+
diff --git a/front/src/store/domains.js b/front/src/store/domains.js
new file mode 100644
index 0000000000000000000000000000000000000000..163c4b17c3815e6fa7f83bb8b7449e49dd9bd995
--- /dev/null
+++ b/front/src/store/domains.js
@@ -0,0 +1,74 @@
+import axios from 'axios'
+import logger from '@/logging'
+
+export default {
+ namespaced: true,
+ state: {
+ followedDomains: [],
+ followsByDomain: {},
+ count: 0
+ },
+ mutations: {
+ follows: (state, { domain, follow }) => {
+ const replacement = { ...state.followsByDomain }
+ if (follow) {
+ if (state.followedDomains.indexOf(domain) === -1) {
+ state.followedDomains.push(domain)
+ replacement[domain] = follow
+ }
+ } else {
+ const i = state.followedDomains.indexOf(domain)
+ if (i > -1) {
+ state.followedDomains.splice(i, 1)
+ replacement[domain] = null
+ }
+ }
+ state.followsByDomain = replacement
+ state.count = state.followedDomains.length
+ },
+ reset (state) {
+ state.followedDomains = []
+ state.followsByDomain = {}
+ state.count = 0
+ }
+ },
+ getters: {
+ follow: (state) => (domain) => {
+ return state.followsByDomain[domain]
+ }
+ },
+ actions: {
+ set ({ commit, state }, { name, value }) {
+ if (value) {
+ return axios.post('federation/follows/domain/', { target: name }).then((response) => {
+ logger.default.info('Successfully subscribed to domain')
+ commit('follows', { domain: name, follow: response.data }
+ )
+ }, (response) => {
+ logger.default.info('Error while subscribing to domain')
+ commit('follows', { domain: name, follow: null })
+ })
+ } else {
+ const follow = state.followsByDomain[name]
+ return axios.delete(`federation/follows/domain/${follow.uuid}/`).then((response) => {
+ logger.default.info('Successfully unsubscribed from domain')
+ commit('follows', { domain: name, follow: null })
+ }, (response) => {
+ logger.default.info('Error while unsubscribing from domain')
+ commit('follows', { domain: name, follow: follow })
+ })
+ }
+ },
+ toggle ({ getters, dispatch }, name) {
+ dispatch('set', { name, value: !getters.follow(name) })
+ },
+ fetchFollows ({ dispatch, state, commit, rootState }, url) {
+ const promise = axios.get('federation/follows/domain/all/')
+ return promise.then((response) => {
+ response.data.results.forEach(result => {
+ commit('follows', { domain: result.domain, follow: result })
+ })
+ })
+ }
+ }
+}
diff --git a/front/src/store/index.js b/front/src/store/index.js
index a7954420346599c335856b82980522dc6239730c..7ad2b90a38a97ebb19bd00a73a564397cd3e13b0 100644
--- a/front/src/store/index.js
+++ b/front/src/store/index.js
@@ -4,6 +4,7 @@ import createPersistedState from 'vuex-persistedstate'
import favorites from './favorites'
import channels from './channels'
+import domains from './domains'
import libraries from './libraries'
import auth from './auth'
import instance from './instance'
@@ -21,6 +22,7 @@ export default new Vuex.Store({
ui,
auth,
channels,
+ domains,
libraries,
favorites,
instance,
diff --git a/front/src/views/admin/moderation/DomainsDetail.vue b/front/src/views/admin/moderation/DomainsDetail.vue
index cfccac97d4846dc0ead602e5788719be50b48a11..2915e09a091cbada6d7b835ca036e7d0027d30c5 100644
--- a/front/src/views/admin/moderation/DomainsDetail.vue
+++ b/front/src/views/admin/moderation/DomainsDetail.vue
@@ -11,6 +11,9 @@
v-title="object.name"
:class="['ui', 'head', 'vertical', 'stripe', 'segment']"
>
+