diff --git a/api/funkwhale_api/federation/authentication.py b/api/funkwhale_api/federation/authentication.py
index 75e0332421feb115eac1ee58acc2906b2ae06ae5..123d7bd89679ae0ece57b49b3f6f21fcfcc17193 100644
--- a/api/funkwhale_api/federation/authentication.py
+++ b/api/funkwhale_api/federation/authentication.py
@@ -1,13 +1,14 @@
 import cryptography
 import logging
 import datetime
-
+import urllib.parse
 from django.contrib.auth.models import AnonymousUser
 from django.utils import timezone
 
 from rest_framework import authentication, exceptions as rest_exceptions
+from funkwhale_api.common import preferences
 from funkwhale_api.moderation import models as moderation_models
-from . import actors, exceptions, keys, signing, tasks, utils
+from . import actors, exceptions, keys, models, signing, tasks, utils
 
 
 logger = logging.getLogger(__name__)
@@ -37,6 +38,16 @@ class SignatureAuthentication(authentication.BaseAuthentication):
         if policies.exists():
             raise exceptions.BlockedActorOrDomain()
 
+        if request.method.lower() == "get" and preferences.get(
+            "moderation__allow_list_enabled"
+        ):
+            # Only GET requests because POST requests with messages will be handled through
+            # MRF
+            domain = urllib.parse.urlparse(actor_url).hostname
+            allowed = models.Domain.objects.filter(name=domain, allowed=True).exists()
+            if not allowed:
+                raise exceptions.BlockedActorOrDomain()
+
         try:
             actor = actors.get_actor(actor_url)
         except Exception as e:
diff --git a/api/funkwhale_api/federation/views.py b/api/funkwhale_api/federation/views.py
index 97bcebbfb374d5a765f30386ab430935d84f68f5..85961e3229df1902999e6428138e237bee0b371e 100644
--- a/api/funkwhale_api/federation/views.py
+++ b/api/funkwhale_api/federation/views.py
@@ -2,7 +2,7 @@ from django import forms
 from django.core import paginator
 from django.http import HttpResponse
 from django.urls import reverse
-from rest_framework import exceptions, mixins, response, viewsets
+from rest_framework import exceptions, mixins, permissions, response, viewsets
 from rest_framework.decorators import action
 
 from funkwhale_api.common import preferences
@@ -12,7 +12,17 @@ from funkwhale_api.music import utils as music_utils
 from . import activity, authentication, models, renderers, serializers, utils, webfinger
 
 
+class AuthenticatedIfAllowListEnabled(permissions.BasePermission):
+    def has_permission(self, request, view):
+        allow_list_enabled = preferences.get("moderation__allow_list_enabled")
+        if not allow_list_enabled:
+            return True
+        return bool(request.actor)
+
+
 class FederationMixin(object):
+    permission_classes = [AuthenticatedIfAllowListEnabled]
+
     def dispatch(self, request, *args, **kwargs):
         if not preferences.get("federation__enabled"):
             return HttpResponse(status=405)
@@ -20,7 +30,6 @@ class FederationMixin(object):
 
 
 class SharedViewSet(FederationMixin, viewsets.GenericViewSet):
-    permission_classes = []
     authentication_classes = [authentication.SignatureAuthentication]
     renderer_classes = renderers.get_ap_renderers()
 
@@ -38,7 +47,6 @@ class SharedViewSet(FederationMixin, viewsets.GenericViewSet):
 class ActorViewSet(FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet):
     lookup_field = "preferred_username"
     authentication_classes = [authentication.SignatureAuthentication]
-    permission_classes = []
     renderer_classes = renderers.get_ap_renderers()
     queryset = models.Actor.objects.local().select_related("user")
     serializer_class = serializers.ActorSerializer
@@ -73,7 +81,6 @@ class ActorViewSet(FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericV
 class EditViewSet(FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet):
     lookup_field = "uuid"
     authentication_classes = [authentication.SignatureAuthentication]
-    permission_classes = []
     renderer_classes = renderers.get_ap_renderers()
     # queryset = common_models.Mutation.objects.local().select_related()
     # serializer_class = serializers.ActorSerializer
@@ -146,7 +153,6 @@ class MusicLibraryViewSet(
     FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet
 ):
     authentication_classes = [authentication.SignatureAuthentication]
-    permission_classes = []
     renderer_classes = renderers.get_ap_renderers()
     serializer_class = serializers.LibrarySerializer
     queryset = music_models.Library.objects.all().select_related("actor")
@@ -201,7 +207,6 @@ class MusicUploadViewSet(
     FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet
 ):
     authentication_classes = [authentication.SignatureAuthentication]
-    permission_classes = []
     renderer_classes = renderers.get_ap_renderers()
     queryset = music_models.Upload.objects.local().select_related(
         "library__actor", "track__artist", "track__album__artist"
@@ -219,7 +224,6 @@ class MusicArtistViewSet(
     FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet
 ):
     authentication_classes = [authentication.SignatureAuthentication]
-    permission_classes = []
     renderer_classes = renderers.get_ap_renderers()
     queryset = music_models.Artist.objects.local()
     serializer_class = serializers.ArtistSerializer
@@ -230,7 +234,6 @@ class MusicAlbumViewSet(
     FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet
 ):
     authentication_classes = [authentication.SignatureAuthentication]
-    permission_classes = []
     renderer_classes = renderers.get_ap_renderers()
     queryset = music_models.Album.objects.local().select_related("artist")
     serializer_class = serializers.AlbumSerializer
@@ -241,7 +244,6 @@ class MusicTrackViewSet(
     FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet
 ):
     authentication_classes = [authentication.SignatureAuthentication]
-    permission_classes = []
     renderer_classes = renderers.get_ap_renderers()
     queryset = music_models.Track.objects.local().select_related(
         "album__artist", "artist"
diff --git a/api/tests/federation/test_authentication.py b/api/tests/federation/test_authentication.py
index 4e837e64177be9919881d0b5fe6adef94bbdaf6f..643bccccb6ca12e80a180d2b1fc5a1e7d6297f63 100644
--- a/api/tests/federation/test_authentication.py
+++ b/api/tests/federation/test_authentication.py
@@ -178,3 +178,28 @@ def test_autenthicate_supports_blind_key_rotation(factories, mocker, api_request
     assert user.is_anonymous is True
     assert actor.public_key == new_public.decode("utf-8")
     assert actor.fid == actor_url
+
+
+def test_authenticate_checks_signature_with_allow_list(
+    preferences, factories, api_request
+):
+    preferences["moderation__allow_list_enabled"] = True
+    domain = factories["federation.Domain"](allowed=False)
+    private, public = keys.get_key_pair()
+    actor_url = "https://{}/actor".format(domain.name)
+
+    signed_request = factories["federation.SignedRequest"](
+        auth__key=private, auth__key_id=actor_url + "#main-key", auth__headers=["date"]
+    )
+    prepared = signed_request.prepare()
+    django_request = api_request.get(
+        "/",
+        **{
+            "HTTP_DATE": prepared.headers["date"],
+            "HTTP_SIGNATURE": prepared.headers["signature"],
+        }
+    )
+    authenticator = authentication.SignatureAuthentication()
+
+    with pytest.raises(exceptions.BlockedActorOrDomain):
+        authenticator.authenticate(django_request)
diff --git a/api/tests/federation/test_views.py b/api/tests/federation/test_views.py
index 93ce05b8ebde3b88cbd1c5528ce4db6579256d7f..51d8e79a93f76d3696b27a6d77bfd9d93476f851 100644
--- a/api/tests/federation/test_views.py
+++ b/api/tests/federation/test_views.py
@@ -5,6 +5,20 @@ from django.urls import reverse
 from funkwhale_api.federation import actors, serializers, webfinger
 
 
+def test_authenticate_skips_anonymous_fetch_when_allow_list_enabled(
+    preferences, api_client
+):
+    preferences["moderation__allow_list_enabled"] = True
+    actor = actors.get_service_actor()
+    url = reverse(
+        "federation:actors-detail",
+        kwargs={"preferred_username": actor.preferred_username},
+    )
+    response = api_client.get(url)
+
+    assert response.status_code == 403
+
+
 def test_wellknown_webfinger_validates_resource(db, api_client, settings, mocker):
     clean = mocker.spy(webfinger, "clean_resource")
     url = reverse("federation:well-known-webfinger")