Commit bd93515f authored by petitminion's avatar petitminion
Browse files

Allowing to follow entire instances / domains

parent 75b33ceb
Pipeline #20826 passed with stage
in 13 seconds
DJANGO_ALLOWED_HOSTS=.funkwhale.test,localhost,nginx,0.0.0.0,127.0.0.1
DJANGO_SETTINGS_MODULE=config.settings.local
DJANGO_SECRET_KEY=dev
C_FORCE_ROOT=true
FUNKWHALE_HOSTNAME=localhost
FUNKWHALE_PROTOCOL=http
PYTHONDONTWRITEBYTECODE=true
VUE_PORT=8080
MUSIC_DIRECTORY_PATH=/music
BROWSABLE_API_ENABLED=True
FORWARDED_PROTO=http
LDAP_ENABLED=False
FUNKWHALE_SPA_HTML_ROOT=http://nginx/front/
PYTHONTRACEMALLOC=0
# Uncomment this if you're using traefik/https
# FORCE_HTTPS_URLS=True
# Customize to your needs
POSTGRES_VERSION=11
DEBUG=true
......@@ -4,37 +4,38 @@ FROM alpine:3.14 as pre-build
# dependencies. This is only required until alpine 3.16 is released, since this
# allows us to install poetry as package.
RUN apk add --no-cache python3 py3-cryptography py3-pip && \
pip3 install poetry
RUN --mount=type=cache,target=/root/.cache/pip \
apk add --no-cache python3 py3-cryptography py3-pip && \
pip3 install poetry
COPY pyproject.toml poetry.lock /
RUN poetry export --without-hashes > requirements.txt
RUN poetry export --dev --without-hashes > dev-requirements.txt
FROM alpine:3.14 as builder
RUN \
echo 'installing dependencies' && \
apk add --no-cache \
git \
musl-dev \
gcc \
postgresql-dev \
python3-dev \
py3-psycopg2 \
py3-cryptography \
libldap \
libffi-dev \
make \
zlib-dev \
jpeg-dev \
openldap-dev \
openssl-dev \
cargo \
libxml2-dev \
libxslt-dev \
curl \
&& \
ln -s /usr/bin/python3 /usr/bin/python
RUN --mount=type=cache,target=/root/.cache/pip \
echo 'installing dependencies' && \
apk add --no-cache \
git \
musl-dev \
gcc \
postgresql-dev \
python3-dev \
py3-psycopg2 \
py3-cryptography \
libldap \
libffi-dev \
make \
zlib-dev \
jpeg-dev \
openldap-dev \
openssl-dev \
cargo \
libxml2-dev \
libxslt-dev \
curl \
&& \
ln -s /usr/bin/python3 /usr/bin/python
# create virtual env for next stage
RUN python -m venv --system-site-packages /venv
......@@ -44,26 +45,26 @@ ENV PATH="/venv/bin:/root/.local/bin:$PATH" VIRTUAL_ENV=/venv
COPY --from=0 /requirements.txt /requirements.txt
COPY --from=0 /dev-requirements.txt /dev-requirements.txt
# hack around https://github.com/pypa/pip/issues/6158#issuecomment-456619072
ENV PIP_DOWNLOAD_CACHE=/noop/
RUN \
echo 'installing pip requirements' && \
pip3 install --upgrade pip && \
pip3 install setuptools wheel && \
# Currently we are unable to relieably build cryptography on armv7. This
# is why we need to use the package shipped by Alpine Linux, which is currently
# version 3.3.2. Since poetry does not allow in-place dependency pinning, we need
# to install the deps using pip.
cat /requirements.txt | grep -Ev 'cryptography|autobahn' | pip3 install -r /dev/stdin cryptography==3.3.2 autobahn==21.2.1 && \
rm -rf $PIP_DOWNLOAD_CACHE
#ENV PIP_DOWNLOAD_CACHE=/noop/
RUN --mount=type=cache,target=/root/.cache/pip \
echo 'installing pip requirements' && \
pip3 install --upgrade pip && \
pip3 install setuptools wheel && \
# Currently we are unable to relieably build cryptography on armv7. This
# is why we need to use the package shipped by Alpine Linux, which is currently
# version 3.3.2. Since poetry does not allow in-place dependency pinning, we need
# to install the deps using pip.
cat /requirements.txt | grep -Ev 'cryptography|autobahn' | pip3 install -r /dev/stdin cryptography==3.3.2 autobahn==21.2.1 && \
#rm -rf $PIP_DOWNLOAD_CACHE
ARG install_dev_deps=0
ARG install_dev_deps=0
RUN \
if [ "$install_dev_deps" = "1" ] ; then \
echo "Installing dev dependencies" && \
cat /dev-requirements.txt | grep -Ev 'cryptography|autobahn' | pip3 install -r /dev/stdin cryptography==3.3.2 autobahn==21.2.1 \
; else \
echo "Skipping dev deps installation" \
; fi
if [ "$install_dev_deps" = "1" ] ; then \
echo "Installing dev dependencies" && \
cat /dev-requirements.txt | grep -Ev 'cryptography|autobahn' | pip3 install -r /dev/stdin cryptography==3.3.2 autobahn==21.2.1 \
; else \
echo "Skipping dev deps installation" \
; fi
FROM alpine:3.14 as build-image
......@@ -73,18 +74,18 @@ COPY --from=builder /venv /venv
ENV PATH="/venv/bin:$PATH"
RUN apk add --no-cache \
libmagic \
bash \
gettext \
python3 \
jpeg-dev \
ffmpeg \
libpq \
libxml2 \
libxslt \
py3-cryptography \
&& \
ln -s /usr/bin/python3 /usr/bin/python
libmagic \
bash \
gettext \
python3 \
jpeg-dev \
ffmpeg \
libpq \
libxml2 \
libxslt \
py3-cryptography \
&& \
ln -s /usr/bin/python3 /usr/bin/python
COPY . /app
WORKDIR /app
......
......@@ -190,6 +190,7 @@ ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=[]) + [FUNKWHALE_HOSTNA
"""
List of allowed hostnames for which the Funkwhale server will answer.
"""
GLOBAL_FEDERATION = env.bool("GLOBAL_FEDERATION", default=True)
# APP CONFIGURATION
# ------------------------------------------------------------------------------
......
......@@ -92,7 +92,7 @@ if DEBUG_TOOLBAR_ENABLED:
# ------------------------------------------------------------------------------
TEST_RUNNER = "django.test.runner.DiscoverRunner"
# CELERY
# CELERY Set to true to debug
CELERY_TASK_ALWAYS_EAGER = False
# END CELERY
......
......@@ -12,6 +12,11 @@ from funkwhale_api.common import utils as funkwhale_utils
from . import contexts
import logging
logger = logging.getLogger(__name__)
recursive_getattr = funkwhale_utils.recursive_getattr
......@@ -119,13 +124,16 @@ def should_reject(fid, actor_id=None, payload={}):
@transaction.atomic
def receive(activity, on_behalf_of, inbox_actor=None):
"""
Receive an activity, find his recipients and save it to the database before dispatching it
"""
from . import models
from . import serializers
from . import tasks
from .routes import inbox
from funkwhale_api.moderation import mrf
logger.debug(
logger.info(
"[federation] Received activity from %s : %s", on_behalf_of.fid, activity
)
# we ensure the activity has the bare minimum structure before storing
......@@ -339,6 +347,10 @@ class OutboxRouter(Router):
deliveries_by_activity_uuid = {}
prepared_activities = []
for activity_data in activities_data:
# If its a domain update we change the object type of the library to "Domain" to allow specific routing
for recipient in activity_data["payload"]["to"]:
if recipient is dict and "domain_followers" in recipient["type"]:
activity_data["payload"]["object"]["type"] = "Domain"
activity_data["payload"]["actor"] = activity_data["actor"].fid
to = activity_data["payload"].pop("to", [])
cc = activity_data["payload"].pop("cc", [])
......@@ -461,6 +473,19 @@ def prepare_deliveries_and_inbox_items(recipient_list, type, allowed_domains=Non
else:
remote_inbox_urls.add(actor.shared_inbox_url or actor.inbox_url)
urls.append(r["target"].followers_url)
elif isinstance(r, dict) and r["type"] == "domain_followers":
received_follows = (
r["target"]
.domainfollow_received_follows.filter(approved=True)
.select_related("actor__user")
)
logger.info("received_follows" + str(received_follows))
for follow in received_follows:
actor = follow.actor
remote_inbox_urls.add(actor.shared_inbox_url or actor.inbox_url)
urls.append(actor.shared_inbox_url)
elif isinstance(r, dict) and r["type"] == "actor_inbox":
actor = r["actor"]
urls.append(actor.fid)
......
......@@ -78,6 +78,14 @@ class LibraryFollowAdmin(admin.ModelAdmin):
list_select_related = True
@admin.register(models.DomainFollow)
class DomainFollowAdmin(admin.ModelAdmin):
list_display = ["actor", "target", "approved", "creation_date"]
list_filter = ["approved"]
search_fields = ["actor__fid", "target__name"]
list_select_related = True
@admin.register(models.InboxItem)
class InboxItemAdmin(admin.ModelAdmin):
list_display = ["actor", "activity", "type", "is_read"]
......
......@@ -12,10 +12,15 @@ from funkwhale_api.common import fields as common_fields
from funkwhale_api.common import serializers as common_serializers
from funkwhale_api.music import models as music_models
from funkwhale_api.users import serializers as users_serializers
from funkwhale_api.federation import models as federation_models
from . import filters
from . import models
from . import serializers as federation_serializers
from . import tasks
import logging
logger = logging.getLogger(__name__)
class NestedLibraryFollowSerializer(serializers.ModelSerializer):
......@@ -24,6 +29,17 @@ class NestedLibraryFollowSerializer(serializers.ModelSerializer):
fields = ["creation_date", "uuid", "fid", "approved", "modification_date"]
class NestedDomainFollowSerializer(serializers.ModelSerializer):
class Meta:
model = models.DomainFollow
fields = [
"creation_date",
"fid",
"approved",
"modification_date",
]
class LibraryScanSerializer(serializers.ModelSerializer):
class Meta:
model = music_models.LibraryScan
......@@ -39,6 +55,33 @@ class LibraryScanSerializer(serializers.ModelSerializer):
class DomainSerializer(serializers.Serializer):
name = serializers.CharField()
follow = serializers.SerializerMethodField()
class Meta:
model = federation_models.Domain
fields = [
"fid",
"name",
"nodeinfo",
]
def get_follow(self, o):
try:
return NestedDomainFollowSerializer(o._follows[0]).data
except (AttributeError, IndexError):
return None
class LibraryFollowListSerializer(serializers.ListSerializer):
def create(self, validated_data):
libs = [models.LibraryFollow(**lib) for lib in validated_data]
return models.LibraryFollow.objects.bulk_create(libs, batch_size=1000)
class LibraryListSerializer(serializers.ListSerializer):
def create(self, validated_data):
libs = [models.Library(**lib) for lib in validated_data]
return models.Library.objects.bulk_create(libs, batch_size=1000)
class LibrarySerializer(serializers.ModelSerializer):
......@@ -61,6 +104,7 @@ class LibrarySerializer(serializers.ModelSerializer):
"follow",
"latest_scan",
]
list_serializer_class = LibraryListSerializer
def get_uploads_count(self, o):
return max(getattr(o, "_uploads_count", 0), o.uploads_count)
......@@ -80,14 +124,28 @@ class LibrarySerializer(serializers.ModelSerializer):
class LibraryFollowSerializer(serializers.ModelSerializer):
target = common_serializers.RelatedField("uuid", LibrarySerializer(), required=True)
actor = serializers.SerializerMethodField()
follow_context = serializers.ChoiceField(
choices=["Domain", "Library"], required=False
)
class Meta:
model = models.LibraryFollow
fields = ["creation_date", "actor", "uuid", "target", "approved"]
fields = [
"creation_date",
"actor",
"uuid",
"target",
"approved",
"follow_context",
]
read_only_fields = ["uuid", "actor", "approved", "creation_date"]
list_serializer_class = LibraryFollowListSerializer
def validate_target(self, v):
actor = self.context["actor"]
try:
actor = self.context["actor"]
except KeyError:
raise KeyError
if v.actor == actor:
raise serializers.ValidationError("You cannot follow your own library")
......@@ -99,16 +157,54 @@ class LibraryFollowSerializer(serializers.ModelSerializer):
return federation_serializers.APIActorSerializer(o.actor).data
class DomainFollowSerializer(serializers.ModelSerializer):
target = common_serializers.RelatedField("name", DomainSerializer(), required=True)
actor = serializers.SerializerMethodField()
class Meta:
model = models.DomainFollow
fields = ["uuid", "creation_date", "actor", "target", "approved"]
read_only_fields = ["uuid", "actor", "approved", "creation_date"]
def validate_target(self, v):
actor = self.context["actor"]
if v.domainfollow_received_follows.filter(actor=actor).exists():
raise serializers.ValidationError("You are already following this Domain")
domain = models.Domain.objects.get(name=v.name)
now = datetime.datetime.now(timezone.utc)
if (
domain.nodeinfo_fetch_date - now
).total_seconds() > 86400 or "error" in domain.nodeinfo:
tasks.refresh_actor_data()
tasks.update_domain_nodeinfo(domain_name=domain)
domain = models.Domain.objects.get(name=v.name)
if "error" in domain.nodeinfo:
e = domain.nodeinfo["error"]
logger.error(f"Could not get domain information because of : {e!r}")
raise serializers.ValidationError(domain.nodeinfo["error"])
return domain
def get_actor(self, o):
logger.info("o is :" + str(o))
return federation_serializers.APIActorSerializer(o.actor).data
def serialize_generic_relation(activity, obj):
data = {"type": obj._meta.label}
logger.info("data[type]" + str(data["type"]))
if data["type"] == "federation.Actor":
data["full_username"] = obj.full_username
elif data["type"] == "federation.Domain":
data["name"] = obj.name
else:
data["uuid"] = obj.uuid
if data["type"] == "music.Library":
data["name"] = obj.name
if data["type"] == "federation.LibraryFollow":
# to do : test if this is works
if (
data["type"] == "federation.LibraryFollow"
or data["type"] == "federation.DomainFollow"
):
data["approved"] = obj.approved
return data
......
......@@ -5,6 +5,7 @@ from . import api_views
router = routers.OptionalSlashRouter()
router.register(r"fetches", api_views.FetchViewSet, "fetches")
router.register(r"follows/library", api_views.LibraryFollowViewSet, "library-follows")
router.register(r"follows/domain", api_views.DomainFollowViewSet, "domain-follows")
router.register(r"inbox", api_views.InboxItemViewSet, "inbox")
router.register(r"libraries", api_views.LibraryViewSet, "libraries")
router.register(r"domains", api_views.DomainViewSet, "domains")
......
from urllib import request
import requests.exceptions
import requests
from django.conf import settings
from django.db import transaction
......@@ -11,6 +13,7 @@ from rest_framework import response
from rest_framework import viewsets
from funkwhale_api.common import preferences
from funkwhale_api.common import permissions as common_permissions
from funkwhale_api.common import utils as common_utils
from funkwhale_api.common.permissions import ConditionalAuthentication
from funkwhale_api.music import models as music_models
......@@ -27,6 +30,10 @@ from . import serializers
from . import tasks
from . import utils
import logging
logger = logging.getLogger(__name__)
@transaction.atomic
def update_follow(follow, approved):
......@@ -263,6 +270,93 @@ class DomainViewSet(
return qs
class DomainFollowViewSet(
mixins.CreateModelMixin,
mixins.ListModelMixin,
mixins.RetrieveModelMixin,
mixins.DestroyModelMixin,
viewsets.GenericViewSet,
):
lookup_field = "uuid"
queryset = (
models.DomainFollow.objects.all()
.order_by("-creation_date")
.select_related("actor")
)
serializer_class = api_serializers.DomainFollowSerializer
permission_classes = [oauth_permissions.ScopePermission]
required_scope = "follows"
filterset_class = filters.DomainFollowFilter
ordering_fields = ("creation_date",)
def get_queryset(self):
qs = super().get_queryset()
return qs.filter(actor=self.request.user.actor).exclude(approved=False)
def perform_create(self, serializer):
follow = serializer.save(actor=self.request.user.actor)
routes.outbox.dispatch({"type": "Follow"}, context={"follow": follow})
@transaction.atomic
def perform_destroy(self, instance):
routes.outbox.dispatch(
{"type": "Undo", "object": {"type": "Follow"}},
context={"follow": instance},
)
tasks.domain_bulk_unfollow_libraries.delay(
instance.target.name, actor_id=self.request.user.actor.pk
)
instance.delete()
def get_serializer_context(self):
context = super().get_serializer_context()
context["actor"] = self.request.user.actor
return context
@decorators.action(methods=["post"], detail=True)
def accept(self, request, *args, **kwargs):
try:
follow = self.queryset.get(
target__actor=self.request.user.actor, uuid=kwargs["uuid"]
)
except models.DomainFollow.DoesNotExist:
return response.Response({}, status=404)
update_follow(follow, approved=True)
return response.Response(status=204)
@decorators.action(methods=["post"], detail=True)
def reject(self, request, *args, **kwargs):
try:
follow = self.queryset.get(
target__actor=self.request.user.actor, uuid=kwargs["uuid"]
)
except models.DomainFollow.DoesNotExist:
return response.Response({}, status=404)
update_follow(follow, approved=False)
return response.Response(status=204)
@decorators.action(methods=["get"], detail=False)
def all(self, request, *args, **kwargs):
"""
Return all the subscriptions of the current user, with only limited data
to have a performant endpoint and avoid lots of queries just to display
subscription status in the UI
"""
follows = list(
self.get_queryset().values_list("uuid", "target__name", "approved")
)
payload = {
"results": [
{"uuid": str(u[0]), "domain": str(u[1]), "approved": u[2]}
for u in follows
],
"count": len(follows),
}
return response.Response(payload, status=200)
class ActorViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet):
queryset = models.Actor.objects.select_related(
"user", "channel", "summary_obj", "attachment_icon"
......
import uuid
import factory
# see https://stackoverflow.com/questions/60914361/python-import-error-module-factory-has-no-attribute-fuzzy
from factory import fuzzy
import requests
import requests_http_message_signatures
from django.conf import settings
......@@ -238,11 +241,23 @@ class DeliveryFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
class LibraryFollowFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
target = factory.SubFactory(MusicLibraryFactory)
actor = factory.SubFactory(ActorFactory)
follow_context = fuzzy.FuzzyChoice(
models.LibraryFollow.FOLLOW_CONTEXT_CHOICES, getter=lambda c: c[0]
)
class Meta:
model = "federation.LibraryFollow"
@registry.register
class DomainollowFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
target = factory.SubFactory(DomainFactory)
actor = factory.SubFactory(ActorFactory)
class Meta:
model = "federation.DomainFollow"
class ArtistMetadataFactory(factory.Factory):
name = factory.Faker("name")
......
......@@ -34,6 +34,12 @@ class LibraryFollowFilter(django_filters.FilterSet):
fields = ["approved"]
class DomainFollowFilter(django_filters.FilterSet):
class Meta:
model = models.DomainFollow
fields = ["approved"]
class InboxItemFilter(django_filters.FilterSet):
is_read = django_filters.BooleanFilter(
"is_read", widget=django_filters.widgets.BooleanWidget()
......
# Generated by Django 3.2.13 on 2022-05-27 20:06
import django.core.serializers.json
from django.db import migrations, models
import django.db.models.deletion
import django.utils.timezone
import funkwhale_api.federation.models
import uuid