api/funkwhale_api/cli/main.py

+import sys
+
+import click
+from rest_framework.exceptions import ValidationError
+
+from . import library  # noqa
+from . import media  # noqa
+from . import plugins  # noqa
+from . import users  # noqa
+from . import base
+
+
+def invoke():
+    try:
+        return base.cli()
+    except ValidationError as e:
+        click.secho("Invalid data:", fg="red")
+        for field, errors in e.detail.items():
+            click.secho(f"  {field}:", fg="red")
+            for error in errors:
+                click.secho(f"    - {error}", fg="red")
+        sys.exit(1)

api/funkwhale_api/cli/media.py

+import click
+from django.conf import settings
+from django.core.cache import cache
+from django.core.files.storage import default_storage
+from versatileimagefield import settings as vif_settings
+from versatileimagefield.image_warmer import VersatileImageFieldWarmer
+
+from funkwhale_api.common import utils as common_utils
+from funkwhale_api.common.models import Attachment
+
+from . import base
+
+
+@base.cli.group()
+def media():
+    """Manage media files (avatars, covers, attachments…)"""
+    pass
+
+
+@media.command("generate-thumbnails")
+@click.option("-d", "--delete", is_flag=True)
+def generate_thumbnails(delete):
+    """
+    Generate thumbnails for all images (avatars, covers, etc.).
+
+    This can take a long time and generate a lot of I/O depending of the size
+    of your library.
+    """
+    click.echo("Deleting existing thumbnails…")
+    if delete:
+        try:
+            # FileSystemStorage doesn't support deleting a non-empty directory
+            # so we reimplemented a method to do so
+            default_storage.force_delete("__sized__")
+        except AttributeError:
+            # backends doesn't support directory deletion
+            pass
+    MODELS = [
+        (Attachment, "file", "attachment_square"),
+    ]
+    for model, attribute, key_set in MODELS:
+        click.echo(f"Generating thumbnails for {model._meta.label}.{attribute}…")
+        qs = model.objects.exclude(**{f"{attribute}__isnull": True})
+        qs = qs.exclude(**{attribute: ""})
+        cache_key = "*{}{}*".format(
+            settings.MEDIA_URL, vif_settings.VERSATILEIMAGEFIELD_SIZED_DIRNAME
+        )
+        entries = cache.keys(cache_key)
+        if entries:
+            click.echo(f"  Clearing {len(entries)} cache entries: {cache_key}…")
+            for keys in common_utils.batch(iter(entries)):
+                cache.delete_many(keys)
+        warmer = VersatileImageFieldWarmer(
+            instance_or_queryset=qs,
+            rendition_key_set=key_set,
+            image_attr=attribute,
+            verbose=True,
+        )
+        click.echo("  Creating images")
+        num_created, failed_to_create = warmer.warm()
+        click.echo(f"  {num_created} created, {len(failed_to_create)} in error")

api/funkwhale_api/cli/plugins.py

+import os
+import subprocess
+import sys
+
+import click
+from django.conf import settings
+
+from . import base
+
+
+@base.cli.group()
+def plugins():
+    """Manage plugins"""
+    pass
+
+
+@plugins.command("install")
+@click.argument("plugin", nargs=-1)
+def install(plugin):
+    """
+    Install a plugin from a given URL (zip, pip or git are supported)
+    """
+    if not plugin:
+        return click.echo("No plugin provided")
+
+    click.echo("Installing plugins…")
+    pip_install(list(plugin), settings.FUNKWHALE_PLUGINS_PATH)
+
+
+def pip_install(deps, target):
+    if not deps:
+        return
+    pip_path = os.path.join(os.path.dirname(sys.executable), "pip")
+    subprocess.check_call([pip_path, "install", "-t", target] + deps)

api/funkwhale_api/cli/users.py

+import click
+from django.db import transaction
+
+from funkwhale_api.federation import models as federation_models
+from funkwhale_api.users import models, serializers, tasks
+
+from . import base, utils
+
+
+class FakeRequest:
+    def __init__(self, session={}):
+        self.session = session
+
+
+@transaction.atomic
+def handler_create_user(
+    username,
+    password,
+    email,
+    is_superuser=False,
+    is_staff=False,
+    permissions=[],
+    upload_quota=None,
+):
+    serializer = serializers.RS(
+        data={
+            "username": username,
+            "email": email,
+            "password1": password,
+            "password2": password,
+        }
+    )
+    utils.logger.debug("Validating user data…")
+    serializer.is_valid(raise_exception=True)
+
+    # Override e-mail validation, we assume accounts created from CLI have a valid e-mail
+    request = FakeRequest(session={"account_verified_email": email})
+    utils.logger.debug("Creating user…")
+    user = serializer.save(request=request)
+    utils.logger.debug("Setting permissions and other attributes…")
+    user.is_staff = is_staff or is_superuser  # Always set staff if superuser is set
+    user.upload_quota = upload_quota
+    user.is_superuser = is_superuser
+    for permission in permissions:
+        if permission in models.PERMISSIONS:
+            utils.logger.debug("Setting %s permission to True", permission)
+            setattr(user, f"permission_{permission}", True)
+        else:
+            utils.logger.warn("Unknown permission %s", permission)
+    utils.logger.debug("Creating actor…")
+    user.actor = models.create_actor(user)
+    models.create_user_libraries(user)
+    user.save()
+    return user
+
+
+@transaction.atomic
+def handler_delete_user(usernames, soft=True):
+    for username in usernames:
+        click.echo(f"Deleting {username}…")
+        actor = None
+        user = None
+        try:
+            user = models.User.objects.get(username=username)
+        except models.User.DoesNotExist:
+            try:
+                actor = federation_models.Actor.objects.local().get(
+                    preferred_username=username
+                )
+            except federation_models.Actor.DoesNotExist:
+                click.echo("  Not found, skipping")
+                continue
+
+        actor = actor or user.actor
+        if user:
+            tasks.delete_account(user_id=user.pk)
+        if not soft:
+            click.echo("  Hard delete, removing actor")
+            actor.delete()
+        click.echo("  Done")
+
+
+@transaction.atomic
+def handler_update_user(usernames, kwargs):
+    users = models.User.objects.filter(username__in=usernames)
+    total = users.count()
+    if not total:
+        click.echo("No matching users")
+        return
+
+    final_kwargs = {}
+    supported_fields = [
+        "is_active",
+        "permission_moderation",
+        "permission_library",
+        "permission_settings",
+        "is_staff",
+        "is_superuser",
+        "upload_quota",
+        "password",
+    ]
+    for field in supported_fields:
+        try:
+            value = kwargs[field]
+        except KeyError:
+            continue
+        final_kwargs[field] = value
+
+    click.echo(
+        "Updating {} on {} matching users…".format(
+            ", ".join(final_kwargs.keys()), total
+        )
+    )
+    if "password" in final_kwargs:
+        new_password = final_kwargs.pop("password")
+        for user in users:
+            user.set_password(new_password)
+        models.User.objects.bulk_update(users, ["password"])
+    if final_kwargs:
+        users.update(**final_kwargs)
+    click.echo("Done!")
+
+
+@base.cli.group()
+def users():
+    """Manage users"""
+    pass
+
+
+@users.command()
+@click.option("--username", "-u", prompt=True, required=True)
+@click.option(
+    "-p",
+    "--password",
+    prompt="Password (leave empty to have a random one generated)",
+    hide_input=True,
+    envvar="FUNKWHALE_CLI_USER_PASSWORD",
+    default="",
+    help="If empty, a random password will be generated and displayed in console output",
+)
+@click.option(
+    "-e",
+    "--email",
+    prompt=True,
+    help="Email address to associate with the account",
+    required=True,
+)
+@click.option(
+    "-q",
+    "--upload-quota",
+    help="Upload quota (leave empty to use default pod quota)",
+    required=False,
+    default=None,
+    type=click.INT,
+)
+@click.option(
+    "--superuser/--no-superuser",
+    default=False,
+)
+@click.option(
+    "--staff/--no-staff",
+    default=False,
+)
+@click.option(
+    "--permission",
+    multiple=True,
+)
+def create(username, password, email, superuser, staff, permission, upload_quota):
+    """Create a new user"""
+    generated_password = None
+    if password == "":
+        generated_password = models.User.objects.make_random_password()
+    user = handler_create_user(
+        username=username,
+        password=password or generated_password,
+        email=email,
+        is_superuser=superuser,
+        is_staff=staff,
+        permissions=permission,
+        upload_quota=upload_quota,
+    )
+    click.echo(f"User {user.username} created!")
+    if generated_password:
+        click.echo(f"  Generated password: {generated_password}")
+
+
+@base.delete_command(group=users, id_var="username")
+@click.argument("username", nargs=-1)
+@click.option(
+    "--hard/--no-hard",
+    default=False,
+    help="Purge all user-related info (allow recreating a user with the same username)",
+)
+def delete(username, hard):
+    """Delete given users"""
+    handler_delete_user(usernames=username, soft=not hard)
+
+
+@base.update_command(group=users, id_var="username")
+@click.argument("username", nargs=-1)
+@click.option(
+    "--active/--inactive",
+    help="Mark as active or inactive (inactive users cannot login or use the service)",
+    default=None,
+)
+@click.option("--superuser/--no-superuser", default=None)
+@click.option("--staff/--no-staff", default=None)
+@click.option("--permission-library/--no-permission-library", default=None)
+@click.option("--permission-moderation/--no-permission-moderation", default=None)
+@click.option("--permission-settings/--no-permission-settings", default=None)
+@click.option("--password", default=None, envvar="FUNKWHALE_CLI_USER_UPDATE_PASSWORD")
+@click.option(
+    "-q",
+    "--upload-quota",
+    type=click.INT,
+)
+def update(username, **kwargs):
+    """Update attributes for given users"""
+    field_mapping = {
+        "active": "is_active",
+        "superuser": "is_superuser",
+        "staff": "is_staff",
+    }
+    final_kwargs = {}
+    for cli_field, value in kwargs.items():
+        if value is None:
+            continue
+        model_field = (
+            field_mapping[cli_field] if cli_field in field_mapping else cli_field
+        )
+        final_kwargs[model_field] = value
+
+    if not final_kwargs:
+        raise click.BadArgumentUsage("You need to update at least one attribute")
+
+    handler_update_user(usernames=username, kwargs=final_kwargs)

api/funkwhale_api/cli/utils.py

+import logging
+
+logger = logging.getLogger("funkwhale_api.cli")

api/funkwhale_api/common/admin.py

+from django.contrib.admin import site  # noqa: F401
+from django.contrib.admin import ModelAdmin
+from django.contrib.admin import register as initial_register
+from django.db.models.fields.related import RelatedField
+
+from . import models, tasks
+
+
+def register(model):
+    """
+    To make the admin more performant, we ensure all the the relations
+    are listed under raw_id_fields
+    """
+
+    def decorator(modeladmin):
+        raw_id_fields = []
+        for field in model._meta.fields:
+            if isinstance(field, RelatedField):
+                raw_id_fields.append(field.name)
+        setattr(modeladmin, "raw_id_fields", raw_id_fields)
+        return initial_register(model)(modeladmin)
+
+    return decorator
+
+
+def apply(modeladmin, request, queryset):
+    queryset.update(is_approved=True)
+    for id in queryset.values_list("id", flat=True):
+        tasks.apply_mutation.delay(mutation_id=id)
+
+
+apply.short_description = "Approve and apply"
+
+
+@register(models.Mutation)
+class MutationAdmin(ModelAdmin):
+    list_display = [
+        "uuid",
+        "type",
+        "created_by",
+        "creation_date",
+        "applied_date",
+        "is_approved",
+        "is_applied",
+    ]
+    search_fields = ["created_by__preferred_username"]
+    list_filter = ["type", "is_approved", "is_applied"]
+    actions = [apply]
+
+
+@register(models.Attachment)
+class AttachmentAdmin(ModelAdmin):
+    list_display = [
+        "uuid",
+        "actor",
+        "url",
+        "file",
+        "size",
+        "mimetype",
+        "creation_date",
+        "last_fetch_date",
+    ]
+    list_select_related = True
+    search_fields = ["actor__domain__name"]
+    list_filter = ["mimetype"]

api/funkwhale_api/common/apps.py

+from django.apps import AppConfig, apps
+from django.conf import settings
+
+from config import plugins
+
+from . import mutations, utils
+
+
+class CommonConfig(AppConfig):
+    name = "funkwhale_api.common"
+
+    def ready(self):
+        super().ready()
+
+        app_names = [app.name for app in apps.app_configs.values()]
+        mutations.registry.autodiscover(app_names)
+        utils.monkey_patch_request_build_absolute_uri()
+        plugins.startup.autodiscover([p + ".funkwhale_ready" for p in settings.PLUGINS])
+        for p in plugins._plugins.values():
+            p["settings"] = plugins.load_settings(p["name"], p["settings"])

api/funkwhale_api/common/auth.py

-from urllib.parse import parse_qs
-
-from django.contrib.auth.models import AnonymousUser
-from rest_framework import exceptions
-from rest_framework_jwt.authentication import BaseJSONWebTokenAuthentication
-
-from funkwhale_api.users.models import User
-
-
-class TokenHeaderAuth(BaseJSONWebTokenAuthentication):
-    def get_jwt_value(self, request):
-
-        try:
-            qs = request.get("query_string", b"").decode("utf-8")
-            parsed = parse_qs(qs)
-            token = parsed["token"][0]
-        except KeyError:
-            raise exceptions.AuthenticationFailed("No token")
-
-        if not token:
-            raise exceptions.AuthenticationFailed("Empty token")
-
-        return token
-
-
-class TokenAuthMiddleware:
-    def __init__(self, inner):
-        # Store the ASGI application we were passed
-        self.inner = inner
-
-    def __call__(self, scope):
-        auth = TokenHeaderAuth()
-        try:
-            user, token = auth.authenticate(scope)
-        except (User.DoesNotExist, exceptions.AuthenticationFailed):
-            user = AnonymousUser()
-
-        scope["user"] = user
-        return self.inner(scope)

api/funkwhale_api/common/authentication.py

-from django.utils.encoding import smart_text
-from django.utils.translation import ugettext as _
+from allauth.account.models import EmailAddress
+from django.core.cache import cache
+from django.utils.translation import gettext as _
+from oauth2_provider.contrib.rest_framework.authentication import (
+    OAuth2Authentication as BaseOAuth2Authentication,
+)
 from rest_framework import exceptions
-from rest_framework_jwt import authentication
-from rest_framework_jwt.settings import api_settings
 
+from funkwhale_api.users import models as users_models
 
-class JSONWebTokenAuthenticationQS(authentication.BaseJSONWebTokenAuthentication):
 
-    www_authenticate_realm = "api"
+class UnverifiedEmail(Exception):
+    def __init__(self, user):
+        self.user = user
 
-    def get_jwt_value(self, request):
-        token = request.query_params.get("jwt")
-        if "jwt" in request.query_params and not token:
-            msg = _("Invalid Authorization header. No credentials provided.")
-            raise exceptions.AuthenticationFailed(msg)
-        return token
 
-    def authenticate_header(self, request):
-        return '{0} realm="{1}"'.format(
-            api_settings.JWT_AUTH_HEADER_PREFIX, self.www_authenticate_realm
-        )
+def resend_confirmation_email(request, user):
+    THROTTLE_DELAY = 500
+    cache_key = f"auth:resent-email-confirmation:{user.pk}"
+    if cache.get(cache_key):
+        return False
 
+    # We do the sending of the conformation by hand because we don't want to pass the request down
+    # to the email rendering, which would cause another UnverifiedEmail Exception and restarts the sending
+    # again and again
+    email = EmailAddress.objects.get_for_user(user, user.email)
+    email.send_confirmation()
+    cache.set(cache_key, True, THROTTLE_DELAY)
+    return True
 
-class BearerTokenHeaderAuth(authentication.BaseJSONWebTokenAuthentication):
-    """
-    For backward compatibility purpose, we used Authorization: JWT <token>
-    but Authorization: Bearer <token> is probably better.
-    """
 
-    www_authenticate_realm = "api"
+class OAuth2Authentication(BaseOAuth2Authentication):
+    def authenticate(self, request):
+        try:
+            return super().authenticate(request)
+        except UnverifiedEmail as e:
+            request.oauth2_error = {"error": "unverified_email"}
+            resend_confirmation_email(request, e.user)
 
-    def get_jwt_value(self, request):
-        auth = authentication.get_authorization_header(request).split()
-        auth_header_prefix = "bearer"
 
-        if not auth:
-            if api_settings.JWT_AUTH_COOKIE:
-                return request.COOKIES.get(api_settings.JWT_AUTH_COOKIE)
-            return None
+class ApplicationTokenAuthentication:
+    def authenticate(self, request):
+        try:
+            header = request.headers["Authorization"]
+        except KeyError:
+            return
 
-        if smart_text(auth[0].lower()) != auth_header_prefix:
-            return None
+        if "Bearer" not in header:
+            return
 
-        if len(auth) == 1:
-            msg = _("Invalid Authorization header. No credentials provided.")
-            raise exceptions.AuthenticationFailed(msg)
-        elif len(auth) > 2:
-            msg = _(
-                "Invalid Authorization header. Credentials string "
-                "should not contain spaces."
+        token = header.split()[-1].strip()
+
+        try:
+            application = users_models.Application.objects.exclude(user=None).get(
+                token=token
             )
+        except users_models.Application.DoesNotExist:
+            return
+        user = users_models.User.objects.all().for_auth().get(id=application.user_id)
+        if not user.is_active:
+            msg = _("User account is disabled.")
             raise exceptions.AuthenticationFailed(msg)
 
-        return auth[1]
+        if user.should_verify_email():
+            raise UnverifiedEmail(user)
 
-    def authenticate_header(self, request):
-        return '{0} realm="{1}"'.format("Bearer", self.www_authenticate_realm)
+        request.scopes = application.scope.split()
+        return user, None

api/funkwhale_api/common/cache.py

+import logging
+
+from django_redis.client import default
+
+logger = logging.getLogger(__name__)
+
+
+class RedisClient(default.DefaultClient):
+    def get(self, key, default=None, version=None, client=None):
+        try:
+            return super().get(key, default=default, version=version, client=client)
+        except ValueError as e:
+            if "unsupported pickle protocol" in str(e):
+                # pickle deserialization error
+                logger.warn("Error while deserializing pickle value from cache")
+                return default
+            else:
+                raise
+
+    def get_many(self, *args, **kwargs):
+        try:
+            return super().get_many(*args, **kwargs)
+        except ValueError as e:
+            if "unsupported pickle protocol" in str(e):
+                # pickle deserialization error
+                logger.warn("Error while deserializing pickle value from cache")
+                return {}
+            else:
+                raise

api/funkwhale_api/common/channels.py

+import json
+import logging
+
 from asgiref.sync import async_to_sync
 from channels.layers import get_channel_layer
+from django.core.serializers.json import DjangoJSONEncoder
 
+logger = logging.getLogger(__name__)
 channel_layer = get_channel_layer()
-group_send = async_to_sync(channel_layer.group_send)
 group_add = async_to_sync(channel_layer.group_add)
+group_discard = async_to_sync(channel_layer.group_discard)
+
+
+def group_send(group, event):
+    # we serialize the payload ourselves and deserialize it to ensure it
+    # works with msgpack. This is dirty, but we'll find a better solution
+    # later
+    s = json.dumps(event, cls=DjangoJSONEncoder)
+    event = json.loads(s)
+    logger.debug(
+        "[channels] Dispatching %s to group %s: %s",
+        event["type"],
+        group,
+        {"type": event["data"]["type"]},
+    )
+    async_to_sync(channel_layer.group_send)(group, event)

api/funkwhale_api/common/consumers.py

@@ -14,5 +14,15 @@ class JsonAuthConsumer(JsonWebsocketConsumer):
 
     def accept(self):
         super().accept()
-        for group in self.groups:
+        groups = self.scope["user"].get_channels_groups() + self.groups
+        for group in groups:
             channels.group_add(group, self.channel_name)
+
+    def disconnect(self, close_code):
+        if self.scope.get("user", False) and self.scope.get("user").pk is not None:
+            groups = self.scope["user"].get_channels_groups() + self.groups
+        else:
+            groups = self.groups
+
+        for group in groups:
+            channels.group_discard(group, self.channel_name)

api/funkwhale_api/common/decorators.py

+from django.db import transaction
+from drf_spectacular.utils import OpenApiParameter, extend_schema
+from rest_framework import decorators, exceptions, response, status
+
+from . import filters, models
+from . import mutations as common_mutations
+from . import serializers, signals, tasks, utils
+
+
+def action_route(serializer_class):
+    @decorators.action(methods=["post"], detail=False)
+    def action(self, request, *args, **kwargs):
+        queryset = self.get_queryset()
+        serializer = serializer_class(request.data, queryset=queryset)
+        serializer.is_valid(raise_exception=True)
+        result = serializer.save()
+        return response.Response(result, status=200)
+
+    return action
+
+
+def mutations_route(types):
+    """
+    Given a queryset and a list of mutation types, return a view
+    that can be included in any viewset, and serve:
+
+    GET /{id}/mutations/ - list of mutations for the given object
+    POST /{id}/mutations/ - create a mutation for the given object
+    """
+
+    @transaction.atomic
+    def mutations(self, request, *args, **kwargs):
+        obj = self.get_object()
+        if request.method == "GET":
+            queryset = models.Mutation.objects.get_for_target(obj).filter(
+                type__in=types
+            )
+            queryset = queryset.order_by("-creation_date")
+            filterset = filters.MutationFilter(request.GET, queryset=queryset)
+            page = self.paginate_queryset(filterset.qs)
+            if page is not None:
+                serializer = serializers.APIMutationSerializer(page, many=True)
+                return self.get_paginated_response(serializer.data)
+
+            serializer = serializers.APIMutationSerializer(queryset, many=True)
+            return response.Response(serializer.data)
+        if request.method == "POST":
+            if not request.user.is_authenticated:
+                raise exceptions.NotAuthenticated()
+            serializer = serializers.APIMutationSerializer(
+                data=request.data, context={"registry": common_mutations.registry}
+            )
+            serializer.is_valid(raise_exception=True)
+            if not common_mutations.registry.has_perm(
+                actor=request.user.actor,
+                type=serializer.validated_data["type"],
+                obj=obj,
+                perm="approve"
+                if serializer.validated_data.get("is_approved", False)
+                else "suggest",
+            ):
+                raise exceptions.PermissionDenied()
+
+            final_payload = common_mutations.registry.get_validated_payload(
+                type=serializer.validated_data["type"],
+                payload=serializer.validated_data["payload"],
+                obj=obj,
+            )
+            mutation = serializer.save(
+                created_by=request.user.actor,
+                target=obj,
+                payload=final_payload,
+                is_approved=serializer.validated_data.get("is_approved", None),
+            )
+            if mutation.is_approved:
+                utils.on_commit(tasks.apply_mutation.delay, mutation_id=mutation.pk)
+
+            utils.on_commit(
+                signals.mutation_created.send, sender=None, mutation=mutation
+            )
+            return response.Response(serializer.data, status=status.HTTP_201_CREATED)
+
+    return extend_schema(
+        methods=["post"], responses=serializers.APIMutationSerializer()
+    )(
+        extend_schema(
+            methods=["get"],
+            responses=serializers.APIMutationSerializer(many=True),
+            parameters=[OpenApiParameter("id", location="query", exclude=True)],
+        )(
+            decorators.action(
+                methods=["get", "post"], detail=True, required_scope="edits"
+            )(mutations)
+        )
+    )

api/funkwhale_api/common/dynamic_preferences_registry.py

 from dynamic_preferences import types
 from dynamic_preferences.registries import global_preferences_registry
 
-from funkwhale_api.common import preferences
-
 common = types.Section("common")
 
 
 @global_preferences_registry.register
-class APIAutenticationRequired(
-    preferences.DefaultFromSettingMixin, types.BooleanPreference
-):
+class APIAutenticationRequired(types.BooleanPreference):
     section = common
     name = "api_authentication_required"
     verbose_name = "API Requires authentication"
-    setting = "API_AUTHENTICATION_REQUIRED"
+    default = True
     help_text = (
         "If disabled, anonymous users will be able to query the API "
         "and access music data (as well as other data exposed in the API "

api/funkwhale_api/common/factories.py

+import factory
+
+from funkwhale_api.factories import NoUpdateOnCreate, registry
+from funkwhale_api.federation import factories as federation_factories
+
+
+@registry.register
+class MutationFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
+    fid = factory.Faker("federation_url")
+    uuid = factory.Faker("uuid4")
+    created_by = factory.SubFactory(federation_factories.ActorFactory)
+    summary = factory.Faker("paragraph")
+    type = "update"
+
+    class Meta:
+        model = "common.Mutation"
+
+
+@registry.register
+class AttachmentFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
+    url = factory.Faker("federation_url")
+    uuid = factory.Faker("uuid4")
+    actor = factory.SubFactory(federation_factories.ActorFactory)
+    file = factory.django.ImageField()
+
+    class Meta:
+        model = "common.Attachment"
+
+
+@registry.register
+class CommonFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
+    text = factory.Faker("paragraph")
+    content_type = "text/plain"
+
+    class Meta:
+        model = "common.Content"
+
+
+@registry.register
+class PluginConfiguration(NoUpdateOnCreate, factory.django.DjangoModelFactory):
+    code = "test"
+    conf = {"foo": "bar"}
+
+    class Meta:
+        model = "common.PluginConfiguration"

api/funkwhale_api/common/fields.py

 import django_filters
+from django import forms
+from django.core.serializers.json import DjangoJSONEncoder
 from django.db import models
+from rest_framework import serializers
 
 from . import search
 
@@ -21,19 +24,38 @@ def privacy_level_query(user, lookup_field="privacy_level", user_field="user"):
     if user.is_anonymous:
         return models.Q(**{lookup_field: "everyone"})
 
-    return models.Q(
-        **{"{}__in".format(lookup_field): ["instance", "everyone"]}
-    ) | models.Q(**{lookup_field: "me", user_field: user})
+    followers_query = models.Q(
+        **{
+            f"{lookup_field}": "followers",
+            f"{user_field}__actor__in": user.actor.get_approved_followings(),
+        }
+    )
+    # Federated TrackFavorite don't have an user associated with the trackfavorite.actor
+    # to do : if we implement the followers privacy_level this will become a problem
+    no_user_query = models.Q(**{f"{user_field}__isnull": True})
+
+    return (
+        models.Q(**{f"{lookup_field}__in": ["instance", "everyone"]})
+        | models.Q(**{lookup_field: "me", user_field: user})
+        | followers_query
+        | no_user_query
+    )
 
 
 class SearchFilter(django_filters.CharFilter):
     def __init__(self, *args, **kwargs):
         self.search_fields = kwargs.pop("search_fields")
+        self.fts_search_fields = kwargs.pop("fts_search_fields", [])
         super().__init__(*args, **kwargs)
 
     def filter(self, qs, value):
         if not value:
             return qs
+        if self.fts_search_fields:
+            query = search.get_fts_query(
+                value, self.fts_search_fields, model=self.parent.Meta.model
+            )
+        else:
             query = search.get_query(value, self.search_fields)
         return qs.filter(query)
 
@@ -46,5 +68,123 @@ class SmartSearchFilter(django_filters.CharFilter):
     def filter(self, qs, value):
         if not value:
             return qs
+        try:
             cleaned = self.config.clean(value)
+        except forms.ValidationError:
+            return qs.none()
         return search.apply(qs, cleaned)
+
+
+def get_generic_filter_query(value, relation_name, choices):
+    parts = value.split(":", 1)
+    type = parts[0]
+    try:
+        conf = choices[type]
+    except KeyError:
+        raise forms.ValidationError("Invalid type")
+    related_queryset = conf["queryset"]
+    related_model = related_queryset.model
+    filter_query = models.Q(
+        **{
+            "{}_content_type__app_label".format(
+                relation_name
+            ): related_model._meta.app_label,
+            "{}_content_type__model".format(
+                relation_name
+            ): related_model._meta.model_name,
+        }
+    )
+    if len(parts) > 1:
+        id_attr = conf.get("id_attr", "id")
+        id_field = conf.get("id_field", serializers.IntegerField(min_value=1))
+        try:
+            id_value = parts[1]
+            id_value = id_field.to_internal_value(id_value)
+        except (TypeError, KeyError, serializers.ValidationError):
+            raise forms.ValidationError("Invalid id")
+        query_getter = conf.get(
+            "get_query", lambda attr, value: models.Q(**{attr: value})
+        )
+        obj_query = query_getter(id_attr, id_value)
+        try:
+            obj = related_queryset.get(obj_query)
+        except related_queryset.model.DoesNotExist:
+            raise forms.ValidationError("Invalid object")
+        filter_query &= models.Q(**{f"{relation_name}_id": obj.id})
+
+    return filter_query
+
+
+class GenericRelationFilter(django_filters.CharFilter):
+    def __init__(self, relation_name, choices, *args, **kwargs):
+        self.relation_name = relation_name
+        self.choices = choices
+        super().__init__(*args, **kwargs)
+
+    def filter(self, qs, value):
+        if not value:
+            return qs
+        try:
+            filter_query = get_generic_filter_query(
+                value, relation_name=self.relation_name, choices=self.choices
+            )
+        except forms.ValidationError:
+            return qs.none()
+        return qs.filter(filter_query)
+
+
+class GenericRelation(serializers.JSONField):
+    def __init__(self, choices, *args, **kwargs):
+        self.choices = choices
+        self.encoder = kwargs.setdefault("encoder", DjangoJSONEncoder)
+        super().__init__(*args, **kwargs)
+
+    def to_representation(self, value):
+        if not value:
+            return
+        type = None
+        id = None
+        id_attr = None
+        for key, choice in self.choices.items():
+            if isinstance(value, choice["queryset"].model):
+                type = key
+                id_attr = choice.get("id_attr", "id")
+                id = getattr(value, id_attr)
+                break
+
+        if type:
+            return {"type": type, id_attr: id}
+
+    def to_internal_value(self, v):
+        v = super().to_internal_value(v)
+
+        if not v or not isinstance(v, dict):
+            raise serializers.ValidationError("Invalid data")
+
+        try:
+            type = v["type"]
+            field = serializers.ChoiceField(choices=list(self.choices.keys()))
+            type = field.to_internal_value(type)
+        except (TypeError, KeyError, serializers.ValidationError):
+            raise serializers.ValidationError("Invalid type")
+
+        conf = self.choices[type]
+        id_attr = conf.get("id_attr", "id")
+        id_field = conf.get("id_field", serializers.IntegerField(min_value=1))
+        queryset = conf["queryset"]
+        try:
+            id_value = v[id_attr]
+            id_value = id_field.to_internal_value(id_value)
+        except (TypeError, KeyError, serializers.ValidationError):
+            raise serializers.ValidationError(f"Invalid {id_attr}")
+
+        query_getter = conf.get(
+            "get_query", lambda attr, value: models.Q(**{attr: value})
+        )
+        query = query_getter(id_attr, id_value)
+        try:
+            obj = queryset.get(query)
+        except queryset.model.DoesNotExist:
+            raise serializers.ValidationError("Object not found")
+
+        return obj

api/funkwhale_api/common/filters.py

+from django import forms
+from django.db.models import Q
+from django.db.models.functions import Lower
+from django_filters import rest_framework as filters
+from django_filters import widgets
+from drf_spectacular.utils import extend_schema_field
+
+from . import fields, models, search, utils
+
+
+class NoneObject:
+    def __eq__(self, other):
+        return other.__class__ == NoneObject
+
+
+NONE = NoneObject()
+BOOLEAN_CHOICES = [
+    (True, True),
+    ("true", True),
+    ("True", True),
+    ("1", True),
+    ("yes", True),
+    (False, False),
+    ("false", False),
+    ("False", False),
+    ("0", False),
+    ("no", False),
+]
+NULL_BOOLEAN_CHOICES = BOOLEAN_CHOICES + [
+    ("None", NONE),
+    ("none", NONE),
+    ("Null", NONE),
+    ("null", NONE),
+]
+
+
+class CoerceChoiceField(forms.ChoiceField):
+    """
+    Same as forms.ChoiceField but will return the second value
+    in the choices tuple instead of the user provided one
+    """
+
+    def clean(self, value):
+        if value is None:
+            return value
+        v = super().clean(value)
+        try:
+            return [b for a, b in self.choices if v == a][0]
+        except IndexError:
+            raise forms.ValidationError(f"Invalid value {value}")
+
+
+@extend_schema_field(bool)
+class NullBooleanFilter(filters.ChoiceFilter):
+    field_class = CoerceChoiceField
+
+    def __init__(self, *args, **kwargs):
+        self.choices = NULL_BOOLEAN_CHOICES
+        kwargs["choices"] = self.choices
+        super().__init__(*args, **kwargs)
+
+    def filter(self, qs, value):
+        if value in ["", None]:
+            return qs
+        if value == NONE:
+            value = None
+        qs = self.get_method(qs)(**{f"{self.field_name}__{self.lookup_expr}": value})
+        return qs.distinct() if self.distinct else qs
+
+
+def clean_null_boolean_filter(v):
+    v = CoerceChoiceField(choices=NULL_BOOLEAN_CHOICES).clean(v)
+    if v == NONE:
+        v = None
+
+    return v
+
+
+def clean_boolean_filter(v):
+    return CoerceChoiceField(choices=BOOLEAN_CHOICES).clean(v)
+
+
+def get_null_boolean_filter(name):
+    return {"handler": lambda v: Q(**{name: clean_null_boolean_filter(v)})}
+
+
+def get_boolean_filter(name):
+    return {"handler": lambda v: Q(**{name: clean_boolean_filter(v)})}
+
+
+def get_generic_relation_filter(relation_name, choices):
+    return {
+        "handler": lambda v: fields.get_generic_filter_query(
+            v, relation_name=relation_name, choices=choices
+        )
+    }
+
+
+class DummyTypedMultipleChoiceField(forms.TypedMultipleChoiceField):
+    def valid_value(self, value):
+        return True
+
+
+class QueryArrayWidget(widgets.QueryArrayWidget):
+    """
+    Until https://github.com/carltongibson/django-filter/issues/1047 is fixed
+    """
+
+    def value_from_datadict(self, data, files, name):
+        data = data.copy()
+        return super().value_from_datadict(data, files, name)
+
+
+class MultipleQueryFilter(filters.TypedMultipleChoiceFilter):
+    field_class = DummyTypedMultipleChoiceField
+
+    def __init__(self, *args, **kwargs):
+        kwargs["widget"] = QueryArrayWidget()
+        super().__init__(*args, **kwargs)
+
+
+def filter_target(value):
+    config = {
+        "artist": ["artist", "target_id", int],
+        "album": ["album", "target_id", int],
+        "track": ["track", "target_id", int],
+    }
+    parts = value.lower().split(" ")
+    if parts[0].strip() not in config:
+        raise forms.ValidationError("Improper target")
+
+    conf = config[parts[0].strip()]
+
+    query = Q(target_content_type__model=conf[0])
+    if len(parts) > 1:
+        _, lookup_field, validator = conf
+        try:
+            lookup_value = validator(parts[1].strip())
+        except TypeError:
+            raise forms.ValidationError("Imparsable target id")
+        return query & Q(**{lookup_field: lookup_value})
+
+    return query
+
+
+class MutationFilter(filters.FilterSet):
+    is_approved = NullBooleanFilter("is_approved")
+    q = fields.SmartSearchFilter(
+        config=search.SearchConfig(
+            search_fields={
+                "summary": {"to": "summary"},
+                "fid": {"to": "fid"},
+                "type": {"to": "type"},
+            },
+            filter_fields={
+                "domain": {"to": "created_by__domain__name__iexact"},
+                "is_approved": get_null_boolean_filter("is_approved"),
+                "target": {"handler": filter_target},
+                "is_applied": get_boolean_filter("is_applied"),
+            },
+        )
+    )
+
+    class Meta:
+        model = models.Mutation
+        fields = ["is_approved", "is_applied", "type"]
+
+
+class EmptyQuerySet(ValueError):
+    pass
+
+
+class ActorScopeFilter(filters.CharFilter):
+    def __init__(self, *args, **kwargs):
+        self.actor_field = kwargs.pop("actor_field")
+        self.library_field = kwargs.pop("library_field", None)
+        super().__init__(*args, **kwargs)
+
+    def filter(self, queryset, value):
+        if not value:
+            return queryset
+
+        request = getattr(self.parent, "request", None)
+        if not request:
+            return queryset.none()
+
+        user = getattr(request, "user", None)
+        actor = getattr(user, "actor", None)
+        scopes = [v.strip().lower() for v in value.split(",")]
+        query = None
+        for scope in scopes:
+            try:
+                right_query = self.get_query(scope, user, actor)
+            except ValueError:
+                return queryset.none()
+            query = utils.join_queries_or(query, right_query)
+
+        return queryset.filter(query).distinct()
+
+    def get_query(self, scope, user, actor):
+        from funkwhale_api.federation import models as federation_models
+
+        if scope == "me":
+            return self.filter_me(actor)
+        elif scope == "all":
+            return Q(pk__gte=0)
+
+        elif scope == "subscribed":
+            if not actor or self.library_field is None:
+                raise EmptyQuerySet()
+            followed_libraries = federation_models.LibraryFollow.objects.filter(
+                approved=True, actor=user.actor
+            ).values_list("target_id", flat=True)
+            if not self.library_field:
+                predicate = "pk__in"
+            else:
+                predicate = f"{self.library_field}__in"
+            return Q(**{predicate: followed_libraries})
+
+        elif scope.startswith("actor:"):
+            full_username = scope.split("actor:", 1)[1]
+            username, domain = full_username.split("@")
+            try:
+                actor = federation_models.Actor.objects.get(
+                    preferred_username__iexact=username,
+                    domain_id=domain,
+                )
+            except federation_models.Actor.DoesNotExist:
+                raise EmptyQuerySet()
+
+            return Q(**{self.actor_field: actor})
+        elif scope.startswith("domain:"):
+            domain = scope.split("domain:", 1)[1]
+            return Q(**{f"{self.actor_field}__domain_id": domain})
+        else:
+            raise EmptyQuerySet()
+
+    def filter_me(self, actor):
+        if not actor:
+            raise EmptyQuerySet()
+
+        return Q(**{self.actor_field: actor})
+
+
+class CaseInsensitiveNameOrderingFilter(filters.OrderingFilter):
+    def filter(self, qs, value):
+        order_by = []
+
+        if value is None:
+            return qs
+
+        for param in value:
+            if param == "name":
+                order_by.append(Lower("name"))
+            else:
+                order_by.append(self.get_ordering_value(param))
+
+        return qs.order_by(*order_by)

api/funkwhale_api/common/locales.py

+# from https://gist.github.com/carlopires/1262033/c52ef0f7ce4f58108619508308372edd8d0bd518
+
+ISO_639_CHOICES = [
+    ("ab", "Abkhaz"),
+    ("aa", "Afar"),
+    ("af", "Afrikaans"),
+    ("ak", "Akan"),
+    ("sq", "Albanian"),
+    ("am", "Amharic"),
+    ("ar", "Arabic"),
+    ("an", "Aragonese"),
+    ("hy", "Armenian"),
+    ("as", "Assamese"),
+    ("av", "Avaric"),
+    ("ae", "Avestan"),
+    ("ay", "Aymara"),
+    ("az", "Azerbaijani"),
+    ("bm", "Bambara"),
+    ("ba", "Bashkir"),
+    ("eu", "Basque"),
+    ("be", "Belarusian"),
+    ("bn", "Bengali"),
+    ("bh", "Bihari"),
+    ("bi", "Bislama"),
+    ("bs", "Bosnian"),
+    ("br", "Breton"),
+    ("bg", "Bulgarian"),
+    ("my", "Burmese"),
+    ("ca", "Catalan; Valencian"),
+    ("ch", "Chamorro"),
+    ("ce", "Chechen"),
+    ("ny", "Chichewa; Chewa; Nyanja"),
+    ("zh", "Chinese"),
+    ("cv", "Chuvash"),
+    ("kw", "Cornish"),
+    ("co", "Corsican"),
+    ("cr", "Cree"),
+    ("hr", "Croatian"),
+    ("cs", "Czech"),
+    ("da", "Danish"),
+    ("dv", "Divehi; Maldivian;"),
+    ("nl", "Dutch"),
+    ("dz", "Dzongkha"),
+    ("en", "English"),
+    ("eo", "Esperanto"),
+    ("et", "Estonian"),
+    ("ee", "Ewe"),
+    ("fo", "Faroese"),
+    ("fj", "Fijian"),
+    ("fi", "Finnish"),
+    ("fr", "French"),
+    ("ff", "Fula"),
+    ("gl", "Galician"),
+    ("ka", "Georgian"),
+    ("de", "German"),
+    ("el", "Greek, Modern"),
+    ("gn", "Guaraní"),
+    ("gu", "Gujarati"),
+    ("ht", "Haitian"),
+    ("ha", "Hausa"),
+    ("he", "Hebrew (modern)"),
+    ("hz", "Herero"),
+    ("hi", "Hindi"),
+    ("ho", "Hiri Motu"),
+    ("hu", "Hungarian"),
+    ("ia", "Interlingua"),
+    ("id", "Indonesian"),
+    ("ie", "Interlingue"),
+    ("ga", "Irish"),
+    ("ig", "Igbo"),
+    ("ik", "Inupiaq"),
+    ("io", "Ido"),
+    ("is", "Icelandic"),
+    ("it", "Italian"),
+    ("iu", "Inuktitut"),
+    ("ja", "Japanese"),
+    ("jv", "Javanese"),
+    ("kl", "Kalaallisut"),
+    ("kn", "Kannada"),
+    ("kr", "Kanuri"),
+    ("ks", "Kashmiri"),
+    ("kk", "Kazakh"),
+    ("km", "Khmer"),
+    ("ki", "Kikuyu, Gikuyu"),
+    ("rw", "Kinyarwanda"),
+    ("ky", "Kirghiz, Kyrgyz"),
+    ("kv", "Komi"),
+    ("kg", "Kongo"),
+    ("ko", "Korean"),
+    ("ku", "Kurdish"),
+    ("kj", "Kwanyama, Kuanyama"),
+    ("la", "Latin"),
+    ("lb", "Luxembourgish"),
+    ("lg", "Luganda"),
+    ("li", "Limburgish"),
+    ("ln", "Lingala"),
+    ("lo", "Lao"),
+    ("lt", "Lithuanian"),
+    ("lu", "Luba-Katanga"),
+    ("lv", "Latvian"),
+    ("gv", "Manx"),
+    ("mk", "Macedonian"),
+    ("mg", "Malagasy"),
+    ("ms", "Malay"),
+    ("ml", "Malayalam"),
+    ("mt", "Maltese"),
+    ("mi", "Māori"),
+    ("mr", "Marathi (Marāṭhī)"),
+    ("mh", "Marshallese"),
+    ("mn", "Mongolian"),
+    ("na", "Nauru"),
+    ("nv", "Navajo, Navaho"),
+    ("nb", "Norwegian Bokmål"),
+    ("nd", "North Ndebele"),
+    ("ne", "Nepali"),
+    ("ng", "Ndonga"),
+    ("nn", "Norwegian Nynorsk"),
+    ("no", "Norwegian"),
+    ("ii", "Nuosu"),
+    ("nr", "South Ndebele"),
+    ("oc", "Occitan"),
+    ("oj", "Ojibwe, Ojibwa"),
+    ("cu", "Old Church Slavonic"),
+    ("om", "Oromo"),
+    ("or", "Oriya"),
+    ("os", "Ossetian, Ossetic"),
+    ("pa", "Panjabi, Punjabi"),
+    ("pi", "Pāli"),
+    ("fa", "Persian"),
+    ("pl", "Polish"),
+    ("ps", "Pashto, Pushto"),
+    ("pt", "Portuguese"),
+    ("qu", "Quechua"),
+    ("rm", "Romansh"),
+    ("rn", "Kirundi"),
+    ("ro", "Romanian, Moldavan"),
+    ("ru", "Russian"),
+    ("sa", "Sanskrit (Saṁskṛta)"),
+    ("sc", "Sardinian"),
+    ("sd", "Sindhi"),
+    ("se", "Northern Sami"),
+    ("sm", "Samoan"),
+    ("sg", "Sango"),
+    ("sr", "Serbian"),
+    ("gd", "Scottish Gaelic"),
+    ("sn", "Shona"),
+    ("si", "Sinhala, Sinhalese"),
+    ("sk", "Slovak"),
+    ("sl", "Slovene"),
+    ("so", "Somali"),
+    ("st", "Southern Sotho"),
+    ("es", "Spanish; Castilian"),
+    ("su", "Sundanese"),
+    ("sw", "Swahili"),
+    ("ss", "Swati"),
+    ("sv", "Swedish"),
+    ("ta", "Tamil"),
+    ("te", "Telugu"),
+    ("tg", "Tajik"),
+    ("th", "Thai"),
+    ("ti", "Tigrinya"),
+    ("bo", "Tibetan"),
+    ("tk", "Turkmen"),
+    ("tl", "Tagalog"),
+    ("tn", "Tswana"),
+    ("to", "Tonga"),
+    ("tr", "Turkish"),
+    ("ts", "Tsonga"),
+    ("tt", "Tatar"),
+    ("tw", "Twi"),
+    ("ty", "Tahitian"),
+    ("ug", "Uighur, Uyghur"),
+    ("uk", "Ukrainian"),
+    ("ur", "Urdu"),
+    ("uz", "Uzbek"),
+    ("ve", "Venda"),
+    ("vi", "Vietnamese"),
+    ("vo", "Volapük"),
+    ("wa", "Walloon"),
+    ("cy", "Welsh"),
+    ("wo", "Wolof"),
+    ("fy", "Western Frisian"),
+    ("xh", "Xhosa"),
+    ("yi", "Yiddish"),
+    ("yo", "Yoruba"),
+    ("za", "Zhuang, Chuang"),
+    ("zu", "Zulu"),
+]
+
+
+ISO_639_BY_CODE = {code: name for code, name in ISO_639_CHOICES}

api/funkwhale_api/common/management/commands/createsuperuser.py

+from django.conf import settings
+from django.contrib.auth.management.commands.createsuperuser import (
+    Command as BaseCommand,
+)
+from django.core.management.base import CommandError
+
+
+class Command(BaseCommand):
+    def handle(self, *apps_label, **options):
+        """
+        Creating Django Superusers would bypass some of our username checks, which can lead to unexpected behaviour.
+        We therefore prohibit the execution of the command.
+        """
+        force = settings.FORCE
+        if not force == 1:
+            raise CommandError(
+                "Running createsuperuser on your Funkwhale instance bypasses some of our checks "
+                "which can lead to unexpected behavior of your instance. We therefore suggest to "
+                "run `funkwhale-manage fw users create --superuser` instead."
+            )
+
+        return super().handle(*apps_label, **options)

api/funkwhale_api/common/management/commands/gitpod.py

+import os
+
+import debugpy
+import uvicorn
+from django.core.management import call_command
+from django.core.management.commands.migrate import Command as BaseCommand
+
+from funkwhale_api.common import preferences
+from funkwhale_api.music.models import Library
+from funkwhale_api.users.models import User
+
+
+class Command(BaseCommand):
+    help = "Manage gitpod environment"
+
+    def add_arguments(self, parser):
+        parser.add_argument("command", nargs="?", type=str)
+
+    def handle(self, *args, **options):
+        command = options["command"]
+
+        if not command:
+            return self.show_help()
+
+        if command == "init":
+            return self.init()
+
+        if command == "dev":
+            return self.dev()
+
+    def show_help(self):
+        self.stdout.write("")
+        self.stdout.write("Available commands:")
+        self.stdout.write("init - Initialize gitpod workspace")
+        self.stdout.write("dev - Run Funkwhale in development mode with debug server")
+        self.stdout.write("")
+
+    def init(self):
+        user = User.objects.get(username="gitpod")
+
+        # Allow anonymous access
+        preferences.set("common__api_authentication_required", False)
+
+        # Download music catalog
+        os.system(
+            "git clone https://dev.funkwhale.audio/funkwhale/catalog.git /tmp/catalog"
+        )
+        os.system("mv -f /tmp/catalog/music /workspace/funkwhale/data")
+        os.system("rm -rf /tmp/catalog/music")
+
+        # Import music catalog into library
+        call_command(
+            "create_library",
+            "gitpod",
+            name="funkwhale/catalog",
+            privacy_level="everyone",
+        )
+        call_command(
+            "import_files",
+            Library.objects.get(actor=user.actor).uuid,
+            "/workspace/funkwhale/data/music/",
+            recursive=True,
+            in_place=True,
+            no_input=False,
+        )
+
+    def dev(self):
+        debugpy.listen(5678)
+        uvicorn.run(
+            "config.asgi:application",
+            host="0.0.0.0",
+            port=5000,
+            reload=True,
+            reload_dirs=[
+                "/workspace/funkwhale/api/config/",
+                "/workspace/funkwhale/api/funkwhale_api/",
+            ],
+        )