api/funkwhale_api/common/mixins.py

+from django.db.models import Q
+from django.shortcuts import get_object_or_404
+from rest_framework import serializers
+
+
+class MultipleLookupDetailMixin:
+    lookup_value_regex = "[^/]+"
+    lookup_field = "composite"
+
+    def get_object(self):
+        queryset = self.filter_queryset(self.get_queryset())
+
+        relevant_lookup = None
+        value = None
+        for lookup in self.url_lookups:
+            field_validator = lookup["validator"]
+            try:
+                value = field_validator(self.kwargs["composite"])
+            except serializers.ValidationError:
+                continue
+            else:
+                relevant_lookup = lookup
+                break
+        get_query = relevant_lookup.get(
+            "get_query", lambda value: Q(**{relevant_lookup["lookup_field"]: value})
+        )
+        query = get_query(value)
+        obj = get_object_or_404(queryset, query)
+
+        # May raise a permission denied
+        self.check_object_permissions(self.request, obj)
+
+        return obj

api/funkwhale_api/common/models.py

-import uuid
-import magic
 import mimetypes
+import uuid
 
-from django.contrib.postgres.fields import JSONField
+import magic
+from django.conf import settings
 from django.contrib.contenttypes.fields import GenericForeignKey
 from django.contrib.contenttypes.models import ContentType
-from django.conf import settings
 from django.core.serializers.json import DjangoJSONEncoder
 from django.db import connections, models, transaction
-from django.db.models import Lookup
+from django.db.models import JSONField, Lookup
 from django.db.models.fields import Field
 from django.db.models.sql.compiler import SQLCompiler
 from django.dispatch import receiver
-from django.utils import timezone
 from django.urls import reverse
-
+from django.utils import timezone
 from versatileimagefield.fields import VersatileImageField
 from versatileimagefield.image_warmer import VersatileImageFieldWarmer
 
 from funkwhale_api.federation import utils as federation_utils
 
-from . import utils
-from . import validators
+from . import utils, validators
+
+CONTENT_TEXT_MAX_LENGTH = 5000
+CONTENT_TEXT_SUPPORTED_TYPES = [
+    "text/html",
+    "text/markdown",
+    "text/plain",
+]
 
 
 @Field.register_lookup
@@ -32,7 +36,7 @@ class NotEqual(Lookup):
         lhs, lhs_params = self.process_lhs(compiler, connection)
         rhs, rhs_params = self.process_rhs(compiler, connection)
         params = lhs_params + rhs_params
-        return "%s <> %s" % (lhs, rhs), params
+        return f"{lhs} <> {rhs}", params
 
 
 class NullsLastSQLCompiler(SQLCompiler):
@@ -56,12 +60,12 @@ class NullsLastSQLCompiler(SQLCompiler):
 class NullsLastQuery(models.sql.query.Query):
     """Use a custom compiler to inject 'NULLS LAST' (for PostgreSQL)."""
 
-    def get_compiler(self, using=None, connection=None):
+    def get_compiler(self, using=None, connection=None, elide_empty=True):
         if using is None and connection is None:
             raise ValueError("Need either using or connection")
         if using:
             connection = connections[using]
-        return NullsLastSQLCompiler(self, connection, using)
+        return NullsLastSQLCompiler(self, connection, using, elide_empty)
 
 
 class NullsLastQuerySet(models.QuerySet):
@@ -73,8 +77,8 @@ class NullsLastQuerySet(models.QuerySet):
 class LocalFromFidQuerySet:
     def local(self, include=True):
         host = settings.FEDERATION_HOSTNAME
-        query = models.Q(fid__startswith="http://{}/".format(host)) | models.Q(
-            fid__startswith="https://{}/".format(host)
+        query = models.Q(fid__startswith=f"http://{host}/") | models.Q(
+            fid__startswith=f"https://{host}/"
         )
         if include:
             return self.filter(query)
@@ -108,10 +112,10 @@ class Mutation(models.Model):
 
     type = models.CharField(max_length=100, db_index=True)
     # None = no choice, True = approved, False = refused
-    is_approved = models.NullBooleanField(default=None)
+    is_approved = models.BooleanField(default=None, null=True)
 
     # None = not applied, True = applied, False = failed
-    is_applied = models.NullBooleanField(default=None)
+    is_applied = models.BooleanField(default=None, null=True)
     creation_date = models.DateTimeField(default=timezone.now, db_index=True)
     applied_date = models.DateTimeField(null=True, blank=True, db_index=True)
     summary = models.TextField(max_length=2000, null=True, blank=True)
@@ -167,13 +171,19 @@ def get_file_path(instance, filename):
 
 class AttachmentQuerySet(models.QuerySet):
     def attached(self, include=True):
-        related_fields = ["covered_album", "mutation_attachment"]
+        related_fields = [
+            "covered_album",
+            "mutation_attachment",
+            "covered_track",
+            "covered_artist",
+            "iconed_actor",
+        ]
         query = None
         for field in related_fields:
             field_query = ~models.Q(**{field: None})
             query = query | field_query if query else field_query
 
-        if include is False:
+        if not include:
             query = ~query
 
         return self.filter(query)
@@ -187,7 +197,7 @@ class AttachmentQuerySet(models.QuerySet):
 
 class Attachment(models.Model):
     # Remote URL where the attachment can be fetched
-    url = models.URLField(max_length=500, unique=True, null=True)
+    url = models.URLField(max_length=500, null=True, blank=True)
     uuid = models.UUIDField(unique=True, db_index=True, default=uuid.uuid4)
     # Actor associated with the attachment
     actor = models.ForeignKey(
@@ -208,7 +218,8 @@ class Attachment(models.Model):
         validators=[
             validators.ImageDimensionsValidator(min_width=50, min_height=50),
             validators.FileValidator(
-                allowed_extensions=["png", "jpg", "jpeg"], max_size=1024 * 1024 * 5,
+                allowed_extensions=["png", "jpg", "jpeg"],
+                max_size=1024 * 1024 * 5,
             ),
         ],
     )
@@ -242,17 +253,31 @@ class Attachment(models.Model):
     @property
     def download_url_original(self):
         if self.file:
-            return federation_utils.full_url(self.file.url)
+            return utils.media_url(self.file.url)
         proxy_url = reverse("api:v1:attachments-proxy", kwargs={"uuid": self.uuid})
         return federation_utils.full_url(proxy_url + "?next=original")
 
+    @property
+    def download_url_small_square_crop(self):
+        if self.file:
+            return utils.media_url(self.file.crop["50x50"].url)
+        proxy_url = reverse("api:v1:attachments-proxy", kwargs={"uuid": self.uuid})
+        return federation_utils.full_url(proxy_url + "?next=small_square_crop")
+
     @property
     def download_url_medium_square_crop(self):
         if self.file:
-            return federation_utils.full_url(self.file.crop["200x200"].url)
+            return utils.media_url(self.file.crop["200x200"].url)
         proxy_url = reverse("api:v1:attachments-proxy", kwargs={"uuid": self.uuid})
         return federation_utils.full_url(proxy_url + "?next=medium_square_crop")
 
+    @property
+    def download_url_large_square_crop(self):
+        if self.file:
+            return utils.media_url(self.file.crop["600x600"].url)
+        proxy_url = reverse("api:v1:attachments-proxy", kwargs={"uuid": self.uuid})
+        return federation_utils.full_url(proxy_url + "?next=large_square_crop")
+
 
 class MutationAttachment(models.Model):
     """
@@ -273,6 +298,35 @@ class MutationAttachment(models.Model):
         unique_together = ("attachment", "mutation")
 
 
+class Content(models.Model):
+    """
+    A text content that can be associated to other models, like a description, a summary, etc.
+    """
+
+    text = models.CharField(max_length=CONTENT_TEXT_MAX_LENGTH, blank=True, null=True)
+    content_type = models.CharField(max_length=100)
+
+    @property
+    def rendered(self):
+        from . import utils
+
+        return utils.render_html(self.text, self.content_type)
+
+    @property
+    def as_plain_text(self):
+        from . import utils
+
+        return utils.render_plain_text(self.rendered)
+
+    def truncate(self, length):
+        text = self.as_plain_text
+        truncated = text[:length]
+        if len(truncated) < len(text):
+            truncated += "…"
+
+        return truncated
+
+
 @receiver(models.signals.post_save, sender=Attachment)
 def warm_attachment_thumbnails(sender, instance, **kwargs):
     if not instance.file or not settings.CREATE_IMAGE_THUMBNAILS:
@@ -302,3 +356,42 @@ def trigger_mutation_post_init(sender, instance, created, **kwargs):
     except AttributeError:
         return
     handler(instance)
+
+
+CONTENT_FKS = {
+    "music.Track": ["description"],
+    "music.Album": ["description"],
+    "music.Artist": ["description"],
+}
+
+
+@receiver(models.signals.post_delete, sender=None)
+def remove_attached_content(sender, instance, **kwargs):
+    fk_fields = CONTENT_FKS.get(instance._meta.label, [])
+    for field in fk_fields:
+        if getattr(instance, f"{field}_id"):
+            try:
+                getattr(instance, field).delete()
+            except Content.DoesNotExist:
+                pass
+
+
+class PluginConfiguration(models.Model):
+    """
+    Store plugin configuration in DB
+    """
+
+    code = models.CharField(max_length=100)
+    user = models.ForeignKey(
+        "users.User",
+        related_name="plugins",
+        on_delete=models.CASCADE,
+        null=True,
+        blank=True,
+    )
+    conf = JSONField(null=True, blank=True)
+    enabled = models.BooleanField(default=False)
+    creation_date = models.DateTimeField(default=timezone.now)
+
+    class Meta:
+        unique_together = ("user", "code")

api/funkwhale_api/common/mutations.py

 import persisting_theory
-
-from rest_framework import serializers
-
 from django.db import models, transaction
+from rest_framework import serializers
 
 
 class ConfNotFound(KeyError):
@@ -45,7 +43,7 @@ class Registry(persisting_theory.Registry):
 
     def has_perm(self, perm, type, obj, actor):
         if perm not in ["approve", "suggest"]:
-            raise ValueError("Invalid permission {}".format(perm))
+            raise ValueError(f"Invalid permission {perm}")
         conf = self.get_conf(type, obj)
         checker = conf["perm_checkers"].get(perm)
         if not checker:
@@ -56,7 +54,7 @@ class Registry(persisting_theory.Registry):
         try:
             type_conf = self[type]
         except KeyError:
-            raise ConfNotFound("{} is not a registered mutation".format(type))
+            raise ConfNotFound(f"{type} is not a registered mutation")
 
         try:
             conf = type_conf[obj.__class__]
@@ -65,7 +63,7 @@ class Registry(persisting_theory.Registry):
                 conf = type_conf[None]
             except KeyError:
                 raise ConfNotFound(
-                    "No mutation configuration found for {}".format(obj.__class__)
+                    f"No mutation configuration found for {obj.__class__}"
                 )
         return conf
 
@@ -85,9 +83,6 @@ class MutationSerializer(serializers.Serializer):
 
 
 class UpdateMutationSerializer(serializers.ModelSerializer, MutationSerializer):
-    serialized_relations = {}
-    previous_state_handlers = {}
-
     def __init__(self, *args, **kwargs):
         # we force partial mode, because update mutations are partial
         kwargs.setdefault("partial", True)
@@ -106,13 +101,14 @@ class UpdateMutationSerializer(serializers.ModelSerializer, MutationSerializer):
         return super().validate(validated_data)
 
     def db_serialize(self, validated_data):
+        serialized_relations = self.get_serialized_relations()
         data = {}
         # ensure model fields are serialized properly
         for key, value in list(validated_data.items()):
             if not isinstance(value, models.Model):
                 data[key] = value
                 continue
-            field = self.serialized_relations[key]
+            field = serialized_relations[key]
             data[key] = getattr(value, field)
         return data
 
@@ -121,7 +117,7 @@ class UpdateMutationSerializer(serializers.ModelSerializer, MutationSerializer):
         # we use our serialized_relations configuration
         # to ensure we store ids instead of model instances in our json
         # payload
-        for field, attr in self.serialized_relations.items():
+        for field, attr in self.get_serialized_relations().items():
             try:
                 obj = data[field]
             except KeyError:
@@ -140,10 +136,16 @@ class UpdateMutationSerializer(serializers.ModelSerializer, MutationSerializer):
         return get_update_previous_state(
             obj,
             *list(validated_data.keys()),
-            serialized_relations=self.serialized_relations,
-            handlers=self.previous_state_handlers,
+            serialized_relations=self.get_serialized_relations(),
+            handlers=self.get_previous_state_handlers(),
         )
 
+    def get_serialized_relations(self):
+        return {}
+
+    def get_previous_state_handlers(self):
+        return {}
+
 
 def get_update_previous_state(obj, *fields, serialized_relations={}, handlers={}):
     if not fields:

api/funkwhale_api/common/permissions.py

@@ -2,7 +2,6 @@ import operator
 
 from django.core.exceptions import ObjectDoesNotExist
 from django.http import Http404
-
 from rest_framework.permissions import BasePermission
 
 from funkwhale_api.common import preferences
@@ -57,3 +56,59 @@ class OwnerPermission(BasePermission):
         if not owner or not request.user.is_authenticated or owner != request.user:
             raise owner_exception
         return True
+
+
+class PrivacyLevelPermission(BasePermission):
+    """
+    Ensure the request actor have access to the object considering the privacylevel configuration
+    of the user.
+    request.user is None if actor, else its Anonymous if user is not auth.
+    """
+
+    def has_object_permission(self, request, view, obj):
+        if (
+            not hasattr(obj, "user")
+            and hasattr(obj, "actor")
+            and not obj.actor.is_local
+        ):
+            # it's a remote actor object. It should be public.
+            # But we could trigger an update of the remote actor data
+            # to avoid leaking data (#2326)
+            return True
+
+        if hasattr(obj, "privacy_level"):
+            privacy_level = obj.privacy_level
+        elif hasattr(obj, "actor") and obj.actor.user:
+            privacy_level = obj.actor.user.privacy_level
+        else:
+            privacy_level = obj.user.privacy_level
+
+        obj_actor = obj.actor if hasattr(obj, "actor") else obj.user.actor
+
+        if privacy_level == "everyone":
+            return True
+
+        # user is anonymous
+        if hasattr(request, "actor"):
+            request_actor = request.actor
+        elif request.user and request.user.is_authenticated:
+            request_actor = request.user.actor
+        else:
+            return False
+
+        if privacy_level == "instance":
+            # user is local
+            if request.user and hasattr(request.user, "actor"):
+                return True
+            elif hasattr(request, "actor") and request.actor and request.actor.is_local:
+                return True
+            else:
+                return False
+
+        elif privacy_level == "me" and obj_actor == request_actor:
+            return True
+
+        elif request_actor in obj_actor.get_approved_followers():
+            return True
+        else:
+            return False

api/funkwhale_api/common/preferences.py

+import json
+
 from django import forms
 from django.conf import settings
+from django.forms import JSONField
 from dynamic_preferences import serializers, types
 from dynamic_preferences.registries import global_preferences_registry
 
 
-class DefaultFromSettingMixin(object):
+class DefaultFromSettingMixin:
     def get_default(self):
         return getattr(settings, self.setting)
 
@@ -35,7 +38,7 @@ class StringListSerializer(serializers.BaseSerializer):
 
         if type(value) not in [list, tuple]:
             raise cls.exception(
-                "Cannot serialize, value {} is not a list or a tuple".format(value)
+                f"Cannot serialize, value {value} is not a list or a tuple"
             )
 
         if cls.sort:
@@ -54,6 +57,50 @@ class StringListPreference(types.BasePreferenceType):
     field_class = forms.MultipleChoiceField
 
     def get_api_additional_data(self):
-        d = super(StringListPreference, self).get_api_additional_data()
+        d = super().get_api_additional_data()
         d["choices"] = self.get("choices")
         return d
+
+
+class JSONSerializer(serializers.BaseSerializer):
+    required = True
+
+    @classmethod
+    def to_db(cls, value, **kwargs):
+        if not cls.required and value is None:
+            return json.dumps(value)
+        data_serializer = cls.data_serializer_class(data=value)
+        if not data_serializer.is_valid():
+            raise cls.exception(
+                f"{value} is not a valid value: {data_serializer.errors}"
+            )
+        value = data_serializer.validated_data
+        try:
+            return json.dumps(value, sort_keys=True)
+        except TypeError:
+            raise cls.exception(
+                f"Cannot serialize, value {value} is not JSON serializable"
+            )
+
+    @classmethod
+    def to_python(cls, value, **kwargs):
+        return json.loads(value)
+
+
+class SerializedPreference(types.BasePreferenceType):
+    """
+    A preference that store arbitrary JSON and validate it using a rest_framework
+    serializer
+    """
+
+    data_serializer_class = None
+    field_class = JSONField
+    widget = forms.Textarea
+
+    @property
+    def serializer(self):
+        class _internal(JSONSerializer):
+            data_serializer_class = self.data_serializer_class
+            required = self.get("required")
+
+        return _internal

api/funkwhale_api/common/renderers.py

+from rest_framework.renderers import JSONRenderer
+
+
+class ActivityStreamRenderer(JSONRenderer):
+    media_type = "application/activity+json"

api/funkwhale_api/common/routers.py

-from rest_framework.routers import SimpleRouter
+from rest_framework.routers import DefaultRouter
 
 
-class OptionalSlashRouter(SimpleRouter):
+class OptionalSlashRouter(DefaultRouter):
     def __init__(self):
         super().__init__()
         self.trailing_slash = "/?"

api/funkwhale_api/common/scripts/__init__.py

-from . import create_actors
-from . import create_image_variations
-from . import django_permissions_to_user_permissions
-from . import migrate_to_user_libraries
-from . import delete_pre_017_federated_uploads
-from . import test
-
+from . import (
+    create_actors,
+    delete_pre_017_federated_uploads,
+    django_permissions_to_user_permissions,
+    migrate_to_user_libraries,
+    test,
+)
 
 __all__ = [
     "create_actors",
-    "create_image_variations",
     "django_permissions_to_user_permissions",
     "migrate_to_user_libraries",
     "delete_pre_017_federated_uploads",

api/funkwhale_api/common/scripts/create_actors.py

@@ -9,15 +9,13 @@ from funkwhale_api.users.models import User, create_actor
 def main(command, **kwargs):
     qs = User.objects.filter(actor__isnull=True).order_by("username")
     total = len(qs)
-    command.stdout.write("{} users found without actors".format(total))
+    command.stdout.write(f"{total} users found without actors")
     for i, user in enumerate(qs):
-        command.stdout.write(
-            "{}/{} creating actor for {}".format(i + 1, total, user.username)
-        )
+        command.stdout.write(f"{i + 1}/{total} creating actor for {user.username}")
         try:
             user.actor = create_actor(user)
         except IntegrityError as e:
             # somehow, an actor with the the url exists in the database
-            command.stderr.write("Error while creating actor: {}".format(str(e)))
+            command.stderr.write(f"Error while creating actor: {str(e)}")
             continue
         user.save(update_fields=["actor"])

api/funkwhale_api/common/scripts/create_image_variations.py

@@ -5,20 +5,15 @@ Compute different sizes of image used for Album covers and User avatars
 from versatileimagefield.image_warmer import VersatileImageFieldWarmer
 
 from funkwhale_api.common.models import Attachment
-from funkwhale_api.music.models import Album
-from funkwhale_api.users.models import User
-
 
 MODELS = [
-    (Album, "cover", "square"),
-    (User, "avatar", "square"),
     (Attachment, "file", "attachment_square"),
 ]
 
 
 def main(command, **kwargs):
     for model, attribute, key_set in MODELS:
-        qs = model.objects.exclude(**{"{}__isnull".format(attribute): True})
+        qs = model.objects.exclude(**{f"{attribute}__isnull": True})
         qs = qs.exclude(**{attribute: ""})
         warmer = VersatileImageFieldWarmer(
             instance_or_queryset=qs,
@@ -26,10 +21,8 @@ def main(command, **kwargs):
             image_attr=attribute,
             verbose=True,
         )
-        command.stdout.write(
-            "Creating images for {} / {}".format(model.__name__, attribute)
-        )
+        command.stdout.write(f"Creating images for {model.__name__} / {attribute}")
         num_created, failed_to_create = warmer.warm()
         command.stdout.write(
-            "  {} created, {} in error".format(num_created, len(failed_to_create))
+            f"  {num_created} created, {len(failed_to_create)} in error"
         )

api/funkwhale_api/common/scripts/delete_pre_017_federated_uploads.py

@@ -10,5 +10,5 @@ def main(command, **kwargs):
         source__startswith="http", source__contains="/federation/music/file/"
     ).exclude(source__contains="youtube")
     total = queryset.count()
-    command.stdout.write("{} uploads found".format(total))
+    command.stdout.write(f"{total} uploads found")
     queryset.delete()

api/funkwhale_api/common/scripts/django_permissions_to_user_permissions.py

@@ -23,6 +23,6 @@ def main(command, **kwargs):
         total = users.count()
 
         command.stdout.write(
-            "Updating {} users with {} permission...".format(total, user_permission)
+            f"Updating {total} users with {user_permission} permission..."
         )
-        users.update(**{"permission_{}".format(user_permission): True})
+        users.update(**{f"permission_{user_permission}": True})

api/funkwhale_api/common/scripts/migrate_to_user_libraries.py

@@ -12,12 +12,12 @@ This command will also generate federation ids for existing resources.
 """
 
 from django.conf import settings
-from django.db.models import functions, CharField, F, Value
+from django.db.models import CharField, F, Value, functions
 
+from funkwhale_api.common import preferences
+from funkwhale_api.federation import models as federation_models
 from funkwhale_api.music import models
 from funkwhale_api.users.models import User
-from funkwhale_api.federation import models as federation_models
-from funkwhale_api.common import preferences
 
 
 def create_libraries(open_api, stdout):
@@ -36,9 +36,7 @@ def create_libraries(open_api, stdout):
         )
         libraries_by_user[library.actor.user.pk] = library.pk
         if created:
-            stdout.write(
-                "  * Created library {} for user {}".format(library.pk, a.user.pk)
-            )
+            stdout.write(f"  * Created library {library.pk} for user {a.user.pk}")
         else:
             stdout.write(
                 "  * Found existing library {} for user {}".format(
@@ -60,13 +58,9 @@ def update_uploads(libraries_by_user, stdout):
         )
         total = candidates.update(library=library_id, import_status="finished")
         if total:
-            stdout.write(
-                "  * Assigned {} uploads to user {}'s library".format(total, user_id)
-            )
+            stdout.write(f"  * Assigned {total} uploads to user {user_id}'s library")
         else:
-            stdout.write(
-                "  * No uploads to assign to user {}'s library".format(user_id)
-            )
+            stdout.write(f"  * No uploads to assign to user {user_id}'s library")
 
 
 def update_orphan_uploads(open_api, stdout):
@@ -105,14 +99,12 @@ def update_orphan_uploads(open_api, stdout):
 def set_fid(queryset, path, stdout):
     model = queryset.model._meta.label
     qs = queryset.filter(fid=None)
-    base_url = "{}{}".format(settings.FUNKWHALE_URL, path)
-    stdout.write(
-        "* Assigning federation ids to {} entries (path: {})".format(model, base_url)
-    )
+    base_url = f"{settings.FUNKWHALE_URL}{path}"
+    stdout.write(f"* Assigning federation ids to {model} entries (path: {base_url})")
     new_fid = functions.Concat(Value(base_url), F("uuid"), output_field=CharField())
     total = qs.update(fid=new_fid)
 
-    stdout.write("  * {} entries updated".format(total))
+    stdout.write(f"  * {total} entries updated")
 
 
 def update_shared_inbox_url(stdout):
@@ -123,16 +115,16 @@ def update_shared_inbox_url(stdout):
 
 
 def generate_actor_urls(part, stdout):
-    field = "{}_url".format(part)
-    stdout.write("* Update {} for local actors...".format(field))
+    field = f"{part}_url"
+    stdout.write(f"* Update {field} for local actors...")
 
     queryset = federation_models.Actor.objects.local().filter(**{field: None})
-    base_url = "{}/federation/actors/".format(settings.FUNKWHALE_URL)
+    base_url = f"{settings.FUNKWHALE_URL}/federation/actors/"
 
     new_field = functions.Concat(
         Value(base_url),
         F("preferred_username"),
-        Value("/{}".format(part)),
+        Value(f"/{part}"),
         output_field=CharField(),
     )
 

api/funkwhale_api/common/search.py

 import re
 
+from django.contrib.postgres.search import SearchQuery
 from django.db.models import Q
 
+from . import utils
 
 QUERY_REGEX = re.compile(r'(((?P<key>\w+):)?(?P<value>"[^"]+"|[\S]+))')
 
@@ -9,7 +11,7 @@ QUERY_REGEX = re.compile(r'(((?P<key>\w+):)?(?P<value>"[^"]+"|[\S]+))')
 def parse_query(query):
     """
     Given a search query such as "hello is:issue status:opened",
-    returns a list of dictionnaries discribing each query token
+    returns a list of dictionaries describing each query token
     """
     matches = [m.groupdict() for m in QUERY_REGEX.finditer(query.lower())]
     for m in matches:
@@ -23,7 +25,7 @@ def normalize_query(
     findterms=re.compile(r'"([^"]+)"|(\S+)').findall,
     normspace=re.compile(r"\s{2,}").sub,
 ):
-    """ Splits the query string in invidual keywords, getting rid of unecessary spaces
+    """Splits the query string in individual keywords, getting rid of unnecessary spaces
     and grouping quoted words together.
     Example:
 
@@ -56,6 +58,59 @@ def get_query(query_string, search_fields):
     return query
 
 
+def remove_chars(string, chars):
+    for char in chars:
+        string = string.replace(char, "")
+    return string
+
+
+def get_fts_query(query_string, fts_fields=["body_text"], model=None):
+    search_type = "raw"
+    if query_string.startswith('"') and query_string.endswith('"'):
+        # we pass the query directly to the FTS engine
+        query_string = query_string[1:-1]
+    else:
+        query_string = remove_chars(query_string, ['"', "&", "(", ")", "!", "'"])
+        parts = query_string.replace(":", "").split(" ")
+        parts = [f"{p}:*" for p in parts if p]
+        if not parts:
+            return Q(pk=None)
+
+        query_string = "&".join(parts)
+
+    if not fts_fields or not query_string.strip():
+        return Q(pk=None)
+    query = None
+    for field in fts_fields:
+        if "__" in field and model:
+            # When we have a nested lookup, we switch to a subquery for enhanced performance
+            fk_field_name, lookup = (
+                field.split("__")[0],
+                "__".join(field.split("__")[1:]),
+            )
+            fk_field = model._meta.get_field(fk_field_name)
+            related_model = fk_field.related_model
+            subquery = related_model.objects.filter(
+                **{
+                    lookup: SearchQuery(
+                        query_string, search_type=search_type, config="english_nostop"
+                    )
+                }
+            ).values_list("pk", flat=True)
+            new_query = Q(**{f"{fk_field_name}__in": list(subquery)})
+        else:
+            new_query = Q(
+                **{
+                    field: SearchQuery(
+                        query_string, search_type=search_type, config="english_nostop"
+                    )
+                }
+            )
+        query = utils.join_queries_or(query, new_query)
+
+    return query
+
+
 def filter_tokens(tokens, valid):
     return [t for t in tokens if t["key"] in valid]
 
@@ -125,7 +180,7 @@ class SearchConfig:
             except KeyError:
                 # no cleaning to apply
                 value = token["value"]
-            q = Q(**{"{}__icontains".format(to): value})
+            q = Q(**{f"{to}__icontains": value})
             if not specific_field_query:
                 specific_field_query = q
             else:

api/funkwhale_api/common/serializers.py

 import collections
 import io
-import PIL
 import os
 
-from rest_framework import serializers
-
+import PIL
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.files.uploadedfile import SimpleUploadedFile
-from django.utils.encoding import smart_text
-from django.utils.translation import ugettext_lazy as _
+from django.utils.encoding import smart_str
+from django.utils.translation import gettext_lazy as _
+from drf_spectacular.types import OpenApiTypes
+from drf_spectacular.utils import extend_schema_field
+from rest_framework import serializers
 
-from . import models
+from . import models, utils
 
 
 class RelatedField(serializers.RelatedField):
@@ -23,6 +24,7 @@ class RelatedField(serializers.RelatedField):
         self.related_field_name = related_field_name
         self.serializer = serializer
         self.filters = kwargs.pop("filters", None)
+        self.queryset_filter = kwargs.pop("queryset_filter", None)
         try:
             kwargs["queryset"] = kwargs.pop("queryset")
         except KeyError:
@@ -35,16 +37,22 @@ class RelatedField(serializers.RelatedField):
             filters.update(self.filters(self.context))
         return filters
 
+    def filter_queryset(self, queryset):
+        if self.queryset_filter:
+            queryset = self.queryset_filter(queryset, self.context)
+        return queryset
+
     def to_internal_value(self, data):
         try:
             queryset = self.get_queryset()
             filters = self.get_filters(data)
+            queryset = self.filter_queryset(queryset)
             return queryset.get(**filters)
         except ObjectDoesNotExist:
             self.fail(
                 "does_not_exist",
                 related_field_name=self.related_field_name,
-                value=smart_text(data),
+                value=smart_str(data),
             )
         except (TypeError, ValueError):
             self.fail("invalid")
@@ -69,18 +77,19 @@ class RelatedField(serializers.RelatedField):
                     self.display_value(item),
                 )
                 for item in queryset
+                if self.serializer
             ]
         )
 
 
-class Action(object):
+class Action:
     def __init__(self, name, allow_all=False, qs_filter=None):
         self.name = name
         self.allow_all = allow_all
         self.qs_filter = qs_filter
 
     def __repr__(self):
-        return "<Action {}>".format(self.name)
+        return f"<Action {self.name}>"
 
 
 class ActionSerializer(serializers.Serializer):
@@ -104,7 +113,7 @@ class ActionSerializer(serializers.Serializer):
             )
 
         for action in self.actions_by_name.keys():
-            handler_name = "handle_{}".format(action)
+            handler_name = f"handle_{action}"
             assert hasattr(self, handler_name), "{} miss a {} method".format(
                 self.__class__.__name__, handler_name
             )
@@ -124,9 +133,9 @@ class ActionSerializer(serializers.Serializer):
         if value == "all":
             return self.queryset.all().order_by("id")
         if type(value) in [list, tuple]:
-            return self.queryset.filter(
-                **{"{}__in".format(self.pk_field): value}
-            ).order_by(self.pk_field)
+            return self.queryset.filter(**{f"{self.pk_field}__in": value}).order_by(
+                self.pk_field
+            )
 
         raise serializers.ValidationError(
             "{} is not a valid value for objects. You must provide either a "
@@ -211,7 +220,7 @@ class StripExifImageField(serializers.ImageField):
         with io.BytesIO() as output:
             image_without_exif.save(
                 output,
-                format=PIL.Image.EXTENSION[os.path.splitext(file_obj.name)[-1]],
+                format=PIL.Image.EXTENSION[os.path.splitext(file_obj.name)[-1].lower()],
                 quality=100,
             )
             content = output.getvalue()
@@ -261,6 +270,7 @@ class APIMutationSerializer(serializers.ModelSerializer):
             "previous_state",
         ]
 
+    @extend_schema_field(OpenApiTypes.OBJECT)
     def get_target(self, obj):
         target = obj.target
         if not target:
@@ -271,7 +281,7 @@ class APIMutationSerializer(serializers.ModelSerializer):
 
     def validate_type(self, value):
         if value not in self.context["registry"]:
-            raise serializers.ValidationError("Invalid mutation type {}".format(value))
+            raise serializers.ValidationError(f"Invalid mutation type {value}")
         return value
 
 
@@ -283,28 +293,86 @@ class AttachmentSerializer(serializers.Serializer):
     file = StripExifImageField(write_only=True)
     urls = serializers.SerializerMethodField()
 
+    @extend_schema_field(
+        {
+            "type": "object",
+            "properties": {
+                "original": {"type": "string"},
+                "small_square_crop": {"type": "string"},
+                "medium_square_crop": {"type": "string"},
+                "large_square_crop": {"type": "string"},
+            },
+        }
+    )
     def get_urls(self, o):
         urls = {}
         urls["source"] = o.url
         urls["original"] = o.download_url_original
+        urls["small_square_crop"] = o.download_url_small_square_crop
         urls["medium_square_crop"] = o.download_url_medium_square_crop
+        urls["large_square_crop"] = o.download_url_large_square_crop
         return urls
 
-    def to_representation(self, o):
-        repr = super().to_representation(o)
-        # XXX: BACKWARD COMPATIBILITY
-        # having the attachment urls in a nested JSON obj is better,
-        # but we can't do this without breaking clients
-        # So we extract the urls and include these in the parent payload
-        repr.update({k: v for k, v in repr["urls"].items() if k != "source"})
-        # also, our legacy images had lots of variations (400x400, 200x200, 50x50)
-        # but we removed some of these, so we emulate these by hand (by redirecting)
-        # to actual, existing attachment variations
-        repr["square_crop"] = repr["medium_square_crop"]
-        repr["small_square_crop"] = repr["medium_square_crop"]
-        return repr
-
     def create(self, validated_data):
         return models.Attachment.objects.create(
             file=validated_data["file"], actor=validated_data["actor"]
         )
+
+
+class ContentSerializer(serializers.Serializer):
+    text = serializers.CharField(
+        max_length=models.CONTENT_TEXT_MAX_LENGTH, allow_null=True
+    )
+    content_type = serializers.ChoiceField(
+        choices=models.CONTENT_TEXT_SUPPORTED_TYPES,
+    )
+    html = serializers.SerializerMethodField()
+
+    def get_html(self, o) -> str:
+        return utils.render_html(o.text, o.content_type)
+
+
+class NullToEmptDict:
+    def get_attribute(self, o):
+        attr = super().get_attribute(o)
+        if attr is None:
+            return {}
+        return attr
+
+    def to_representation(self, v):
+        if not v:
+            return v
+        return super().to_representation(v)
+
+
+class ScopesSerializer(serializers.Serializer):
+    id = serializers.CharField()
+    rate = serializers.CharField()
+    description = serializers.CharField()
+    limit = serializers.IntegerField()
+    duration = serializers.IntegerField()
+    remaining = serializers.IntegerField()
+    available = serializers.IntegerField()
+    available_seconds = serializers.IntegerField()
+    reset = serializers.IntegerField()
+    reset_seconds = serializers.IntegerField()
+
+
+class IdentSerializer(serializers.Serializer):
+    type = serializers.CharField()
+    id = serializers.CharField()
+
+
+class RateLimitSerializer(serializers.Serializer):
+    enabled = serializers.BooleanField()
+    ident = IdentSerializer()
+    scopes = serializers.ListField(child=ScopesSerializer())
+
+
+class ErrorDetailSerializer(serializers.Serializer):
+    detail = serializers.CharField(source="*")
+
+
+class TextPreviewSerializer(serializers.Serializer):
+    rendered = serializers.CharField(read_only=True, source="*")
+    text = serializers.CharField(write_only=True)

api/funkwhale_api/common/signals.py

 import django.dispatch
 
-mutation_created = django.dispatch.Signal(providing_args=["mutation"])
-mutation_updated = django.dispatch.Signal(
-    providing_args=["mutation", "old_is_approved", "new_is_approved"]
-)
+""" Required args: mutation """
+mutation_created = django.dispatch.Signal()
+""" Required args: mutation, old_is_approved, new_is_approved """
+mutation_updated = django.dispatch.Signal()

api/funkwhale_api/common/storage.py

-import slugify
+import os
+import shutil
 
+import slugify
 from django.core.files.storage import FileSystemStorage
 from storages.backends.s3boto3 import S3Boto3Storage
 
@@ -15,6 +17,16 @@ class ASCIIFileSystemStorage(FileSystemStorage):
     def get_valid_name(self, name):
         return super().get_valid_name(asciionly(name))
 
+    def force_delete(self, name):
+        path = self.path(name)
+        try:
+            if os.path.isdir(path):
+                shutil.rmtree(path)
+            else:
+                return super().delete(name)
+        except FileNotFoundError:
+            pass
+
 
 class ASCIIS3Boto3Storage(S3Boto3Storage):
     def get_valid_name(self, name):

api/funkwhale_api/common/tasks.py

@@ -11,10 +11,7 @@ from django.utils import timezone
 from funkwhale_api.common import channels
 from funkwhale_api.taskapp import celery
 
-from . import models
-from . import serializers
-from . import session
-from . import signals
+from . import models, serializers, session, signals
 
 logger = logging.getLogger(__name__)
 
@@ -77,12 +74,13 @@ def fetch_remote_attachment(attachment, filename=None, save=True):
     attachment.last_fetch_date = timezone.now()
     with tempfile.TemporaryFile() as tf:
         with s.get(attachment.url, timeout=5, stream=True) as r:
-            for chunk in r.iter_content():
+            for chunk in r.iter_content(chunk_size=1024 * 100):
                 tf.write(chunk)
             tf.seek(0)
-            attachment.file.save(
-                filename or attachment.url.split("/")[-1], File(tf), save=save
-            )
+            if not filename:
+                filename = attachment.url.split("/")[-1]
+                filename = filename[-50:]
+            attachment.file.save(filename, File(tf), save=save)
 
 
 @celery.app.task(name="common.prune_unattached_attachments")

api/funkwhale_api/common/throttling.py

 import collections
 
+from django.conf import settings
 from django.core.cache import cache
 from rest_framework import throttling as rest_throttling
 
-from django.conf import settings
-
 
-def get_ident(request):
-    if hasattr(request, "user") and request.user.is_authenticated:
-        return {"type": "authenticated", "id": request.user.pk}
+def get_ident(user, request):
+    if user and user.is_authenticated:
+        return {"type": "authenticated", "id": f"{user.pk}"}
     ident = rest_throttling.BaseThrottle().get_ident(request)
 
     return {"type": "anonymous", "id": ident}
@@ -89,7 +88,7 @@ class FunkwhaleThrottle(rest_throttling.SimpleRateThrottle):
 
     def allow_request(self, request, view):
         self.request = request
-        self.ident = get_ident(request)
+        self.ident = get_ident(getattr(request, "user", None), request)
         action = getattr(view, "action", "*")
         view_scopes = getattr(view, "throttling_scopes", {})
         if view_scopes is None: