api/funkwhale_api/common/management/commands/testdata.py

+from django.core.management.commands.migrate import Command as BaseCommand
+
+from funkwhale_api.federation import factories
+from funkwhale_api.federation.models import Actor
+
+
+class Command(BaseCommand):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.help = "Helper to generate randomized testdata"
+        self.type_choices = {"notifications": self.handle_notifications}
+        self.missing_args_message = f"Please specify one of the following sub-commands: {*self.type_choices.keys(), }"
+
+    def add_arguments(self, parser):
+        subparsers = parser.add_subparsers(dest="subcommand")
+
+        notification_parser = subparsers.add_parser("notifications")
+        notification_parser.add_argument(
+            "username", type=str, help="Username to send the notifications to"
+        )
+        notification_parser.add_argument(
+            "--count", type=int, help="Number of elements to create", default=1
+        )
+
+    def handle(self, *args, **options):
+        self.type_choices[options["subcommand"]](options)
+
+    def handle_notifications(self, options):
+        self.stdout.write(
+            f"Create {options['count']} notification(s) for {options['username']}"
+        )
+        try:
+            actor = Actor.objects.get(preferred_username=options["username"])
+        except Actor.DoesNotExist:
+            self.stdout.write(
+                "The user you want to create notifications for does not exist"
+            )
+            return
+
+        follow_activity = factories.ActivityFactory(type="Follow")
+        for _ in range(options["count"]):
+            factories.InboxItemFactory(actor=actor, activity=follow_activity)

api/funkwhale_api/common/middleware.py

 import html
-import requests
+import io
+import logging
+import os
+import re
 import time
+import tracemalloc
+import urllib.parse
 import xml.sax.saxutils
 
-from django import http
+from django import http, urls
 from django.conf import settings
+from django.contrib import auth
 from django.core.cache import caches
-from django import urls
+from django.middleware import csrf
 from rest_framework import views
 
-from . import preferences
-from . import throttling
-from . import utils
+from funkwhale_api.federation import utils as federation_utils
+
+from . import preferences, session, throttling, utils
 
 EXCLUDED_PATHS = ["/api", "/federation", "/.well-known"]
 
+logger = logging.getLogger(__name__)
+
 
 def should_fallback_to_spa(path):
     if path == "/":
@@ -25,6 +33,16 @@ def should_fallback_to_spa(path):
 def serve_spa(request):
     html = get_spa_html(settings.FUNKWHALE_SPA_HTML_ROOT)
     head, tail = html.split("</head>", 1)
+    if settings.FUNKWHALE_SPA_REWRITE_MANIFEST:
+        new_url = (
+            settings.FUNKWHALE_SPA_REWRITE_MANIFEST_URL
+            or federation_utils.full_url(urls.reverse("api:v1:instance:spa-manifest"))
+        )
+        title = preferences.get("instance__name")
+        if title:
+            head = replace_title(head, title)
+        head = replace_manifest_url(head, new_url)
+
     if not preferences.get("common__api_authentication_required"):
         try:
             request_tags = get_request_head_tags(request) or []
@@ -60,26 +78,55 @@ def serve_spa(request):
         # We add the style add the end of the body to ensure it has the highest
         # priority (since it will come after other stylesheets)
         body, tail = tail.split("</body>", 1)
-        css = "<style>{}</style>".format(css)
+        css = f"<style>{css}</style>"
         tail = body + "\n" + css + "\n</body>" + tail
-    return http.HttpResponse(head + tail)
+
+    # set a csrf token so that visitor can login / query API if needed
+    token = csrf.get_token(request)
+    response = http.HttpResponse(head + tail)
+    response.set_cookie("csrftoken", token, max_age=None)
+    return response
+
+
+MANIFEST_LINK_REGEX = re.compile(r"<link [^>]*rel=(?:'|\")?manifest(?:'|\")?[^>]*>")
+TITLE_REGEX = re.compile(r"<title>.*</title>")
+
+
+def replace_manifest_url(head, new_url):
+    replacement = f'<link rel=manifest href="{new_url}">'
+    head = MANIFEST_LINK_REGEX.sub(replacement, head)
+    return head
+
+
+def replace_title(head, new_title):
+    replacement = f"<title>{html.escape(new_title)}</title>"
+    head = TITLE_REGEX.sub(replacement, head)
+    return head
 
 
 def get_spa_html(spa_url):
+    return get_spa_file(spa_url, "index.html")
+
+
+def get_spa_file(spa_url, name):
     if spa_url.startswith("/"):
+        # spa_url is an absolute path to index.html, on the local disk.
+        # However, we may want to access manifest.json or other files as well, so we
+        # strip the filename
+        path = os.path.join(os.path.dirname(spa_url), name)
         # we try to open a local file
-        with open(spa_url) as f:
-            return f.read()
-    cache_key = "spa-html:{}".format(spa_url)
+        with open(path, "rb") as f:
+            return f.read().decode("utf-8")
+    cache_key = f"spa-file:{spa_url}:{name}"
     cached = caches["local"].get(cache_key)
     if cached:
         return cached
 
-    response = requests.get(
-        utils.join_url(spa_url, "index.html"),
-        verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
+    response = session.get_session().get(
+        utils.join_url(spa_url, name),
     )
     response.raise_for_status()
+    response.encoding = "utf-8"
     content = response.text
     caches["local"].set(cache_key, content, settings.FUNKWHALE_SPA_HTML_CACHE_DURATION)
     return content
@@ -103,7 +150,9 @@ def get_default_head_tags(path):
         {
             "tag": "meta",
             "property": "og:image",
-            "content": utils.join_url(settings.FUNKWHALE_URL, "/front/favicon.png"),
+            "content": utils.join_url(
+                settings.FUNKWHALE_URL, "/android-chrome-512x512.png"
+            ),
         },
         {
             "tag": "meta",
@@ -120,22 +169,25 @@ def render_tags(tags):
     <meta hello="world" />
     """
     for tag in tags:
-
         yield "<{tag} {attrs} />".format(
             tag=tag.pop("tag"),
             attrs=" ".join(
-                [
-                    '{}="{}"'.format(a, html.escape(str(v)))
-                    for a, v in sorted(tag.items())
-                    if v
-                ]
+                [f'{a}="{html.escape(str(v))}"' for a, v in sorted(tag.items()) if v]
             ),
         )
 
 
 def get_request_head_tags(request):
+    accept_header = request.headers.get("Accept") or None
+    redirect_to_ap = (
+        False
+        if not accept_header
+        else not federation_utils.should_redirect_ap_to_html(accept_header)
+    )
     match = urls.resolve(request.path, urlconf=settings.SPA_URLCONF)
-    return match.func(request, *match.args, **match.kwargs)
+    return match.func(
+        request, *match.args, redirect_to_ap=redirect_to_ap, **match.kwargs
+    )
 
 
 def get_custom_css():
@@ -146,6 +198,31 @@ def get_custom_css():
     return xml.sax.saxutils.escape(css)
 
 
+class ApiRedirect(Exception):
+    def __init__(self, url):
+        self.url = url
+
+
+def get_api_response(request, url):
+    """
+    Quite ugly but we have no choice. When Accept header is set to application/activity+json
+    some clients expect to get a JSON payload (instead of the HTML we return). Since
+    redirecting to the URL does not work (because it makes the signature verification fail),
+    we grab the internal view corresponding to the URL, call it and return this as the
+    response
+    """
+    path = urllib.parse.urlparse(url).path
+
+    try:
+        match = urls.resolve(path)
+    except urls.exceptions.Resolver404:
+        return http.HttpResponseNotFound()
+    response = match.func(request, *match.args, **match.kwargs)
+    if hasattr(response, "render"):
+        response.render()
+    return response
+
+
 class SPAFallbackMiddleware:
     def __init__(self, get_response):
         self.get_response = get_response
@@ -154,7 +231,10 @@ class SPAFallbackMiddleware:
         response = self.get_response(request)
 
         if response.status_code == 404 and should_fallback_to_spa(request.path):
+            try:
                 return serve_spa(request)
+            except ApiRedirect as e:
+                return get_api_response(request, e.url)
 
         return response
 
@@ -200,6 +280,25 @@ def monkey_patch_rest_initialize_request():
 monkey_patch_rest_initialize_request()
 
 
+def monkey_patch_auth_get_user():
+    """
+    We need an actor on our users for many endpoints, so we monkey patch
+    auth.get_user to create it if it's missing
+    """
+    original = auth.get_user
+
+    def replacement(request):
+        r = original(request)
+        if not r.is_anonymous and not r.actor:
+            r.create_actor()
+        return r
+
+    setattr(auth, "get_user", replacement)
+
+
+monkey_patch_auth_get_user()
+
+
 class ThrottleStatusMiddleware:
     """
     Include useful information regarding throttling in API responses to
@@ -242,3 +341,80 @@ class ThrottleStatusMiddleware:
                 response["X-RateLimit-ResetSeconds"] = str(remaining)
 
         return response
+
+
+class VerboseBadRequestsMiddleware:
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        response = self.get_response(request)
+        if response.status_code == 400:
+            logger.warning("Bad request: %s", response.content)
+        return response
+
+
+class ProfilerMiddleware:
+    """
+    from https://github.com/omarish/django-cprofile-middleware/blob/master/django_cprofile_middleware/middleware.py
+    Simple profile middleware to profile django views. To run it, add ?prof to
+    the URL like this:
+        http://localhost:8000/view/?prof
+    Optionally pass the following to modify the output:
+    ?sort => Sort the output by a given metric. Default is time.
+        See
+        http://docs.python.org/2/library/profile.html#pstats.Stats.sort_stats
+        for all sort options.
+    ?count => The number of rows to display. Default is 100.
+    ?download => Download profile file suitable for visualization. For example
+        in snakeviz or RunSnakeRun
+    This is adapted from an example found here:
+    http://www.slideshare.net/zeeg/django-con-high-performance-django-presentation.
+    """
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        if "prof" not in request.GET:
+            return self.get_response(request)
+        import profile
+        import pstats
+
+        profiler = profile.Profile()
+        response = profiler.runcall(self.get_response, request)
+        profiler.create_stats()
+        if "prof-download" in request.GET:
+            import marshal
+
+            output = marshal.dumps(profiler.stats)
+            response = http.HttpResponse(
+                output, content_type="application/octet-stream"
+            )
+            response["Content-Disposition"] = "attachment; filename=view.prof"
+            response["Content-Length"] = len(output)
+        stream = io.StringIO()
+        stats = pstats.Stats(profiler, stream=stream)
+
+        stats.sort_stats(request.GET.get("prof-sort", "cumtime"))
+        stats.print_stats(int(request.GET.get("count", 100)))
+
+        response = http.HttpResponse("<pre>%s</pre>" % stream.getvalue())
+
+        return response
+
+
+class PymallocMiddleware:
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        if tracemalloc.is_tracing():
+            snapshot = tracemalloc.take_snapshot()
+            stats = snapshot.statistics("lineno")
+
+            print("Memory trace")
+            for stat in stats[:25]:
+                print(stat)
+
+        return self.get_response(request)

api/funkwhale_api/common/migrations/0002_mutation.py

@@ -36,8 +36,8 @@ class Migration(migrations.Migration):
                     models.UUIDField(db_index=True, default=uuid.uuid4, unique=True),
                 ),
                 ("type", models.CharField(db_index=True, max_length=100)),
-                ("is_approved", models.NullBooleanField(default=None)),
-                ("is_applied", models.NullBooleanField(default=None)),
+                ("is_approved", models.BooleanField(default=None, null=True)),
+                ("is_applied", models.BooleanField(default=None, null=True)),
                 (
                     "creation_date",
                     models.DateTimeField(

api/funkwhale_api/common/migrations/0004_auto_20191111_1338.py

+# Generated by Django 2.2.6 on 2019-11-11 13:38
+
+import django.contrib.postgres.fields.jsonb
+import django.core.serializers.json
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import funkwhale_api.common.models
+import funkwhale_api.common.validators
+import uuid
+import versatileimagefield.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('common', '0003_cit_extension'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Attachment',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('url', models.URLField(max_length=500, unique=True, null=True)),
+                ('uuid', models.UUIDField(db_index=True, default=uuid.uuid4, unique=True)),
+                ('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
+                ('last_fetch_date', models.DateTimeField(blank=True, null=True)),
+                ('size', models.IntegerField(blank=True, null=True)),
+                ('mimetype', models.CharField(blank=True, max_length=200, null=True)),
+                ('file', versatileimagefield.fields.VersatileImageField(max_length=255, upload_to=funkwhale_api.common.models.get_file_path, validators=[funkwhale_api.common.validators.ImageDimensionsValidator(min_height=50, min_width=50), funkwhale_api.common.validators.FileValidator(allowed_extensions=['png', 'jpg', 'jpeg'], max_size=5242880)])),
+                ('actor', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='attachments', to='federation.Actor', null=True)),
+            ],
+        ),
+    ]

api/funkwhale_api/common/migrations/0005_auto_20191125_1421.py

+# Generated by Django 2.2.7 on 2019-11-25 14:21
+
+import django.contrib.postgres.fields.jsonb
+import django.core.serializers.json
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('common', '0004_auto_20191111_1338'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='mutation',
+            name='payload',
+            field=django.contrib.postgres.fields.jsonb.JSONField(encoder=django.core.serializers.json.DjangoJSONEncoder),
+        ),
+        migrations.AlterField(
+            model_name='mutation',
+            name='previous_state',
+            field=django.contrib.postgres.fields.jsonb.JSONField(default=None, encoder=django.core.serializers.json.DjangoJSONEncoder, null=True),
+        ),
+        migrations.CreateModel(
+            name='MutationAttachment',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('attachment', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='mutation_attachment', to='common.Attachment')),
+                ('mutation', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='mutation_attachment', to='common.Mutation')),
+            ],
+            options={
+                'unique_together': {('attachment', 'mutation')},
+            },
+        ),
+    ]

api/funkwhale_api/common/migrations/0006_content.py

+# Generated by Django 2.2.7 on 2020-01-13 10:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('common', '0005_auto_20191125_1421'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Content',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('text', models.CharField(blank=True, max_length=5000, null=True)),
+                ('content_type', models.CharField(max_length=100)),
+            ],
+        ),
+    ]

api/funkwhale_api/common/migrations/0007_auto_20200116_1610.py

+# Generated by Django 2.2.9 on 2020-01-16 16:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('common', '0006_content'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='attachment',
+            name='url',
+            field=models.URLField(max_length=500, null=True),
+        ),
+    ]

api/funkwhale_api/common/migrations/0008_auto_20200701_1317.py

+# Generated by Django 3.0.8 on 2020-07-01 13:17
+
+from django.conf import settings
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('common', '0007_auto_20200116_1610'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='attachment',
+            name='url',
+            field=models.URLField(blank=True, max_length=500, null=True),
+        ),
+        migrations.CreateModel(
+            name='PluginConfiguration',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('code', models.CharField(max_length=100)),
+                ('conf', django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True)),
+                ('enabled', models.BooleanField(default=False)),
+                ('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
+                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='plugins', to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'unique_together': {('user', 'code')},
+            },
+        ),
+    ]

api/funkwhale_api/common/migrations/0009_auto_20220627_1915.py

+# Generated by Django 3.2.13 on 2022-06-27 19:15
+
+import django.core.serializers.json
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('common', '0008_auto_20200701_1317'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='mutation',
+            name='is_applied',
+            field=models.BooleanField(default=None, null=True),
+        ),
+        migrations.AlterField(
+            model_name='mutation',
+            name='is_approved',
+            field=models.BooleanField(default=None, null=True),
+        ),
+        migrations.AlterField(
+            model_name='mutation',
+            name='payload',
+            field=models.JSONField(encoder=django.core.serializers.json.DjangoJSONEncoder),
+        ),
+        migrations.AlterField(
+            model_name='mutation',
+            name='previous_state',
+            field=models.JSONField(default=None, encoder=django.core.serializers.json.DjangoJSONEncoder, null=True),
+        ),
+        migrations.AlterField(
+            model_name='pluginconfiguration',
+            name='conf',
+            field=models.JSONField(blank=True, null=True),
+        ),
+    ]

api/funkwhale_api/common/mixins.py

+from django.db.models import Q
+from django.shortcuts import get_object_or_404
+from rest_framework import serializers
+
+
+class MultipleLookupDetailMixin:
+    lookup_value_regex = "[^/]+"
+    lookup_field = "composite"
+
+    def get_object(self):
+        queryset = self.filter_queryset(self.get_queryset())
+
+        relevant_lookup = None
+        value = None
+        for lookup in self.url_lookups:
+            field_validator = lookup["validator"]
+            try:
+                value = field_validator(self.kwargs["composite"])
+            except serializers.ValidationError:
+                continue
+            else:
+                relevant_lookup = lookup
+                break
+        get_query = relevant_lookup.get(
+            "get_query", lambda value: Q(**{relevant_lookup["lookup_field"]: value})
+        )
+        query = get_query(value)
+        obj = get_object_or_404(queryset, query)
+
+        # May raise a permission denied
+        self.check_object_permissions(self.request, obj)
+
+        return obj

api/funkwhale_api/common/models.py

+import mimetypes
 import uuid
 
-from django.contrib.postgres.fields import JSONField
+import magic
+from django.conf import settings
 from django.contrib.contenttypes.fields import GenericForeignKey
 from django.contrib.contenttypes.models import ContentType
-from django.conf import settings
 from django.core.serializers.json import DjangoJSONEncoder
 from django.db import connections, models, transaction
-from django.db.models import Lookup
+from django.db.models import JSONField, Lookup
 from django.db.models.fields import Field
 from django.db.models.sql.compiler import SQLCompiler
-from django.utils import timezone
+from django.dispatch import receiver
 from django.urls import reverse
+from django.utils import timezone
+from versatileimagefield.fields import VersatileImageField
+from versatileimagefield.image_warmer import VersatileImageFieldWarmer
 
 from funkwhale_api.federation import utils as federation_utils
 
+from . import utils, validators
+
+CONTENT_TEXT_MAX_LENGTH = 5000
+CONTENT_TEXT_SUPPORTED_TYPES = [
+    "text/html",
+    "text/markdown",
+    "text/plain",
+]
+
 
 @Field.register_lookup
 class NotEqual(Lookup):
@@ -23,7 +36,7 @@ class NotEqual(Lookup):
         lhs, lhs_params = self.process_lhs(compiler, connection)
         rhs, rhs_params = self.process_rhs(compiler, connection)
         params = lhs_params + rhs_params
-        return "%s <> %s" % (lhs, rhs), params
+        return f"{lhs} <> {rhs}", params
 
 
 class NullsLastSQLCompiler(SQLCompiler):
@@ -47,12 +60,12 @@ class NullsLastSQLCompiler(SQLCompiler):
 class NullsLastQuery(models.sql.query.Query):
     """Use a custom compiler to inject 'NULLS LAST' (for PostgreSQL)."""
 
-    def get_compiler(self, using=None, connection=None):
+    def get_compiler(self, using=None, connection=None, elide_empty=True):
         if using is None and connection is None:
             raise ValueError("Need either using or connection")
         if using:
             connection = connections[using]
-        return NullsLastSQLCompiler(self, connection, using)
+        return NullsLastSQLCompiler(self, connection, using, elide_empty)
 
 
 class NullsLastQuerySet(models.QuerySet):
@@ -64,8 +77,8 @@ class NullsLastQuerySet(models.QuerySet):
 class LocalFromFidQuerySet:
     def local(self, include=True):
         host = settings.FEDERATION_HOSTNAME
-        query = models.Q(fid__startswith="http://{}/".format(host)) | models.Q(
-            fid__startswith="https://{}/".format(host)
+        query = models.Q(fid__startswith=f"http://{host}/") | models.Q(
+            fid__startswith=f"https://{host}/"
         )
         if include:
             return self.filter(query)
@@ -99,10 +112,10 @@ class Mutation(models.Model):
 
     type = models.CharField(max_length=100, db_index=True)
     # None = no choice, True = approved, False = refused
-    is_approved = models.NullBooleanField(default=None)
+    is_approved = models.BooleanField(default=None, null=True)
 
     # None = not applied, True = applied, False = failed
-    is_applied = models.NullBooleanField(default=None)
+    is_applied = models.BooleanField(default=None, null=True)
     creation_date = models.DateTimeField(default=timezone.now, db_index=True)
     applied_date = models.DateTimeField(null=True, blank=True, db_index=True)
     summary = models.TextField(max_length=2000, null=True, blank=True)
@@ -150,3 +163,235 @@ class Mutation(models.Model):
         self.applied_date = timezone.now()
         self.save(update_fields=["is_applied", "applied_date", "previous_state"])
         return previous_state
+
+
+def get_file_path(instance, filename):
+    return utils.ChunkedPath("attachments")(instance, filename)
+
+
+class AttachmentQuerySet(models.QuerySet):
+    def attached(self, include=True):
+        related_fields = [
+            "covered_album",
+            "mutation_attachment",
+            "covered_track",
+            "covered_artist",
+            "iconed_actor",
+        ]
+        query = None
+        for field in related_fields:
+            field_query = ~models.Q(**{field: None})
+            query = query | field_query if query else field_query
+
+        if not include:
+            query = ~query
+
+        return self.filter(query)
+
+    def local(self, include=True):
+        if include:
+            return self.filter(actor__domain_id=settings.FEDERATION_HOSTNAME)
+        else:
+            return self.exclude(actor__domain_id=settings.FEDERATION_HOSTNAME)
+
+
+class Attachment(models.Model):
+    # Remote URL where the attachment can be fetched
+    url = models.URLField(max_length=500, null=True, blank=True)
+    uuid = models.UUIDField(unique=True, db_index=True, default=uuid.uuid4)
+    # Actor associated with the attachment
+    actor = models.ForeignKey(
+        "federation.Actor",
+        related_name="attachments",
+        on_delete=models.CASCADE,
+        null=True,
+    )
+    creation_date = models.DateTimeField(default=timezone.now)
+    last_fetch_date = models.DateTimeField(null=True, blank=True)
+    # File size
+    size = models.IntegerField(null=True, blank=True)
+    mimetype = models.CharField(null=True, blank=True, max_length=200)
+
+    file = VersatileImageField(
+        upload_to=get_file_path,
+        max_length=255,
+        validators=[
+            validators.ImageDimensionsValidator(min_width=50, min_height=50),
+            validators.FileValidator(
+                allowed_extensions=["png", "jpg", "jpeg"],
+                max_size=1024 * 1024 * 5,
+            ),
+        ],
+    )
+
+    objects = AttachmentQuerySet.as_manager()
+
+    def save(self, **kwargs):
+        if self.file and not self.size:
+            self.size = self.file.size
+
+        if self.file and not self.mimetype:
+            self.mimetype = self.guess_mimetype()
+
+        return super().save()
+
+    @property
+    def is_local(self):
+        return federation_utils.is_local(self.fid)
+
+    def guess_mimetype(self):
+        f = self.file
+        b = min(1000000, f.size)
+        t = magic.from_buffer(f.read(b), mime=True)
+        if not t.startswith("image/"):
+            # failure, we try guessing by extension
+            mt, _ = mimetypes.guess_type(f.name)
+            if mt:
+                t = mt
+        return t
+
+    @property
+    def download_url_original(self):
+        if self.file:
+            return utils.media_url(self.file.url)
+        proxy_url = reverse("api:v1:attachments-proxy", kwargs={"uuid": self.uuid})
+        return federation_utils.full_url(proxy_url + "?next=original")
+
+    @property
+    def download_url_small_square_crop(self):
+        if self.file:
+            return utils.media_url(self.file.crop["50x50"].url)
+        proxy_url = reverse("api:v1:attachments-proxy", kwargs={"uuid": self.uuid})
+        return federation_utils.full_url(proxy_url + "?next=small_square_crop")
+
+    @property
+    def download_url_medium_square_crop(self):
+        if self.file:
+            return utils.media_url(self.file.crop["200x200"].url)
+        proxy_url = reverse("api:v1:attachments-proxy", kwargs={"uuid": self.uuid})
+        return federation_utils.full_url(proxy_url + "?next=medium_square_crop")
+
+    @property
+    def download_url_large_square_crop(self):
+        if self.file:
+            return utils.media_url(self.file.crop["600x600"].url)
+        proxy_url = reverse("api:v1:attachments-proxy", kwargs={"uuid": self.uuid})
+        return federation_utils.full_url(proxy_url + "?next=large_square_crop")
+
+
+class MutationAttachment(models.Model):
+    """
+    When using attachments in mutations, we need to keep a reference to
+    the attachment to ensure it is not pruned by common/tasks.py.
+
+    This is what this model does.
+    """
+
+    attachment = models.OneToOneField(
+        Attachment, related_name="mutation_attachment", on_delete=models.CASCADE
+    )
+    mutation = models.OneToOneField(
+        Mutation, related_name="mutation_attachment", on_delete=models.CASCADE
+    )
+
+    class Meta:
+        unique_together = ("attachment", "mutation")
+
+
+class Content(models.Model):
+    """
+    A text content that can be associated to other models, like a description, a summary, etc.
+    """
+
+    text = models.CharField(max_length=CONTENT_TEXT_MAX_LENGTH, blank=True, null=True)
+    content_type = models.CharField(max_length=100)
+
+    @property
+    def rendered(self):
+        from . import utils
+
+        return utils.render_html(self.text, self.content_type)
+
+    @property
+    def as_plain_text(self):
+        from . import utils
+
+        return utils.render_plain_text(self.rendered)
+
+    def truncate(self, length):
+        text = self.as_plain_text
+        truncated = text[:length]
+        if len(truncated) < len(text):
+            truncated += "…"
+
+        return truncated
+
+
+@receiver(models.signals.post_save, sender=Attachment)
+def warm_attachment_thumbnails(sender, instance, **kwargs):
+    if not instance.file or not settings.CREATE_IMAGE_THUMBNAILS:
+        return
+    warmer = VersatileImageFieldWarmer(
+        instance_or_queryset=instance,
+        rendition_key_set="attachment_square",
+        image_attr="file",
+    )
+    num_created, failed_to_create = warmer.warm()
+
+
+@receiver(models.signals.post_save, sender=Mutation)
+def trigger_mutation_post_init(sender, instance, created, **kwargs):
+    if not created:
+        return
+
+    from . import mutations
+
+    try:
+        conf = mutations.registry.get_conf(instance.type, instance.target)
+    except mutations.ConfNotFound:
+        return
+    serializer = conf["serializer_class"]()
+    try:
+        handler = serializer.mutation_post_init
+    except AttributeError:
+        return
+    handler(instance)
+
+
+CONTENT_FKS = {
+    "music.Track": ["description"],
+    "music.Album": ["description"],
+    "music.Artist": ["description"],
+}
+
+
+@receiver(models.signals.post_delete, sender=None)
+def remove_attached_content(sender, instance, **kwargs):
+    fk_fields = CONTENT_FKS.get(instance._meta.label, [])
+    for field in fk_fields:
+        if getattr(instance, f"{field}_id"):
+            try:
+                getattr(instance, field).delete()
+            except Content.DoesNotExist:
+                pass
+
+
+class PluginConfiguration(models.Model):
+    """
+    Store plugin configuration in DB
+    """
+
+    code = models.CharField(max_length=100)
+    user = models.ForeignKey(
+        "users.User",
+        related_name="plugins",
+        on_delete=models.CASCADE,
+        null=True,
+        blank=True,
+    )
+    conf = JSONField(null=True, blank=True)
+    enabled = models.BooleanField(default=False)
+    creation_date = models.DateTimeField(default=timezone.now)
+
+    class Meta:
+        unique_together = ("user", "code")

api/funkwhale_api/common/mutations.py

 import persisting_theory
-
-from rest_framework import serializers
-
 from django.db import models, transaction
+from rest_framework import serializers
 
 
 class ConfNotFound(KeyError):
@@ -45,7 +43,7 @@ class Registry(persisting_theory.Registry):
 
     def has_perm(self, perm, type, obj, actor):
         if perm not in ["approve", "suggest"]:
-            raise ValueError("Invalid permission {}".format(perm))
+            raise ValueError(f"Invalid permission {perm}")
         conf = self.get_conf(type, obj)
         checker = conf["perm_checkers"].get(perm)
         if not checker:
@@ -56,7 +54,7 @@ class Registry(persisting_theory.Registry):
         try:
             type_conf = self[type]
         except KeyError:
-            raise ConfNotFound("{} is not a registered mutation".format(type))
+            raise ConfNotFound(f"{type} is not a registered mutation")
 
         try:
             conf = type_conf[obj.__class__]
@@ -65,7 +63,7 @@ class Registry(persisting_theory.Registry):
                 conf = type_conf[None]
             except KeyError:
                 raise ConfNotFound(
-                    "No mutation configuration found for {}".format(obj.__class__)
+                    f"No mutation configuration found for {obj.__class__}"
                 )
         return conf
 
@@ -85,9 +83,6 @@ class MutationSerializer(serializers.Serializer):
 
 
 class UpdateMutationSerializer(serializers.ModelSerializer, MutationSerializer):
-    serialized_relations = {}
-    previous_state_handlers = {}
-
     def __init__(self, *args, **kwargs):
         # we force partial mode, because update mutations are partial
         kwargs.setdefault("partial", True)
@@ -106,13 +101,14 @@ class UpdateMutationSerializer(serializers.ModelSerializer, MutationSerializer):
         return super().validate(validated_data)
 
     def db_serialize(self, validated_data):
+        serialized_relations = self.get_serialized_relations()
         data = {}
         # ensure model fields are serialized properly
         for key, value in list(validated_data.items()):
             if not isinstance(value, models.Model):
                 data[key] = value
                 continue
-            field = self.serialized_relations[key]
+            field = serialized_relations[key]
             data[key] = getattr(value, field)
         return data
 
@@ -121,7 +117,7 @@ class UpdateMutationSerializer(serializers.ModelSerializer, MutationSerializer):
         # we use our serialized_relations configuration
         # to ensure we store ids instead of model instances in our json
         # payload
-        for field, attr in self.serialized_relations.items():
+        for field, attr in self.get_serialized_relations().items():
             try:
                 obj = data[field]
             except KeyError:
@@ -140,10 +136,16 @@ class UpdateMutationSerializer(serializers.ModelSerializer, MutationSerializer):
         return get_update_previous_state(
             obj,
             *list(validated_data.keys()),
-            serialized_relations=self.serialized_relations,
-            handlers=self.previous_state_handlers,
+            serialized_relations=self.get_serialized_relations(),
+            handlers=self.get_previous_state_handlers(),
         )
 
+    def get_serialized_relations(self):
+        return {}
+
+    def get_previous_state_handlers(self):
+        return {}
+
 
 def get_update_previous_state(obj, *fields, serialized_relations={}, handlers={}):
     if not fields:

api/funkwhale_api/common/permissions.py

 import operator
 
+from django.core.exceptions import ObjectDoesNotExist
 from django.http import Http404
 from rest_framework.permissions import BasePermission
 
@@ -46,7 +47,68 @@ class OwnerPermission(BasePermission):
             return True
 
         owner_field = getattr(view, "owner_field", "user")
+        owner_exception = getattr(view, "owner_exception", Http404)
+        try:
             owner = operator.attrgetter(owner_field)(obj)
+        except ObjectDoesNotExist:
+            raise owner_exception
+
         if not owner or not request.user.is_authenticated or owner != request.user:
-            raise Http404
+            raise owner_exception
+        return True
+
+
+class PrivacyLevelPermission(BasePermission):
+    """
+    Ensure the request actor have access to the object considering the privacylevel configuration
+    of the user.
+    request.user is None if actor, else its Anonymous if user is not auth.
+    """
+
+    def has_object_permission(self, request, view, obj):
+        if (
+            not hasattr(obj, "user")
+            and hasattr(obj, "actor")
+            and not obj.actor.is_local
+        ):
+            # it's a remote actor object. It should be public.
+            # But we could trigger an update of the remote actor data
+            # to avoid leaking data (#2326)
+            return True
+
+        if hasattr(obj, "privacy_level"):
+            privacy_level = obj.privacy_level
+        elif hasattr(obj, "actor") and obj.actor.user:
+            privacy_level = obj.actor.user.privacy_level
+        else:
+            privacy_level = obj.user.privacy_level
+
+        obj_actor = obj.actor if hasattr(obj, "actor") else obj.user.actor
+
+        if privacy_level == "everyone":
+            return True
+
+        # user is anonymous
+        if hasattr(request, "actor"):
+            request_actor = request.actor
+        elif request.user and request.user.is_authenticated:
+            request_actor = request.user.actor
+        else:
+            return False
+
+        if privacy_level == "instance":
+            # user is local
+            if request.user and hasattr(request.user, "actor"):
+                return True
+            elif hasattr(request, "actor") and request.actor and request.actor.is_local:
+                return True
+            else:
+                return False
+
+        elif privacy_level == "me" and obj_actor == request_actor:
+            return True
+
+        elif request_actor in obj_actor.get_approved_followers():
             return True
+        else:
+            return False

api/funkwhale_api/common/preferences.py

+import json
+
 from django import forms
 from django.conf import settings
+from django.forms import JSONField
 from dynamic_preferences import serializers, types
 from dynamic_preferences.registries import global_preferences_registry
 
 
-class DefaultFromSettingMixin(object):
+class DefaultFromSettingMixin:
     def get_default(self):
         return getattr(settings, self.setting)
 
@@ -35,7 +38,7 @@ class StringListSerializer(serializers.BaseSerializer):
 
         if type(value) not in [list, tuple]:
             raise cls.exception(
-                "Cannot serialize, value {} is not a list or a tuple".format(value)
+                f"Cannot serialize, value {value} is not a list or a tuple"
             )
 
         if cls.sort:
@@ -54,6 +57,50 @@ class StringListPreference(types.BasePreferenceType):
     field_class = forms.MultipleChoiceField
 
     def get_api_additional_data(self):
-        d = super(StringListPreference, self).get_api_additional_data()
+        d = super().get_api_additional_data()
         d["choices"] = self.get("choices")
         return d
+
+
+class JSONSerializer(serializers.BaseSerializer):
+    required = True
+
+    @classmethod
+    def to_db(cls, value, **kwargs):
+        if not cls.required and value is None:
+            return json.dumps(value)
+        data_serializer = cls.data_serializer_class(data=value)
+        if not data_serializer.is_valid():
+            raise cls.exception(
+                f"{value} is not a valid value: {data_serializer.errors}"
+            )
+        value = data_serializer.validated_data
+        try:
+            return json.dumps(value, sort_keys=True)
+        except TypeError:
+            raise cls.exception(
+                f"Cannot serialize, value {value} is not JSON serializable"
+            )
+
+    @classmethod
+    def to_python(cls, value, **kwargs):
+        return json.loads(value)
+
+
+class SerializedPreference(types.BasePreferenceType):
+    """
+    A preference that store arbitrary JSON and validate it using a rest_framework
+    serializer
+    """
+
+    data_serializer_class = None
+    field_class = JSONField
+    widget = forms.Textarea
+
+    @property
+    def serializer(self):
+        class _internal(JSONSerializer):
+            data_serializer_class = self.data_serializer_class
+            required = self.get("required")
+
+        return _internal

api/funkwhale_api/common/renderers.py

+from rest_framework.renderers import JSONRenderer
+
+
+class ActivityStreamRenderer(JSONRenderer):
+    media_type = "application/activity+json"

api/funkwhale_api/common/routers.py

-from rest_framework.routers import SimpleRouter
+from rest_framework.routers import DefaultRouter
 
 
-class OptionalSlashRouter(SimpleRouter):
+class OptionalSlashRouter(DefaultRouter):
     def __init__(self):
         super().__init__()
         self.trailing_slash = "/?"

api/funkwhale_api/common/scripts/__init__.py

-from . import create_actors
-from . import create_image_variations
-from . import django_permissions_to_user_permissions
-from . import migrate_to_user_libraries
-from . import delete_pre_017_federated_uploads
-from . import test
-
+from . import (
+    create_actors,
+    delete_pre_017_federated_uploads,
+    django_permissions_to_user_permissions,
+    migrate_to_user_libraries,
+    test,
+)
 
 __all__ = [
     "create_actors",
-    "create_image_variations",
     "django_permissions_to_user_permissions",
     "migrate_to_user_libraries",
     "delete_pre_017_federated_uploads",

api/funkwhale_api/common/scripts/create_actors.py

@@ -9,15 +9,13 @@ from funkwhale_api.users.models import User, create_actor
 def main(command, **kwargs):
     qs = User.objects.filter(actor__isnull=True).order_by("username")
     total = len(qs)
-    command.stdout.write("{} users found without actors".format(total))
+    command.stdout.write(f"{total} users found without actors")
     for i, user in enumerate(qs):
-        command.stdout.write(
-            "{}/{} creating actor for {}".format(i + 1, total, user.username)
-        )
+        command.stdout.write(f"{i + 1}/{total} creating actor for {user.username}")
         try:
             user.actor = create_actor(user)
         except IntegrityError as e:
             # somehow, an actor with the the url exists in the database
-            command.stderr.write("Error while creating actor: {}".format(str(e)))
+            command.stderr.write(f"Error while creating actor: {str(e)}")
             continue
         user.save(update_fields=["actor"])

api/funkwhale_api/common/scripts/create_image_variations.py

@@ -4,16 +4,16 @@ Compute different sizes of image used for Album covers and User avatars
 
 from versatileimagefield.image_warmer import VersatileImageFieldWarmer
 
-from funkwhale_api.music.models import Album
-from funkwhale_api.users.models import User
+from funkwhale_api.common.models import Attachment
 
-
-MODELS = [(Album, "cover", "square"), (User, "avatar", "square")]
+MODELS = [
+    (Attachment, "file", "attachment_square"),
+]
 
 
 def main(command, **kwargs):
     for model, attribute, key_set in MODELS:
-        qs = model.objects.exclude(**{"{}__isnull".format(attribute): True})
+        qs = model.objects.exclude(**{f"{attribute}__isnull": True})
         qs = qs.exclude(**{attribute: ""})
         warmer = VersatileImageFieldWarmer(
             instance_or_queryset=qs,
@@ -21,10 +21,8 @@ def main(command, **kwargs):
             image_attr=attribute,
             verbose=True,
         )
-        command.stdout.write(
-            "Creating images for {} / {}".format(model.__name__, attribute)
-        )
+        command.stdout.write(f"Creating images for {model.__name__} / {attribute}")
         num_created, failed_to_create = warmer.warm()
         command.stdout.write(
-            "  {} created, {} in error".format(num_created, len(failed_to_create))
+            f"  {num_created} created, {len(failed_to_create)} in error"
         )