api/funkwhale_api/common/consumers.py

@@ -14,7 +14,15 @@ class JsonAuthConsumer(JsonWebsocketConsumer):
 
     def accept(self):
         super().accept()
-        for group in self.groups:
-            channels.group_add(group, self.channel_name)
-        for group in self.scope["user"].get_channels_groups():
+        groups = self.scope["user"].get_channels_groups() + self.groups
+        for group in groups:
             channels.group_add(group, self.channel_name)
+
+    def disconnect(self, close_code):
+        if self.scope.get("user", False) and self.scope.get("user").pk is not None:
+            groups = self.scope["user"].get_channels_groups() + self.groups
+        else:
+            groups = self.groups
+
+        for group in groups:
+            channels.group_discard(group, self.channel_name)

api/funkwhale_api/common/decorators.py

 from django.db import transaction
+from drf_spectacular.utils import OpenApiParameter, extend_schema
+from rest_framework import decorators, exceptions, response, status
 
-from rest_framework import decorators
-from rest_framework import exceptions
-from rest_framework import response
-from rest_framework import status
-
-from . import filters
-from . import models
+from . import filters, models
 from . import mutations as common_mutations
-from . import serializers
-from . import signals
-from . import tasks
-from . import utils
+from . import serializers, signals, tasks, utils
 
 
 def action_route(serializer_class):
@@ -87,6 +80,16 @@ def mutations_route(types):
             )
             return response.Response(serializer.data, status=status.HTTP_201_CREATED)
 
-    return decorators.action(
+    return extend_schema(
+        methods=["post"], responses=serializers.APIMutationSerializer()
+    )(
+        extend_schema(
+            methods=["get"],
+            responses=serializers.APIMutationSerializer(many=True),
+            parameters=[OpenApiParameter("id", location="query", exclude=True)],
+        )(
+            decorators.action(
                 methods=["get", "post"], detail=True, required_scope="edits"
             )(mutations)
+        )
+    )

api/funkwhale_api/common/dynamic_preferences_registry.py

 from dynamic_preferences import types
 from dynamic_preferences.registries import global_preferences_registry
 
-from funkwhale_api.common import preferences
-
 common = types.Section("common")
 
 
 @global_preferences_registry.register
-class APIAutenticationRequired(
-    preferences.DefaultFromSettingMixin, types.BooleanPreference
-):
+class APIAutenticationRequired(types.BooleanPreference):
     section = common
     name = "api_authentication_required"
     verbose_name = "API Requires authentication"
-    setting = "API_AUTHENTICATION_REQUIRED"
+    default = True
     help_text = (
         "If disabled, anonymous users will be able to query the API "
         "and access music data (as well as other data exposed in the API "

api/funkwhale_api/common/factories.py

 import factory
 
-from funkwhale_api.factories import registry, NoUpdateOnCreate
-
+from funkwhale_api.factories import NoUpdateOnCreate, registry
 from funkwhale_api.federation import factories as federation_factories
 
 
@@ -26,3 +25,21 @@ class AttachmentFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
 
     class Meta:
         model = "common.Attachment"
+
+
+@registry.register
+class CommonFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
+    text = factory.Faker("paragraph")
+    content_type = "text/plain"
+
+    class Meta:
+        model = "common.Content"
+
+
+@registry.register
+class PluginConfiguration(NoUpdateOnCreate, factory.django.DjangoModelFactory):
+    code = "test"
+    conf = {"foo": "bar"}
+
+    class Meta:
+        model = "common.PluginConfiguration"

api/funkwhale_api/common/fields.py

@@ -2,7 +2,6 @@ import django_filters
 from django import forms
 from django.core.serializers.json import DjangoJSONEncoder
 from django.db import models
-
 from rest_framework import serializers
 
 from . import search
@@ -25,19 +24,38 @@ def privacy_level_query(user, lookup_field="privacy_level", user_field="user"):
     if user.is_anonymous:
         return models.Q(**{lookup_field: "everyone"})
 
-    return models.Q(
-        **{"{}__in".format(lookup_field): ["instance", "everyone"]}
-    ) | models.Q(**{lookup_field: "me", user_field: user})
+    followers_query = models.Q(
+        **{
+            f"{lookup_field}": "followers",
+            f"{user_field}__actor__in": user.actor.get_approved_followings(),
+        }
+    )
+    # Federated TrackFavorite don't have an user associated with the trackfavorite.actor
+    # to do : if we implement the followers privacy_level this will become a problem
+    no_user_query = models.Q(**{f"{user_field}__isnull": True})
+
+    return (
+        models.Q(**{f"{lookup_field}__in": ["instance", "everyone"]})
+        | models.Q(**{lookup_field: "me", user_field: user})
+        | followers_query
+        | no_user_query
+    )
 
 
 class SearchFilter(django_filters.CharFilter):
     def __init__(self, *args, **kwargs):
         self.search_fields = kwargs.pop("search_fields")
+        self.fts_search_fields = kwargs.pop("fts_search_fields", [])
         super().__init__(*args, **kwargs)
 
     def filter(self, qs, value):
         if not value:
             return qs
+        if self.fts_search_fields:
+            query = search.get_fts_query(
+                value, self.fts_search_fields, model=self.parent.Meta.model
+            )
+        else:
             query = search.get_query(value, self.search_fields)
         return qs.filter(query)
 
@@ -52,7 +70,7 @@ class SmartSearchFilter(django_filters.CharFilter):
             return qs
         try:
             cleaned = self.config.clean(value)
-        except (forms.ValidationError):
+        except forms.ValidationError:
             return qs.none()
         return search.apply(qs, cleaned)
 
@@ -92,7 +110,7 @@ def get_generic_filter_query(value, relation_name, choices):
             obj = related_queryset.get(obj_query)
         except related_queryset.model.DoesNotExist:
             raise forms.ValidationError("Invalid object")
-        filter_query &= models.Q(**{"{}_id".format(relation_name): obj.id})
+        filter_query &= models.Q(**{f"{relation_name}_id": obj.id})
 
     return filter_query
 
@@ -158,7 +176,7 @@ class GenericRelation(serializers.JSONField):
             id_value = v[id_attr]
             id_value = id_field.to_internal_value(id_value)
         except (TypeError, KeyError, serializers.ValidationError):
-            raise serializers.ValidationError("Invalid {}".format(id_attr))
+            raise serializers.ValidationError(f"Invalid {id_attr}")
 
         query_getter = conf.get(
             "get_query", lambda attr, value: models.Q(**{attr: value})

api/funkwhale_api/common/filters.py

 from django import forms
 from django.db.models import Q
-
-from django_filters import widgets
+from django.db.models.functions import Lower
 from django_filters import rest_framework as filters
+from django_filters import widgets
+from drf_spectacular.utils import extend_schema_field
 
-from . import fields
-from . import models
-from . import search
+from . import fields, models, search, utils
 
 
-class NoneObject(object):
+class NoneObject:
     def __eq__(self, other):
         return other.__class__ == NoneObject
 
@@ -48,9 +47,10 @@ class CoerceChoiceField(forms.ChoiceField):
         try:
             return [b for a, b in self.choices if v == a][0]
         except IndexError:
-            raise forms.ValidationError("Invalid value {}".format(value))
+            raise forms.ValidationError(f"Invalid value {value}")
 
 
+@extend_schema_field(bool)
 class NullBooleanFilter(filters.ChoiceFilter):
     field_class = CoerceChoiceField
 
@@ -64,9 +64,7 @@ class NullBooleanFilter(filters.ChoiceFilter):
             return qs
         if value == NONE:
             value = None
-        qs = self.get_method(qs)(
-            **{"%s__%s" % (self.field_name, self.lookup_expr): value}
-        )
+        qs = self.get_method(qs)(**{f"{self.field_name}__{self.lookup_expr}": value})
         return qs.distinct() if self.distinct else qs
 
 
@@ -119,11 +117,9 @@ class MultipleQueryFilter(filters.TypedMultipleChoiceFilter):
     def __init__(self, *args, **kwargs):
         kwargs["widget"] = QueryArrayWidget()
         super().__init__(*args, **kwargs)
-        self.lookup_expr = "in"
 
 
 def filter_target(value):
-
     config = {
         "artist": ["artist", "target_id", int],
         "album": ["album", "target_id", int],
@@ -170,9 +166,14 @@ class MutationFilter(filters.FilterSet):
         fields = ["is_approved", "is_applied", "type"]
 
 
+class EmptyQuerySet(ValueError):
+    pass
+
+
 class ActorScopeFilter(filters.CharFilter):
     def __init__(self, *args, **kwargs):
         self.actor_field = kwargs.pop("actor_field")
+        self.library_field = kwargs.pop("library_field", None)
         super().__init__(*args, **kwargs)
 
     def filter(self, queryset, value):
@@ -184,21 +185,74 @@ class ActorScopeFilter(filters.CharFilter):
             return queryset.none()
 
         user = getattr(request, "user", None)
-        qs = queryset
-        if value.lower() == "me":
-            qs = self.filter_me(user=user, queryset=queryset)
-        elif value.lower() == "all":
-            return queryset
-        else:
+        actor = getattr(user, "actor", None)
+        scopes = [v.strip().lower() for v in value.split(",")]
+        query = None
+        for scope in scopes:
+            try:
+                right_query = self.get_query(scope, user, actor)
+            except ValueError:
                 return queryset.none()
+            query = utils.join_queries_or(query, right_query)
+
+        return queryset.filter(query).distinct()
+
+    def get_query(self, scope, user, actor):
+        from funkwhale_api.federation import models as federation_models
+
+        if scope == "me":
+            return self.filter_me(actor)
+        elif scope == "all":
+            return Q(pk__gte=0)
+
+        elif scope == "subscribed":
+            if not actor or self.library_field is None:
+                raise EmptyQuerySet()
+            followed_libraries = federation_models.LibraryFollow.objects.filter(
+                approved=True, actor=user.actor
+            ).values_list("target_id", flat=True)
+            if not self.library_field:
+                predicate = "pk__in"
+            else:
+                predicate = f"{self.library_field}__in"
+            return Q(**{predicate: followed_libraries})
 
-        if self.distinct:
-            qs = qs.distinct()
-        return qs
+        elif scope.startswith("actor:"):
+            full_username = scope.split("actor:", 1)[1]
+            username, domain = full_username.split("@")
+            try:
+                actor = federation_models.Actor.objects.get(
+                    preferred_username__iexact=username,
+                    domain_id=domain,
+                )
+            except federation_models.Actor.DoesNotExist:
+                raise EmptyQuerySet()
 
-    def filter_me(self, user, queryset):
-        actor = getattr(user, "actor", None)
+            return Q(**{self.actor_field: actor})
+        elif scope.startswith("domain:"):
+            domain = scope.split("domain:", 1)[1]
+            return Q(**{f"{self.actor_field}__domain_id": domain})
+        else:
+            raise EmptyQuerySet()
+
+    def filter_me(self, actor):
         if not actor:
-            return queryset.none()
+            raise EmptyQuerySet()
+
+        return Q(**{self.actor_field: actor})
+
+
+class CaseInsensitiveNameOrderingFilter(filters.OrderingFilter):
+    def filter(self, qs, value):
+        order_by = []
+
+        if value is None:
+            return qs
+
+        for param in value:
+            if param == "name":
+                order_by.append(Lower("name"))
+            else:
+                order_by.append(self.get_ordering_value(param))
 
-        return queryset.filter(**{self.actor_field: actor})
+        return qs.order_by(*order_by)

api/funkwhale_api/common/locales.py

+# from https://gist.github.com/carlopires/1262033/c52ef0f7ce4f58108619508308372edd8d0bd518
+
+ISO_639_CHOICES = [
+    ("ab", "Abkhaz"),
+    ("aa", "Afar"),
+    ("af", "Afrikaans"),
+    ("ak", "Akan"),
+    ("sq", "Albanian"),
+    ("am", "Amharic"),
+    ("ar", "Arabic"),
+    ("an", "Aragonese"),
+    ("hy", "Armenian"),
+    ("as", "Assamese"),
+    ("av", "Avaric"),
+    ("ae", "Avestan"),
+    ("ay", "Aymara"),
+    ("az", "Azerbaijani"),
+    ("bm", "Bambara"),
+    ("ba", "Bashkir"),
+    ("eu", "Basque"),
+    ("be", "Belarusian"),
+    ("bn", "Bengali"),
+    ("bh", "Bihari"),
+    ("bi", "Bislama"),
+    ("bs", "Bosnian"),
+    ("br", "Breton"),
+    ("bg", "Bulgarian"),
+    ("my", "Burmese"),
+    ("ca", "Catalan; Valencian"),
+    ("ch", "Chamorro"),
+    ("ce", "Chechen"),
+    ("ny", "Chichewa; Chewa; Nyanja"),
+    ("zh", "Chinese"),
+    ("cv", "Chuvash"),
+    ("kw", "Cornish"),
+    ("co", "Corsican"),
+    ("cr", "Cree"),
+    ("hr", "Croatian"),
+    ("cs", "Czech"),
+    ("da", "Danish"),
+    ("dv", "Divehi; Maldivian;"),
+    ("nl", "Dutch"),
+    ("dz", "Dzongkha"),
+    ("en", "English"),
+    ("eo", "Esperanto"),
+    ("et", "Estonian"),
+    ("ee", "Ewe"),
+    ("fo", "Faroese"),
+    ("fj", "Fijian"),
+    ("fi", "Finnish"),
+    ("fr", "French"),
+    ("ff", "Fula"),
+    ("gl", "Galician"),
+    ("ka", "Georgian"),
+    ("de", "German"),
+    ("el", "Greek, Modern"),
+    ("gn", "Guaraní"),
+    ("gu", "Gujarati"),
+    ("ht", "Haitian"),
+    ("ha", "Hausa"),
+    ("he", "Hebrew (modern)"),
+    ("hz", "Herero"),
+    ("hi", "Hindi"),
+    ("ho", "Hiri Motu"),
+    ("hu", "Hungarian"),
+    ("ia", "Interlingua"),
+    ("id", "Indonesian"),
+    ("ie", "Interlingue"),
+    ("ga", "Irish"),
+    ("ig", "Igbo"),
+    ("ik", "Inupiaq"),
+    ("io", "Ido"),
+    ("is", "Icelandic"),
+    ("it", "Italian"),
+    ("iu", "Inuktitut"),
+    ("ja", "Japanese"),
+    ("jv", "Javanese"),
+    ("kl", "Kalaallisut"),
+    ("kn", "Kannada"),
+    ("kr", "Kanuri"),
+    ("ks", "Kashmiri"),
+    ("kk", "Kazakh"),
+    ("km", "Khmer"),
+    ("ki", "Kikuyu, Gikuyu"),
+    ("rw", "Kinyarwanda"),
+    ("ky", "Kirghiz, Kyrgyz"),
+    ("kv", "Komi"),
+    ("kg", "Kongo"),
+    ("ko", "Korean"),
+    ("ku", "Kurdish"),
+    ("kj", "Kwanyama, Kuanyama"),
+    ("la", "Latin"),
+    ("lb", "Luxembourgish"),
+    ("lg", "Luganda"),
+    ("li", "Limburgish"),
+    ("ln", "Lingala"),
+    ("lo", "Lao"),
+    ("lt", "Lithuanian"),
+    ("lu", "Luba-Katanga"),
+    ("lv", "Latvian"),
+    ("gv", "Manx"),
+    ("mk", "Macedonian"),
+    ("mg", "Malagasy"),
+    ("ms", "Malay"),
+    ("ml", "Malayalam"),
+    ("mt", "Maltese"),
+    ("mi", "Māori"),
+    ("mr", "Marathi (Marāṭhī)"),
+    ("mh", "Marshallese"),
+    ("mn", "Mongolian"),
+    ("na", "Nauru"),
+    ("nv", "Navajo, Navaho"),
+    ("nb", "Norwegian Bokmål"),
+    ("nd", "North Ndebele"),
+    ("ne", "Nepali"),
+    ("ng", "Ndonga"),
+    ("nn", "Norwegian Nynorsk"),
+    ("no", "Norwegian"),
+    ("ii", "Nuosu"),
+    ("nr", "South Ndebele"),
+    ("oc", "Occitan"),
+    ("oj", "Ojibwe, Ojibwa"),
+    ("cu", "Old Church Slavonic"),
+    ("om", "Oromo"),
+    ("or", "Oriya"),
+    ("os", "Ossetian, Ossetic"),
+    ("pa", "Panjabi, Punjabi"),
+    ("pi", "Pāli"),
+    ("fa", "Persian"),
+    ("pl", "Polish"),
+    ("ps", "Pashto, Pushto"),
+    ("pt", "Portuguese"),
+    ("qu", "Quechua"),
+    ("rm", "Romansh"),
+    ("rn", "Kirundi"),
+    ("ro", "Romanian, Moldavan"),
+    ("ru", "Russian"),
+    ("sa", "Sanskrit (Saṁskṛta)"),
+    ("sc", "Sardinian"),
+    ("sd", "Sindhi"),
+    ("se", "Northern Sami"),
+    ("sm", "Samoan"),
+    ("sg", "Sango"),
+    ("sr", "Serbian"),
+    ("gd", "Scottish Gaelic"),
+    ("sn", "Shona"),
+    ("si", "Sinhala, Sinhalese"),
+    ("sk", "Slovak"),
+    ("sl", "Slovene"),
+    ("so", "Somali"),
+    ("st", "Southern Sotho"),
+    ("es", "Spanish; Castilian"),
+    ("su", "Sundanese"),
+    ("sw", "Swahili"),
+    ("ss", "Swati"),
+    ("sv", "Swedish"),
+    ("ta", "Tamil"),
+    ("te", "Telugu"),
+    ("tg", "Tajik"),
+    ("th", "Thai"),
+    ("ti", "Tigrinya"),
+    ("bo", "Tibetan"),
+    ("tk", "Turkmen"),
+    ("tl", "Tagalog"),
+    ("tn", "Tswana"),
+    ("to", "Tonga"),
+    ("tr", "Turkish"),
+    ("ts", "Tsonga"),
+    ("tt", "Tatar"),
+    ("tw", "Twi"),
+    ("ty", "Tahitian"),
+    ("ug", "Uighur, Uyghur"),
+    ("uk", "Ukrainian"),
+    ("ur", "Urdu"),
+    ("uz", "Uzbek"),
+    ("ve", "Venda"),
+    ("vi", "Vietnamese"),
+    ("vo", "Volapük"),
+    ("wa", "Walloon"),
+    ("cy", "Welsh"),
+    ("wo", "Wolof"),
+    ("fy", "Western Frisian"),
+    ("xh", "Xhosa"),
+    ("yi", "Yiddish"),
+    ("yo", "Yoruba"),
+    ("za", "Zhuang, Chuang"),
+    ("zu", "Zulu"),
+]
+
+
+ISO_639_BY_CODE = {code: name for code, name in ISO_639_CHOICES}

api/funkwhale_api/common/management/commands/createsuperuser.py

+from django.conf import settings
+from django.contrib.auth.management.commands.createsuperuser import (
+    Command as BaseCommand,
+)
+from django.core.management.base import CommandError
+
+
+class Command(BaseCommand):
+    def handle(self, *apps_label, **options):
+        """
+        Creating Django Superusers would bypass some of our username checks, which can lead to unexpected behaviour.
+        We therefore prohibit the execution of the command.
+        """
+        force = settings.FORCE
+        if not force == 1:
+            raise CommandError(
+                "Running createsuperuser on your Funkwhale instance bypasses some of our checks "
+                "which can lead to unexpected behavior of your instance. We therefore suggest to "
+                "run `funkwhale-manage fw users create --superuser` instead."
+            )
+
+        return super().handle(*apps_label, **options)

api/funkwhale_api/common/management/commands/gitpod.py

+import os
+
+import debugpy
+import uvicorn
+from django.core.management import call_command
+from django.core.management.commands.migrate import Command as BaseCommand
+
+from funkwhale_api.common import preferences
+from funkwhale_api.music.models import Library
+from funkwhale_api.users.models import User
+
+
+class Command(BaseCommand):
+    help = "Manage gitpod environment"
+
+    def add_arguments(self, parser):
+        parser.add_argument("command", nargs="?", type=str)
+
+    def handle(self, *args, **options):
+        command = options["command"]
+
+        if not command:
+            return self.show_help()
+
+        if command == "init":
+            return self.init()
+
+        if command == "dev":
+            return self.dev()
+
+    def show_help(self):
+        self.stdout.write("")
+        self.stdout.write("Available commands:")
+        self.stdout.write("init - Initialize gitpod workspace")
+        self.stdout.write("dev - Run Funkwhale in development mode with debug server")
+        self.stdout.write("")
+
+    def init(self):
+        user = User.objects.get(username="gitpod")
+
+        # Allow anonymous access
+        preferences.set("common__api_authentication_required", False)
+
+        # Download music catalog
+        os.system(
+            "git clone https://dev.funkwhale.audio/funkwhale/catalog.git /tmp/catalog"
+        )
+        os.system("mv -f /tmp/catalog/music /workspace/funkwhale/data")
+        os.system("rm -rf /tmp/catalog/music")
+
+        # Import music catalog into library
+        call_command(
+            "create_library",
+            "gitpod",
+            name="funkwhale/catalog",
+            privacy_level="everyone",
+        )
+        call_command(
+            "import_files",
+            Library.objects.get(actor=user.actor).uuid,
+            "/workspace/funkwhale/data/music/",
+            recursive=True,
+            in_place=True,
+            no_input=False,
+        )
+
+    def dev(self):
+        debugpy.listen(5678)
+        uvicorn.run(
+            "config.asgi:application",
+            host="0.0.0.0",
+            port=5000,
+            reload=True,
+            reload_dirs=[
+                "/workspace/funkwhale/api/config/",
+                "/workspace/funkwhale/api/funkwhale_api/",
+            ],
+        )

api/funkwhale_api/common/management/commands/inplace_to_s3.py

+import pathlib
+from argparse import RawTextHelpFormatter
+
+from django.core.management.base import BaseCommand
+from django.db import transaction
+
+from funkwhale_api.music import models
+
+
+class Command(BaseCommand):
+    help = """
+    Update the reference for Uploads that have been imported with --in-place and are now moved to s3.
+
+    Please note: This does not move any file! Make sure you already moved the files to your s3 bucket.
+
+    Specify --source to filter the reference to update to files from a specific in-place directory. If no
+    --source is given, all in-place imported track references will be updated.
+
+    Specify --target to specify a subdirectory in the S3 bucket where you moved the files. If no --target is
+    given, the file is expected to be stored in the same path as before.
+
+    Examples:
+
+    Music File: /music/Artist/Album/track.ogg
+    --source: /music
+    --target unset
+
+    All files imported from /music will be updated and expected to be in the same folder structure in the bucket
+
+    Music File: /music/Artist/Album/track.ogg
+    --source: /music
+    --target: /in_place
+
+    The music file is expected to be stored in the bucket in the directory /in_place/Artist/Album/track.ogg
+    """
+
+    def create_parser(self, *args, **kwargs):
+        parser = super().create_parser(*args, **kwargs)
+        parser.formatter_class = RawTextHelpFormatter
+        return parser
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--no-dry-run",
+            action="store_false",
+            dest="dry_run",
+            default=True,
+            help="Disable dry run mode and apply updates for real on the database",
+        )
+        parser.add_argument(
+            "--source",
+            type=pathlib.Path,
+            required=True,
+            help="Specify the path of the directory where the files originally were stored to update their reference.",
+        )
+        parser.add_argument(
+            "--target",
+            type=pathlib.Path,
+            help="Specify a subdirectory in the S3 bucket where you moved the files to.",
+        )
+
+    @transaction.atomic
+    def handle(self, *args, **options):
+        if options["dry_run"]:
+            self.stdout.write("Dry-run on, will not touch the database")
+        else:
+            self.stdout.write("Dry-run off, *changing the database*")
+        self.stdout.write("")
+
+        prefix = f"file://{options['source']}"
+
+        to_change = models.Upload.objects.filter(source__startswith=prefix)
+
+        self.stdout.write(f"Found {to_change.count()} uploads to update.")
+
+        target = options.get("target")
+        if target is None:
+            target = options["source"]
+
+        for upl in to_change:
+            upl.audio_file = str(upl.source).replace(str(prefix), str(target))
+            upl.source = None
+            self.stdout.write(f"Upload expected in {upl.audio_file}")
+            if not options["dry_run"]:
+                upl.save()
+
+        self.stdout.write("")
+        if options["dry_run"]:
+            self.stdout.write(
+                "Nothing was updated, rerun this command with --no-dry-run to apply the changes"
+            )
+        else:
+            self.stdout.write("Updating completed!")
+
+        self.stdout.write("")

api/funkwhale_api/common/management/commands/load_test_data.py

@@ -5,14 +5,12 @@ from django.conf import settings
 from django.core.management.base import BaseCommand
 from django.db import transaction
 
-
 from funkwhale_api.federation import keys
 from funkwhale_api.federation import models as federation_models
 from funkwhale_api.music import models as music_models
 from funkwhale_api.tags import models as tags_models
 from funkwhale_api.users import models as users_models
 
-
 BATCH_SIZE = 500
 
 
@@ -48,7 +46,6 @@ def create_local_accounts(factories, count, dependencies):
 
 def create_taggable_items(dependency):
     def inner(factories, count, dependencies):
-
         objs = []
         tagged_objects = dependencies.get(
             dependency, list(CONFIG_BY_ID[dependency]["model"].objects.all().only("pk"))
@@ -71,22 +68,33 @@ def create_taggable_items(dependency):
 
 
 CONFIG = [
+    {
+        "id": "artist_credit",
+        "model": music_models.ArtistCredit,
+        "factory": "music.ArtistCredit",
+        "factory_kwargs": {"joinphrase": ""},
+        "depends_on": [
+            {"field": "artist", "id": "artists", "default_factor": 0.5},
+        ],
+    },
     {
         "id": "tracks",
         "model": music_models.Track,
         "factory": "music.Track",
-        "factory_kwargs": {"artist": None, "album": None},
+        "factory_kwargs": {"album": None},
         "depends_on": [
             {"field": "album", "id": "albums", "default_factor": 0.1},
-            {"field": "artist", "id": "artists", "default_factor": 0.05},
+            {"field": "artist_credit", "id": "artist_credit", "default_factor": 0.05},
         ],
     },
     {
         "id": "albums",
         "model": music_models.Album,
         "factory": "music.Album",
-        "factory_kwargs": {"artist": None},
-        "depends_on": [{"field": "artist", "id": "artists", "default_factor": 0.3}],
+        "factory_kwargs": {},
+        "depends_on": [
+            {"field": "artist_credit", "id": "artist_credit", "default_factor": 0.3}
+        ],
     },
     {"id": "artists", "model": music_models.Artist, "factory": "music.Artist"},
     {
@@ -238,6 +246,7 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):
         from django.apps import apps
+
         from funkwhale_api import factories
 
         app_names = [app.name for app in apps.app_configs.values()]
@@ -261,7 +270,6 @@ class Command(BaseCommand):
 
         self.stdout.write("")
         if options["dry_run"]:
-
             self.stdout.write(
                 "Run this command with --no-dry-run to commit the changes to the database"
             )
@@ -313,12 +321,23 @@ class Command(BaseCommand):
                 candidates = list(queryset.values_list("pk", flat=True))
                 picked_pks = [random.choice(candidates) for _ in objects]
                 picked_objects = {o.pk: o for o in queryset.filter(pk__in=picked_pks)}
+
+            saved_obj = []
             for i, obj in enumerate(objects):
                 if create_dependencies:
                     value = random.choice(candidates)
                 else:
                     value = picked_objects[picked_pks[i]]
+                if dependency["field"] == "artist_credit":
+                    obj.save()
+                    obj.artist_credit.set([value])
+                    saved_obj.append(obj)
+
+                else:
                     setattr(obj, dependency["field"], value)
+            if saved_obj:
+                return saved_obj
+
         if not handler:
             objects = row["model"].objects.bulk_create(objects, batch_size=BATCH_SIZE)
         results[row["id"]] = objects

api/funkwhale_api/common/management/commands/makemigrations.py

-import os
-
+from django.conf import settings
 from django.core.management.base import CommandError
 from django.core.management.commands.makemigrations import Command as BaseCommand
 
@@ -11,8 +10,8 @@ class Command(BaseCommand):
 
         We ensure the command is disabled, unless a specific env var is provided.
         """
-        force = os.environ.get("FORCE") == "1"
-        if not force:
+        force = settings.FORCE
+        if not force == 1:
             raise CommandError(
                 "Running makemigrations on your Funkwhale instance can have desastrous"
                 " consequences. This command is disabled, and should only be run in "

api/funkwhale_api/common/management/commands/script.py

@@ -26,7 +26,7 @@ class Command(BaseCommand):
             script = available_scripts[name]
         except KeyError:
             raise CommandError(
-                "{} is not a valid script. Run python manage.py script for a "
+                "{} is not a valid script. Run funkwhale-manage script for a "
                 "list of available scripts".format(name)
             )
 
@@ -43,14 +43,14 @@ class Command(BaseCommand):
     def show_help(self):
         self.stdout.write("")
         self.stdout.write("Available scripts:")
-        self.stdout.write("Launch with: python manage.py <script_name>")
+        self.stdout.write("Launch with: funkwhale-manage script <script_name>")
         available_scripts = self.get_scripts()
         for name, script in sorted(available_scripts.items()):
             self.stdout.write("")
             self.stdout.write(self.style.SUCCESS(name))
             self.stdout.write("")
             for line in script["help"].splitlines():
-                self.stdout.write("     {}".format(line))
+                self.stdout.write(f"     {line}")
         self.stdout.write("")
 
     def get_scripts(self):

api/funkwhale_api/common/management/commands/testdata.py

+from django.core.management.commands.migrate import Command as BaseCommand
+
+from funkwhale_api.federation import factories
+from funkwhale_api.federation.models import Actor
+
+
+class Command(BaseCommand):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.help = "Helper to generate randomized testdata"
+        self.type_choices = {"notifications": self.handle_notifications}
+        self.missing_args_message = f"Please specify one of the following sub-commands: {*self.type_choices.keys(), }"
+
+    def add_arguments(self, parser):
+        subparsers = parser.add_subparsers(dest="subcommand")
+
+        notification_parser = subparsers.add_parser("notifications")
+        notification_parser.add_argument(
+            "username", type=str, help="Username to send the notifications to"
+        )
+        notification_parser.add_argument(
+            "--count", type=int, help="Number of elements to create", default=1
+        )
+
+    def handle(self, *args, **options):
+        self.type_choices[options["subcommand"]](options)
+
+    def handle_notifications(self, options):
+        self.stdout.write(
+            f"Create {options['count']} notification(s) for {options['username']}"
+        )
+        try:
+            actor = Actor.objects.get(preferred_username=options["username"])
+        except Actor.DoesNotExist:
+            self.stdout.write(
+                "The user you want to create notifications for does not exist"
+            )
+            return
+
+        follow_activity = factories.ActivityFactory(type="Follow")
+        for _ in range(options["count"]):
+            factories.InboxItemFactory(actor=actor, activity=follow_activity)

api/funkwhale_api/common/middleware.py

 import html
 import io
+import logging
+import os
+import re
 import time
+import tracemalloc
+import urllib.parse
 import xml.sax.saxutils
 
-from django import http
+from django import http, urls
 from django.conf import settings
+from django.contrib import auth
 from django.core.cache import caches
-from django import urls
+from django.middleware import csrf
 from rest_framework import views
 
-from . import preferences
-from . import session
-from . import throttling
-from . import utils
+from funkwhale_api.federation import utils as federation_utils
+
+from . import preferences, session, throttling, utils
 
 EXCLUDED_PATHS = ["/api", "/federation", "/.well-known"]
 
+logger = logging.getLogger(__name__)
+
 
 def should_fallback_to_spa(path):
     if path == "/":
@@ -26,6 +33,16 @@ def should_fallback_to_spa(path):
 def serve_spa(request):
     html = get_spa_html(settings.FUNKWHALE_SPA_HTML_ROOT)
     head, tail = html.split("</head>", 1)
+    if settings.FUNKWHALE_SPA_REWRITE_MANIFEST:
+        new_url = (
+            settings.FUNKWHALE_SPA_REWRITE_MANIFEST_URL
+            or federation_utils.full_url(urls.reverse("api:v1:instance:spa-manifest"))
+        )
+        title = preferences.get("instance__name")
+        if title:
+            head = replace_title(head, title)
+        head = replace_manifest_url(head, new_url)
+
     if not preferences.get("common__api_authentication_required"):
         try:
             request_tags = get_request_head_tags(request) or []
@@ -61,23 +78,55 @@ def serve_spa(request):
         # We add the style add the end of the body to ensure it has the highest
         # priority (since it will come after other stylesheets)
         body, tail = tail.split("</body>", 1)
-        css = "<style>{}</style>".format(css)
+        css = f"<style>{css}</style>"
         tail = body + "\n" + css + "\n</body>" + tail
-    return http.HttpResponse(head + tail)
+
+    # set a csrf token so that visitor can login / query API if needed
+    token = csrf.get_token(request)
+    response = http.HttpResponse(head + tail)
+    response.set_cookie("csrftoken", token, max_age=None)
+    return response
+
+
+MANIFEST_LINK_REGEX = re.compile(r"<link [^>]*rel=(?:'|\")?manifest(?:'|\")?[^>]*>")
+TITLE_REGEX = re.compile(r"<title>.*</title>")
+
+
+def replace_manifest_url(head, new_url):
+    replacement = f'<link rel=manifest href="{new_url}">'
+    head = MANIFEST_LINK_REGEX.sub(replacement, head)
+    return head
+
+
+def replace_title(head, new_title):
+    replacement = f"<title>{html.escape(new_title)}</title>"
+    head = TITLE_REGEX.sub(replacement, head)
+    return head
 
 
 def get_spa_html(spa_url):
+    return get_spa_file(spa_url, "index.html")
+
+
+def get_spa_file(spa_url, name):
     if spa_url.startswith("/"):
+        # spa_url is an absolute path to index.html, on the local disk.
+        # However, we may want to access manifest.json or other files as well, so we
+        # strip the filename
+        path = os.path.join(os.path.dirname(spa_url), name)
         # we try to open a local file
-        with open(spa_url) as f:
-            return f.read()
-    cache_key = "spa-html:{}".format(spa_url)
+        with open(path, "rb") as f:
+            return f.read().decode("utf-8")
+    cache_key = f"spa-file:{spa_url}:{name}"
     cached = caches["local"].get(cache_key)
     if cached:
         return cached
 
-    response = session.get_session().get(utils.join_url(spa_url, "index.html"),)
+    response = session.get_session().get(
+        utils.join_url(spa_url, name),
+    )
     response.raise_for_status()
+    response.encoding = "utf-8"
     content = response.text
     caches["local"].set(cache_key, content, settings.FUNKWHALE_SPA_HTML_CACHE_DURATION)
     return content
@@ -101,7 +150,9 @@ def get_default_head_tags(path):
         {
             "tag": "meta",
             "property": "og:image",
-            "content": utils.join_url(settings.FUNKWHALE_URL, "/front/favicon.png"),
+            "content": utils.join_url(
+                settings.FUNKWHALE_URL, "/android-chrome-512x512.png"
+            ),
         },
         {
             "tag": "meta",
@@ -118,22 +169,25 @@ def render_tags(tags):
     <meta hello="world" />
     """
     for tag in tags:
-
         yield "<{tag} {attrs} />".format(
             tag=tag.pop("tag"),
             attrs=" ".join(
-                [
-                    '{}="{}"'.format(a, html.escape(str(v)))
-                    for a, v in sorted(tag.items())
-                    if v
-                ]
+                [f'{a}="{html.escape(str(v))}"' for a, v in sorted(tag.items()) if v]
             ),
         )
 
 
 def get_request_head_tags(request):
+    accept_header = request.headers.get("Accept") or None
+    redirect_to_ap = (
+        False
+        if not accept_header
+        else not federation_utils.should_redirect_ap_to_html(accept_header)
+    )
     match = urls.resolve(request.path, urlconf=settings.SPA_URLCONF)
-    return match.func(request, *match.args, **match.kwargs)
+    return match.func(
+        request, *match.args, redirect_to_ap=redirect_to_ap, **match.kwargs
+    )
 
 
 def get_custom_css():
@@ -144,6 +198,31 @@ def get_custom_css():
     return xml.sax.saxutils.escape(css)
 
 
+class ApiRedirect(Exception):
+    def __init__(self, url):
+        self.url = url
+
+
+def get_api_response(request, url):
+    """
+    Quite ugly but we have no choice. When Accept header is set to application/activity+json
+    some clients expect to get a JSON payload (instead of the HTML we return). Since
+    redirecting to the URL does not work (because it makes the signature verification fail),
+    we grab the internal view corresponding to the URL, call it and return this as the
+    response
+    """
+    path = urllib.parse.urlparse(url).path
+
+    try:
+        match = urls.resolve(path)
+    except urls.exceptions.Resolver404:
+        return http.HttpResponseNotFound()
+    response = match.func(request, *match.args, **match.kwargs)
+    if hasattr(response, "render"):
+        response.render()
+    return response
+
+
 class SPAFallbackMiddleware:
     def __init__(self, get_response):
         self.get_response = get_response
@@ -152,7 +231,10 @@ class SPAFallbackMiddleware:
         response = self.get_response(request)
 
         if response.status_code == 404 and should_fallback_to_spa(request.path):
+            try:
                 return serve_spa(request)
+            except ApiRedirect as e:
+                return get_api_response(request, e.url)
 
         return response
 
@@ -198,6 +280,25 @@ def monkey_patch_rest_initialize_request():
 monkey_patch_rest_initialize_request()
 
 
+def monkey_patch_auth_get_user():
+    """
+    We need an actor on our users for many endpoints, so we monkey patch
+    auth.get_user to create it if it's missing
+    """
+    original = auth.get_user
+
+    def replacement(request):
+        r = original(request)
+        if not r.is_anonymous and not r.actor:
+            r.create_actor()
+        return r
+
+    setattr(auth, "get_user", replacement)
+
+
+monkey_patch_auth_get_user()
+
+
 class ThrottleStatusMiddleware:
     """
     Include useful information regarding throttling in API responses to
@@ -242,6 +343,17 @@ class ThrottleStatusMiddleware:
         return response
 
 
+class VerboseBadRequestsMiddleware:
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        response = self.get_response(request)
+        if response.status_code == 400:
+            logger.warning("Bad request: %s", response.content)
+        return response
+
+
 class ProfilerMiddleware:
     """
     from https://github.com/omarish/django-cprofile-middleware/blob/master/django_cprofile_middleware/middleware.py
@@ -290,3 +402,19 @@ class ProfilerMiddleware:
         response = http.HttpResponse("<pre>%s</pre>" % stream.getvalue())
 
         return response
+
+
+class PymallocMiddleware:
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        if tracemalloc.is_tracing():
+            snapshot = tracemalloc.take_snapshot()
+            stats = snapshot.statistics("lineno")
+
+            print("Memory trace")
+            for stat in stats[:25]:
+                print(stat)
+
+        return self.get_response(request)

api/funkwhale_api/common/migrations/0002_mutation.py

@@ -36,8 +36,8 @@ class Migration(migrations.Migration):
                     models.UUIDField(db_index=True, default=uuid.uuid4, unique=True),
                 ),
                 ("type", models.CharField(db_index=True, max_length=100)),
-                ("is_approved", models.NullBooleanField(default=None)),
-                ("is_applied", models.NullBooleanField(default=None)),
+                ("is_approved", models.BooleanField(default=None, null=True)),
+                ("is_applied", models.BooleanField(default=None, null=True)),
                 (
                     "creation_date",
                     models.DateTimeField(

api/funkwhale_api/common/migrations/0006_content.py

+# Generated by Django 2.2.7 on 2020-01-13 10:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('common', '0005_auto_20191125_1421'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Content',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('text', models.CharField(blank=True, max_length=5000, null=True)),
+                ('content_type', models.CharField(max_length=100)),
+            ],
+        ),
+    ]

api/funkwhale_api/common/migrations/0007_auto_20200116_1610.py

+# Generated by Django 2.2.9 on 2020-01-16 16:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('common', '0006_content'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='attachment',
+            name='url',
+            field=models.URLField(max_length=500, null=True),
+        ),
+    ]

api/funkwhale_api/common/migrations/0008_auto_20200701_1317.py

+# Generated by Django 3.0.8 on 2020-07-01 13:17
+
+from django.conf import settings
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('common', '0007_auto_20200116_1610'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='attachment',
+            name='url',
+            field=models.URLField(blank=True, max_length=500, null=True),
+        ),
+        migrations.CreateModel(
+            name='PluginConfiguration',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('code', models.CharField(max_length=100)),
+                ('conf', django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True)),
+                ('enabled', models.BooleanField(default=False)),
+                ('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
+                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='plugins', to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'unique_together': {('user', 'code')},
+            },
+        ),
+    ]

api/funkwhale_api/common/migrations/0009_auto_20220627_1915.py

+# Generated by Django 3.2.13 on 2022-06-27 19:15
+
+import django.core.serializers.json
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('common', '0008_auto_20200701_1317'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='mutation',
+            name='is_applied',
+            field=models.BooleanField(default=None, null=True),
+        ),
+        migrations.AlterField(
+            model_name='mutation',
+            name='is_approved',
+            field=models.BooleanField(default=None, null=True),
+        ),
+        migrations.AlterField(
+            model_name='mutation',
+            name='payload',
+            field=models.JSONField(encoder=django.core.serializers.json.DjangoJSONEncoder),
+        ),
+        migrations.AlterField(
+            model_name='mutation',
+            name='previous_state',
+            field=models.JSONField(default=None, encoder=django.core.serializers.json.DjangoJSONEncoder, null=True),
+        ),
+        migrations.AlterField(
+            model_name='pluginconfiguration',
+            name='conf',
+            field=models.JSONField(blank=True, null=True),
+        ),
+    ]