See #689: Fixed some performance issues with filtering on library/upload/track pages

api/funkwhale_api/common/search.py

@@ -65,6 +65,9 @@ def apply(qs, config_data):
         q = config_data.get(k)
         if q:
             qs = qs.filter(q)
+    distinct = config_data.get("distinct", False)
+    if distinct:
+        qs = qs.distinct()
     return qs
 
 
@@ -86,7 +89,19 @@ class SearchConfig:
             for t in tokens
             if t["key"] not in [None, "is", "in"] + list(self.search_fields.keys())
         ]
-        cleaned_data["filter_query"] = self.clean_filter_query(unhandled_tokens)
+        cleaned_data["filter_query"], matching_filters = self.clean_filter_query(
+            unhandled_tokens
+        )
+        if matching_filters:
+            cleaned_data["distinct"] = any(
+                [
+                    self.filter_fields[k].get("distinct", False)
+                    for k in matching_filters
+                    if k in self.filter_fields
+                ]
+            )
+        else:
+            cleaned_data["distinct"] = False
         return cleaned_data
 
     def clean_search_query(self, tokens):
@@ -128,7 +143,7 @@ class SearchConfig:
 
     def clean_filter_query(self, tokens):
         if not self.filter_fields or not tokens:
-            return
+            return None, []
 
         matching = [t for t in tokens if t["key"] in self.filter_fields]
         queries = [self.get_filter_query(token) for token in matching]
@@ -138,7 +153,7 @@ class SearchConfig:
                 query = q
             else:
                 query = query & q
-        return query
+        return query, [m["key"] for m in matching]
 
     def get_filter_query(self, token):
         raw_value = token["value"]

api/funkwhale_api/manage/filters.py

@@ -58,6 +58,7 @@ class ManageArtistFilterSet(filters.FilterSet):
                 "library_id": {
                     "to": "tracks__uploads__library_id",
                     "field": forms.IntegerField(),
+                    "distinct": True,
                 },
             },
         )
@@ -85,6 +86,7 @@ class ManageAlbumFilterSet(filters.FilterSet):
                 "library_id": {
                     "to": "tracks__uploads__library_id",
                     "field": forms.IntegerField(),
+                    "distinct": True,
                 },
             },
         )
@@ -121,6 +123,7 @@ class ManageTrackFilterSet(filters.FilterSet):
                 "library_id": {
                     "to": "uploads__library_id",
                     "field": forms.IntegerField(),
+                    "distinct": True,
                 },
             },
         )
@@ -151,12 +154,18 @@ class ManageLibraryFilterSet(filters.FilterSet):
                 "artist_id": {
                     "to": "uploads__track__artist_id",
                     "field": forms.IntegerField(),
+                    "distinct": True,
                 },
                 "album_id": {
                     "to": "uploads__track__album_id",
                     "field": forms.IntegerField(),
+                    "distinct": True,
+                },
+                "track_id": {
+                    "to": "uploads__track__id",
+                    "field": forms.IntegerField(),
+                    "distinct": True,
                 },
-                "track_id": {"to": "uploads__track__id", "field": forms.IntegerField()},
                 "domain": {"to": "actor__domain_id"},
                 "account": get_actor_filter("actor"),
                 "privacy_level": {"to": "privacy_level"},

api/funkwhale_api/manage/views.py

 from rest_framework import mixins, response, viewsets
 from rest_framework import decorators as rest_decorators
 
-from django.db.models import Count, Prefetch, Q, Sum
+from django.db.models import Count, Prefetch, Q, Sum, OuterRef, Subquery
+from django.db.models.functions import Coalesce
 from django.shortcuts import get_object_or_404
 
 from funkwhale_api.common import models as common_models
@@ -59,7 +60,6 @@ class ManageArtistViewSet(
 ):
     queryset = (
         music_models.Artist.objects.all()
-        .distinct()
         .order_by("-id")
         .select_related("attributed_to")
         .prefetch_related(
@@ -105,7 +105,6 @@ class ManageAlbumViewSet(
 ):
     queryset = (
         music_models.Album.objects.all()
-        .distinct()
         .order_by("-id")
         .select_related("attributed_to", "artist")
         .prefetch_related("tracks")
@@ -132,6 +131,15 @@ class ManageAlbumViewSet(
         return response.Response(result, status=200)
 
 
+uploads_subquery = (
+    music_models.Upload.objects.filter(track_id=OuterRef("pk"))
+    .order_by()
+    .values("track_id")
+    .annotate(track_count=Count("track_id"))
+    .values("track_count")
+)
+
+
 class ManageTrackViewSet(
     mixins.ListModelMixin,
     mixins.RetrieveModelMixin,
@@ -140,10 +148,9 @@ class ManageTrackViewSet(
 ):
     queryset = (
         music_models.Track.objects.all()
-        .distinct()
         .order_by("-id")
         .select_related("attributed_to", "artist", "album__artist")
-        .annotate(uploads_count=Count("uploads"))
+        .annotate(uploads_count=Coalesce(Subquery(uploads_subquery), 0))
     )
     serializer_class = serializers.ManageTrackSerializer
     filterset_class = filters.ManageTrackFilterSet
@@ -173,6 +180,23 @@ class ManageTrackViewSet(
         return response.Response(result, status=200)
 
 
+uploads_subquery = (
+    music_models.Upload.objects.filter(library_id=OuterRef("pk"))
+    .order_by()
+    .values("library_id")
+    .annotate(library_count=Count("library_id"))
+    .values("library_count")
+)
+
+follows_subquery = (
+    federation_models.LibraryFollow.objects.filter(target_id=OuterRef("pk"))
+    .order_by()
+    .values("target_id")
+    .annotate(library_count=Count("target_id"))
+    .values("library_count")
+)
+
+
 class ManageLibraryViewSet(
     mixins.ListModelMixin,
     mixins.RetrieveModelMixin,
@@ -182,12 +206,11 @@ class ManageLibraryViewSet(
     lookup_field = "uuid"
     queryset = (
         music_models.Library.objects.all()
-        .distinct()
         .order_by("-id")
         .select_related("actor")
         .annotate(
-            followers_count=Count("received_follows", distinct=True),
-            _uploads_count=Count("uploads", distinct=True),
+            followers_count=Coalesce(Subquery(follows_subquery), 0),
+            _uploads_count=Coalesce(Subquery(uploads_subquery), 0),
         )
     )
     serializer_class = serializers.ManageLibrarySerializer
@@ -244,7 +267,6 @@ class ManageUploadViewSet(
     lookup_field = "uuid"
     queryset = (
         music_models.Upload.objects.all()
-        .distinct()
         .order_by("-id")
         .select_related("library__actor", "track__artist", "track__album__artist")
     )