From d595d168e37b032edcbabfac8c4e21f7fbfae65f Mon Sep 17 00:00:00 2001
From: Eliot Berriot <contact@eliotberriot.com>
Date: Fri, 19 Apr 2019 14:16:23 +0200
Subject: [PATCH] See #689: Fixed some performance issues with filtering on
library/upload/track pages
---
api/funkwhale_api/common/search.py | 21 ++++++++++++---
api/funkwhale_api/manage/filters.py | 11 +++++++-
api/funkwhale_api/manage/views.py | 40 ++++++++++++++++++++++-------
3 files changed, 59 insertions(+), 13 deletions(-)
diff --git a/api/funkwhale_api/common/search.py b/api/funkwhale_api/common/search.py
index cc046f758..4e42fd346 100644
--- a/api/funkwhale_api/common/search.py
+++ b/api/funkwhale_api/common/search.py
@@ -65,6 +65,9 @@ def apply(qs, config_data):
q = config_data.get(k)
if q:
qs = qs.filter(q)
+ distinct = config_data.get("distinct", False)
+ if distinct:
+ qs = qs.distinct()
return qs
@@ -86,7 +89,19 @@ class SearchConfig:
for t in tokens
if t["key"] not in [None, "is", "in"] + list(self.search_fields.keys())
]
- cleaned_data["filter_query"] = self.clean_filter_query(unhandled_tokens)
+ cleaned_data["filter_query"], matching_filters = self.clean_filter_query(
+ unhandled_tokens
+ )
+ if matching_filters:
+ cleaned_data["distinct"] = any(
+ [
+ self.filter_fields[k].get("distinct", False)
+ for k in matching_filters
+ if k in self.filter_fields
+ ]
+ )
+ else:
+ cleaned_data["distinct"] = False
return cleaned_data
def clean_search_query(self, tokens):
@@ -128,7 +143,7 @@ class SearchConfig:
def clean_filter_query(self, tokens):
if not self.filter_fields or not tokens:
- return
+ return None, []
matching = [t for t in tokens if t["key"] in self.filter_fields]
queries = [self.get_filter_query(token) for token in matching]
@@ -138,7 +153,7 @@ class SearchConfig:
query = q
else:
query = query & q
- return query
+ return query, [m["key"] for m in matching]
def get_filter_query(self, token):
raw_value = token["value"]
diff --git a/api/funkwhale_api/manage/filters.py b/api/funkwhale_api/manage/filters.py
index c6f5db53a..f7458a5f5 100644
--- a/api/funkwhale_api/manage/filters.py
+++ b/api/funkwhale_api/manage/filters.py
@@ -58,6 +58,7 @@ class ManageArtistFilterSet(filters.FilterSet):
"library_id": {
"to": "tracks__uploads__library_id",
"field": forms.IntegerField(),
+ "distinct": True,
},
},
)
@@ -85,6 +86,7 @@ class ManageAlbumFilterSet(filters.FilterSet):
"library_id": {
"to": "tracks__uploads__library_id",
"field": forms.IntegerField(),
+ "distinct": True,
},
},
)
@@ -121,6 +123,7 @@ class ManageTrackFilterSet(filters.FilterSet):
"library_id": {
"to": "uploads__library_id",
"field": forms.IntegerField(),
+ "distinct": True,
},
},
)
@@ -151,12 +154,18 @@ class ManageLibraryFilterSet(filters.FilterSet):
"artist_id": {
"to": "uploads__track__artist_id",
"field": forms.IntegerField(),
+ "distinct": True,
},
"album_id": {
"to": "uploads__track__album_id",
"field": forms.IntegerField(),
+ "distinct": True,
+ },
+ "track_id": {
+ "to": "uploads__track__id",
+ "field": forms.IntegerField(),
+ "distinct": True,
},
- "track_id": {"to": "uploads__track__id", "field": forms.IntegerField()},
"domain": {"to": "actor__domain_id"},
"account": get_actor_filter("actor"),
"privacy_level": {"to": "privacy_level"},
diff --git a/api/funkwhale_api/manage/views.py b/api/funkwhale_api/manage/views.py
index 100e83d39..83981116c 100644
--- a/api/funkwhale_api/manage/views.py
+++ b/api/funkwhale_api/manage/views.py
@@ -1,7 +1,8 @@
from rest_framework import mixins, response, viewsets
from rest_framework import decorators as rest_decorators
-from django.db.models import Count, Prefetch, Q, Sum
+from django.db.models import Count, Prefetch, Q, Sum, OuterRef, Subquery
+from django.db.models.functions import Coalesce
from django.shortcuts import get_object_or_404
from funkwhale_api.common import models as common_models
@@ -59,7 +60,6 @@ class ManageArtistViewSet(
):
queryset = (
music_models.Artist.objects.all()
- .distinct()
.order_by("-id")
.select_related("attributed_to")
.prefetch_related(
@@ -105,7 +105,6 @@ class ManageAlbumViewSet(
):
queryset = (
music_models.Album.objects.all()
- .distinct()
.order_by("-id")
.select_related("attributed_to", "artist")
.prefetch_related("tracks")
@@ -132,6 +131,15 @@ class ManageAlbumViewSet(
return response.Response(result, status=200)
+uploads_subquery = (
+ music_models.Upload.objects.filter(track_id=OuterRef("pk"))
+ .order_by()
+ .values("track_id")
+ .annotate(track_count=Count("track_id"))
+ .values("track_count")
+)
+
+
class ManageTrackViewSet(
mixins.ListModelMixin,
mixins.RetrieveModelMixin,
@@ -140,10 +148,9 @@ class ManageTrackViewSet(
):
queryset = (
music_models.Track.objects.all()
- .distinct()
.order_by("-id")
.select_related("attributed_to", "artist", "album__artist")
- .annotate(uploads_count=Count("uploads"))
+ .annotate(uploads_count=Coalesce(Subquery(uploads_subquery), 0))
)
serializer_class = serializers.ManageTrackSerializer
filterset_class = filters.ManageTrackFilterSet
@@ -173,6 +180,23 @@ class ManageTrackViewSet(
return response.Response(result, status=200)
+uploads_subquery = (
+ music_models.Upload.objects.filter(library_id=OuterRef("pk"))
+ .order_by()
+ .values("library_id")
+ .annotate(library_count=Count("library_id"))
+ .values("library_count")
+)
+
+follows_subquery = (
+ federation_models.LibraryFollow.objects.filter(target_id=OuterRef("pk"))
+ .order_by()
+ .values("target_id")
+ .annotate(library_count=Count("target_id"))
+ .values("library_count")
+)
+
+
class ManageLibraryViewSet(
mixins.ListModelMixin,
mixins.RetrieveModelMixin,
@@ -182,12 +206,11 @@ class ManageLibraryViewSet(
lookup_field = "uuid"
queryset = (
music_models.Library.objects.all()
- .distinct()
.order_by("-id")
.select_related("actor")
.annotate(
- followers_count=Count("received_follows", distinct=True),
- _uploads_count=Count("uploads", distinct=True),
+ followers_count=Coalesce(Subquery(follows_subquery), 0),
+ _uploads_count=Coalesce(Subquery(uploads_subquery), 0),
)
)
serializer_class = serializers.ManageLibrarySerializer
@@ -244,7 +267,6 @@ class ManageUploadViewSet(
lookup_field = "uuid"
queryset = (
music_models.Upload.objects.all()
- .distinct()
.order_by("-id")
.select_related("library__actor", "track__artist", "track__album__artist")
)
--
GitLab