api/funkwhale_api/music/models.py

@@ -178,12 +178,13 @@ class ArtistQuerySet(common_models.LocalFromFidQuerySet, models.QuerySet):
         return self.annotate(is_playable_by_actor=subquery)
 
     def playable_by(self, actor, include=True):
-        tracks = Track.objects.playable_by(actor, include)
-        matches = self.filter(tracks__in=tracks).values_list("pk")
+        query = get_viewable_by_query(actor, "tracks__uploads__library") & models.Q(
+            tracks__uploads__import_status="finished"
+        )
         if include:
-            return self.filter(pk__in=matches)
+            return self.filter(query)
         else:
-            return self.exclude(pk__in=matches)
+            return self.exclude(query)
 
 
 class Artist(APIModelMixin):
@@ -262,12 +263,13 @@ class AlbumQuerySet(common_models.LocalFromFidQuerySet, models.QuerySet):
         return self.annotate(is_playable_by_actor=subquery)
 
     def playable_by(self, actor, include=True):
-        tracks = Track.objects.playable_by(actor, include)
-        matches = self.filter(tracks__in=tracks).values_list("pk")
+        query = get_viewable_by_query(actor, "tracks__uploads__library") & models.Q(
+            tracks__uploads__import_status="finished"
+        )
         if include:
-            return self.filter(pk__in=matches)
+            return self.filter(query)
         else:
-            return self.exclude(pk__in=matches)
+            return self.exclude(query)
 
     def with_prefetched_tracks_and_playable_uploads(self, actor):
         tracks = Track.objects.with_playable_uploads(actor)
@@ -418,12 +420,13 @@ class TrackQuerySet(common_models.LocalFromFidQuerySet, models.QuerySet):
         return self.annotate(is_playable_by_actor=subquery)
 
     def playable_by(self, actor, include=True):
-        files = Upload.objects.playable_by(actor, include)
-        matches = self.filter(uploads__in=files).values_list("pk")
+        query = get_viewable_by_query(actor, "uploads__library") & models.Q(
+            uploads__import_status="finished"
+        )
         if include:
-            return self.filter(pk__in=matches)
+            return self.filter(query)
         else:
-            return self.exclude(pk__in=matches)
+            return self.exclude(query)
 
     def with_playable_uploads(self, actor):
         uploads = Upload.objects.playable_by(actor).select_related("track")
@@ -598,11 +601,10 @@ class Track(APIModelMixin):
 
 class UploadQuerySet(common_models.NullsLastQuerySet):
     def playable_by(self, actor, include=True):
-        libraries = Library.objects.viewable_by(actor)
-
+        query = get_viewable_by_query(actor, "library")
         if include:
-            return self.filter(library__in=libraries, import_status="finished")
-        return self.exclude(library__in=libraries, import_status="finished")
+            return self.filter(query, import_status="finished")
+        return self.exclude(query, import_status="finished")
 
     def local(self, include=True):
         return self.exclude(library__actor__user__isnull=include)
@@ -1025,34 +1027,50 @@ LIBRARY_PRIVACY_LEVEL_CHOICES = [
 ]
 
 
-class LibraryQuerySet(models.QuerySet):
-    def with_follows(self, actor):
-        return self.prefetch_related(
-            models.Prefetch(
-                "received_follows",
-                queryset=federation_models.LibraryFollow.objects.filter(actor=actor),
-                to_attr="_follows",
-            )
-        )
+def prefixed_query(path, **kwargs):
+    if not path:
+        return models.Q(**kwargs)
 
-    def viewable_by(self, actor):
+    prefixed_kwargs = {
+        "{}__{}".format(path, key): value for key, value in kwargs.items()
+    }
+    return models.Q(**prefixed_kwargs)
+
+
+def get_viewable_by_query(actor, path=None):
     from funkwhale_api.federation.models import LibraryFollow
 
     if actor is None:
-            return Library.objects.filter(privacy_level="everyone")
+        return prefixed_query(path, privacy_level="everyone")
 
-        me_query = models.Q(privacy_level="me", actor=actor)
-        instance_query = models.Q(privacy_level="instance", actor__domain=actor.domain)
+    me_query = prefixed_query(path, privacy_level="me", actor=actor)
+    instance_query = prefixed_query(
+        path, privacy_level="instance", actor__domain=actor.domain
+    )
     followed_libraries = LibraryFollow.objects.filter(
         actor=actor, approved=True
     ).values_list("target", flat=True)
-        return Library.objects.filter(
+    return (
         me_query
         | instance_query
-            | models.Q(privacy_level="everyone")
-            | models.Q(pk__in=followed_libraries)
+        | prefixed_query(path, privacy_level="everyone")
+        | prefixed_query(path, pk__in=followed_libraries)
+    )
+
+
+class LibraryQuerySet(models.QuerySet):
+    def with_follows(self, actor):
+        return self.prefetch_related(
+            models.Prefetch(
+                "received_follows",
+                queryset=federation_models.LibraryFollow.objects.filter(actor=actor),
+                to_attr="_follows",
+            )
         )
 
+    def viewable_by(self, actor):
+        return self.filter(get_viewable_by_query(actor))
+
 
 class Library(federation_models.FederationMixin):
     uuid = models.UUIDField(unique=True, db_index=True, default=uuid.uuid4)

api/requirements/test.txt

@@ -11,3 +11,4 @@ pytest-env
 requests-mock
 pytest-randomly
 pytest-profiling<1.4
+django-perf-rec

api/tests/music/test_models.perf.yml

+test_album_playable_by_anonymous:
+- db: 'SELECT ... FROM "music_album" INNER JOIN "music_track" ON ("music_album"."id" = "music_track"."album_id") INNER JOIN "music_upload" ON ("music_track"."id" = "music_upload"."track_id") INNER JOIN "music_library" ON ("music_upload"."library_id" = "music_library"."id") WHERE ("music_library"."privacy_level" = # AND "music_upload"."import_status" = #) ORDER BY "music_album"."creation_date" DESC'
+test_album_playable_by_anonymous.2:
+- db: 'SELECT ... FROM "music_album" INNER JOIN "music_track" ON ("music_album"."id" = "music_track"."album_id") INNER JOIN "music_upload" ON ("music_track"."id" = "music_upload"."track_id") INNER JOIN "music_library" ON ("music_upload"."library_id" = "music_library"."id") WHERE ("music_library"."privacy_level" = # AND "music_upload"."import_status" = #) ORDER BY "music_album"."creation_date" DESC'
+test_album_playable_by_correct_actor:
+- db: 'SELECT ... FROM "music_album" INNER JOIN "music_track" ON ("music_album"."id" = "music_track"."album_id") INNER JOIN "music_upload" ON ("music_track"."id" = "music_upload"."track_id") INNER JOIN "music_library" ON ("music_upload"."library_id" = "music_library"."id") INNER JOIN "federation_actor" ON ("music_library"."actor_id" = "federation_actor"."id") WHERE ((("music_library"."actor_id" = # AND "music_library"."privacy_level" = #) OR ("federation_actor"."domain_id" = # AND "music_library"."privacy_level" = #) OR "music_library"."privacy_level" = # OR "music_upload"."library_id" IN (SELECT U0."target_id" FROM "federation_libraryfollow" U0 WHERE (U0."actor_id" = # AND U0."approved" = true))) AND "music_upload"."import_status" = #) ORDER BY "music_album"."creation_date" DESC'
+test_album_playable_by_instance_actor:
+- db: 'SELECT ... FROM "music_album" INNER JOIN "music_track" ON ("music_album"."id" = "music_track"."album_id") INNER JOIN "music_upload" ON ("music_track"."id" = "music_upload"."track_id") INNER JOIN "music_library" ON ("music_upload"."library_id" = "music_library"."id") INNER JOIN "federation_actor" ON ("music_library"."actor_id" = "federation_actor"."id") WHERE ((("music_library"."actor_id" = # AND "music_library"."privacy_level" = #) OR ("federation_actor"."domain_id" = # AND "music_library"."privacy_level" = #) OR "music_library"."privacy_level" = # OR "music_upload"."library_id" IN (SELECT U0."target_id" FROM "federation_libraryfollow" U0 WHERE (U0."actor_id" = # AND U0."approved" = true))) AND "music_upload"."import_status" = #) ORDER BY "music_album"."creation_date" DESC'
+test_artist_playable_by_anonymous:
+- db: 'SELECT ... FROM "music_artist" INNER JOIN "music_track" ON ("music_artist"."id" = "music_track"."artist_id") INNER JOIN "music_upload" ON ("music_track"."id" = "music_upload"."track_id") INNER JOIN "music_library" ON ("music_upload"."library_id" = "music_library"."id") WHERE ("music_library"."privacy_level" = # AND "music_upload"."import_status" = #) ORDER BY "music_artist"."creation_date" DESC'
+test_artist_playable_by_correct_actor:
+- db: 'SELECT ... FROM "music_artist" INNER JOIN "music_track" ON ("music_artist"."id" = "music_track"."artist_id") INNER JOIN "music_upload" ON ("music_track"."id" = "music_upload"."track_id") INNER JOIN "music_library" ON ("music_upload"."library_id" = "music_library"."id") INNER JOIN "federation_actor" ON ("music_library"."actor_id" = "federation_actor"."id") WHERE ((("music_library"."actor_id" = # AND "music_library"."privacy_level" = #) OR ("federation_actor"."domain_id" = # AND "music_library"."privacy_level" = #) OR "music_library"."privacy_level" = # OR "music_upload"."library_id" IN (SELECT U0."target_id" FROM "federation_libraryfollow" U0 WHERE (U0."actor_id" = # AND U0."approved" = true))) AND "music_upload"."import_status" = #) ORDER BY "music_artist"."creation_date" DESC'
+test_artist_playable_by_instance_actor:
+- db: 'SELECT ... FROM "music_artist" INNER JOIN "music_track" ON ("music_artist"."id" = "music_track"."artist_id") INNER JOIN "music_upload" ON ("music_track"."id" = "music_upload"."track_id") INNER JOIN "music_library" ON ("music_upload"."library_id" = "music_library"."id") INNER JOIN "federation_actor" ON ("music_library"."actor_id" = "federation_actor"."id") WHERE ((("music_library"."actor_id" = # AND "music_library"."privacy_level" = #) OR ("federation_actor"."domain_id" = # AND "music_library"."privacy_level" = #) OR "music_library"."privacy_level" = # OR "music_upload"."library_id" IN (SELECT U0."target_id" FROM "federation_libraryfollow" U0 WHERE (U0."actor_id" = # AND U0."approved" = true))) AND "music_upload"."import_status" = #) ORDER BY "music_artist"."creation_date" DESC'
+test_artist_playable_by_instance_actor.2:
+- db: 'SELECT ... FROM "music_artist" INNER JOIN "music_track" ON ("music_artist"."id" = "music_track"."artist_id") INNER JOIN "music_upload" ON ("music_track"."id" = "music_upload"."track_id") INNER JOIN "music_library" ON ("music_upload"."library_id" = "music_library"."id") INNER JOIN "federation_actor" ON ("music_library"."actor_id" = "federation_actor"."id") WHERE ((("music_library"."actor_id" = # AND "music_library"."privacy_level" = #) OR ("federation_actor"."domain_id" = # AND "music_library"."privacy_level" = #) OR "music_library"."privacy_level" = # OR "music_upload"."library_id" IN (SELECT U0."target_id" FROM "federation_libraryfollow" U0 WHERE (U0."actor_id" = # AND U0."approved" = true))) AND "music_upload"."import_status" = #) ORDER BY "music_artist"."creation_date" DESC'
+test_playable_by_anonymous:
+- db: 'SELECT ... FROM "music_upload" INNER JOIN "music_library" ON ("music_upload"."library_id" = "music_library"."id") WHERE ("music_upload"."library_id" = # AND "music_library"."privacy_level" = # AND "music_upload"."import_status" = #)'
+test_playable_by_correct_actor:
+- db: 'SELECT ... FROM "music_upload" INNER JOIN "music_library" ON ("music_upload"."library_id" = "music_library"."id") INNER JOIN "federation_actor" ON ("music_library"."actor_id" = "federation_actor"."id") WHERE ("music_upload"."library_id" = # AND (("music_library"."actor_id" = # AND "music_library"."privacy_level" = #) OR ("federation_actor"."domain_id" = # AND "music_library"."privacy_level" = #) OR "music_library"."privacy_level" = # OR "music_upload"."library_id" IN (SELECT U0."target_id" FROM "federation_libraryfollow" U0 WHERE (U0."actor_id" = # AND U0."approved" = true))) AND "music_upload"."import_status" = #)'
+test_playable_by_correct_actor.2:
+- db: 'SELECT ... FROM "music_upload" INNER JOIN "music_library" ON ("music_upload"."library_id" = "music_library"."id") INNER JOIN "federation_actor" ON ("music_library"."actor_id" = "federation_actor"."id") WHERE ("music_upload"."library_id" = # AND (("music_library"."actor_id" = # AND "music_library"."privacy_level" = #) OR ("federation_actor"."domain_id" = # AND "music_library"."privacy_level" = #) OR "music_library"."privacy_level" = # OR "music_upload"."library_id" IN (SELECT U0."target_id" FROM "federation_libraryfollow" U0 WHERE (U0."actor_id" = # AND U0."approved" = true))) AND "music_upload"."import_status" = #)'
+test_playable_by_correct_status:
+- db: 'SELECT ... FROM "music_upload" INNER JOIN "music_library" ON ("music_upload"."library_id" = "music_library"."id") WHERE ("music_upload"."library_id" = # AND "music_library"."privacy_level" = # AND "music_upload"."import_status" = #)'
+test_playable_by_follower:
+- db: 'SELECT ... FROM "music_upload" INNER JOIN "music_library" ON ("music_upload"."library_id" = "music_library"."id") INNER JOIN "federation_actor" ON ("music_library"."actor_id" = "federation_actor"."id") WHERE ("music_upload"."library_id" = # AND (("music_library"."actor_id" = # AND "music_library"."privacy_level" = #) OR ("federation_actor"."domain_id" = # AND "music_library"."privacy_level" = #) OR "music_library"."privacy_level" = # OR "music_upload"."library_id" IN (SELECT U0."target_id" FROM "federation_libraryfollow" U0 WHERE (U0."actor_id" = # AND U0."approved" = true))) AND "music_upload"."import_status" = #)'
+test_playable_by_instance_actor:
+- db: 'SELECT ... FROM "music_upload" INNER JOIN "music_library" ON ("music_upload"."library_id" = "music_library"."id") INNER JOIN "federation_actor" ON ("music_library"."actor_id" = "federation_actor"."id") WHERE ("music_upload"."library_id" = # AND (("music_library"."actor_id" = # AND "music_library"."privacy_level" = #) OR ("federation_actor"."domain_id" = # AND "music_library"."privacy_level" = #) OR "music_library"."privacy_level" = # OR "music_upload"."library_id" IN (SELECT U0."target_id" FROM "federation_libraryfollow" U0 WHERE (U0."actor_id" = # AND U0."approved" = true))) AND "music_upload"."import_status" = #)'
+test_playable_by_instance_actor.2:
+- db: 'SELECT ... FROM "music_upload" INNER JOIN "music_library" ON ("music_upload"."library_id" = "music_library"."id") INNER JOIN "federation_actor" ON ("music_library"."actor_id" = "federation_actor"."id") WHERE ("music_upload"."library_id" = # AND (("music_library"."actor_id" = # AND "music_library"."privacy_level" = #) OR ("federation_actor"."domain_id" = # AND "music_library"."privacy_level" = #) OR "music_library"."privacy_level" = # OR "music_upload"."library_id" IN (SELECT U0."target_id" FROM "federation_libraryfollow" U0 WHERE (U0."actor_id" = # AND U0."approved" = true))) AND "music_upload"."import_status" = #)'
+test_track_playable_by_anonymous:
+- db: 'SELECT ... FROM "music_track" INNER JOIN "music_upload" ON ("music_track"."id" = "music_upload"."track_id") INNER JOIN "music_library" ON ("music_upload"."library_id" = "music_library"."id") LEFT OUTER JOIN "music_album" ON ("music_track"."album_id" = "music_album"."id") WHERE ("music_library"."privacy_level" = # AND "music_upload"."import_status" = #) ORDER BY "music_album"."creation_date" DESC, "music_track"."disc_number" ASC, "music_track"."position" ASC'
+test_track_playable_by_correct_actor:
+- db: 'SELECT ... FROM "music_track" INNER JOIN "music_upload" ON ("music_track"."id" = "music_upload"."track_id") INNER JOIN "music_library" ON ("music_upload"."library_id" = "music_library"."id") INNER JOIN "federation_actor" ON ("music_library"."actor_id" = "federation_actor"."id") LEFT OUTER JOIN "music_album" ON ("music_track"."album_id" = "music_album"."id") WHERE ((("music_library"."actor_id" = # AND "music_library"."privacy_level" = #) OR ("federation_actor"."domain_id" = # AND "music_library"."privacy_level" = #) OR "music_library"."privacy_level" = # OR "music_upload"."library_id" IN (SELECT U0."target_id" FROM "federation_libraryfollow" U0 WHERE (U0."actor_id" = # AND U0."approved" = true))) AND "music_upload"."import_status" = #) ORDER BY "music_album"."creation_date" DESC, "music_track"."disc_number" ASC, "music_track"."position" ASC'
+test_track_playable_by_instance_actor:
+- db: 'SELECT ... FROM "music_track" INNER JOIN "music_upload" ON ("music_track"."id" = "music_upload"."track_id") INNER JOIN "music_library" ON ("music_upload"."library_id" = "music_library"."id") INNER JOIN "federation_actor" ON ("music_library"."actor_id" = "federation_actor"."id") LEFT OUTER JOIN "music_album" ON ("music_track"."album_id" = "music_album"."id") WHERE ((("music_library"."actor_id" = # AND "music_library"."privacy_level" = #) OR ("federation_actor"."domain_id" = # AND "music_library"."privacy_level" = #) OR "music_library"."privacy_level" = # OR "music_upload"."library_id" IN (SELECT U0."target_id" FROM "federation_libraryfollow" U0 WHERE (U0."actor_id" = # AND U0."approved" = true))) AND "music_upload"."import_status" = #) ORDER BY "music_album"."creation_date" DESC, "music_track"."disc_number" ASC, "music_track"."position" ASC'
+test_track_playable_by_instance_actor.2:
+- db: 'SELECT ... FROM "music_track" INNER JOIN "music_upload" ON ("music_track"."id" = "music_upload"."track_id") INNER JOIN "music_library" ON ("music_upload"."library_id" = "music_library"."id") INNER JOIN "federation_actor" ON ("music_library"."actor_id" = "federation_actor"."id") LEFT OUTER JOIN "music_album" ON ("music_track"."album_id" = "music_album"."id") WHERE ((("music_library"."actor_id" = # AND "music_library"."privacy_level" = #) OR ("federation_actor"."domain_id" = # AND "music_library"."privacy_level" = #) OR "music_library"."privacy_level" = # OR "music_upload"."library_id" IN (SELECT U0."target_id" FROM "federation_libraryfollow" U0 WHERE (U0."actor_id" = # AND U0."approved" = true))) AND "music_upload"."import_status" = #) ORDER BY "music_album"."creation_date" DESC, "music_track"."disc_number" ASC, "music_track"."position" ASC'

api/tests/music/test_models.py

 import os
 
+import django_perf_rec
 import pytest
 
 from django.utils import timezone
@@ -215,6 +216,7 @@ def test_playable_by_correct_status(status, expected, factories):
         library__privacy_level="everyone", import_status=status
     )
     queryset = upload.library.uploads.playable_by(None)
+    with django_perf_rec.record():
         match = upload in list(queryset)
     assert match is expected
 
@@ -227,6 +229,7 @@ def test_playable_by_correct_actor(privacy_level, expected, factories):
         library__privacy_level=privacy_level, import_status="finished"
     )
     queryset = upload.library.uploads.playable_by(upload.library.actor)
+    with django_perf_rec.record():
         match = upload in list(queryset)
     assert match is expected
 
@@ -240,6 +243,7 @@ def test_playable_by_instance_actor(privacy_level, expected, factories):
     )
     instance_actor = factories["federation.Actor"](domain=upload.library.actor.domain)
     queryset = upload.library.uploads.playable_by(instance_actor)
+    with django_perf_rec.record():
         match = upload in list(queryset)
     assert match is expected
 
@@ -252,6 +256,7 @@ def test_playable_by_anonymous(privacy_level, expected, factories):
         library__privacy_level=privacy_level, import_status="finished"
     )
     queryset = upload.library.uploads.playable_by(None)
+    with django_perf_rec.record():
         match = upload in list(queryset)
     assert match is expected
 
@@ -266,6 +271,7 @@ def test_playable_by_follower(approved, factories):
         target=upload.library, actor=actor, approved=approved
     )
     queryset = upload.library.uploads.playable_by(actor)
+    with django_perf_rec.record():
         match = upload in list(queryset)
     expected = approved
     assert match is expected
@@ -279,6 +285,7 @@ def test_track_playable_by_correct_actor(privacy_level, expected, factories):
     queryset = models.Track.objects.playable_by(
         upload.library.actor
     ).annotate_playable_by_actor(upload.library.actor)
+    with django_perf_rec.record():
         match = upload.track in list(queryset)
     assert match is expected
     if expected:
@@ -296,6 +303,7 @@ def test_track_playable_by_instance_actor(privacy_level, expected, factories):
     queryset = models.Track.objects.playable_by(
         instance_actor
     ).annotate_playable_by_actor(instance_actor)
+    with django_perf_rec.record():
         match = upload.track in list(queryset)
     assert match is expected
     if expected:
@@ -310,6 +318,7 @@ def test_track_playable_by_anonymous(privacy_level, expected, factories):
         library__privacy_level=privacy_level, import_status="finished"
     )
     queryset = models.Track.objects.playable_by(None).annotate_playable_by_actor(None)
+    with django_perf_rec.record():
         match = upload.track in list(queryset)
     assert match is expected
     if expected:
@@ -325,6 +334,7 @@ def test_album_playable_by_correct_actor(privacy_level, expected, factories):
     queryset = models.Album.objects.playable_by(
         upload.library.actor
     ).annotate_playable_by_actor(upload.library.actor)
+    with django_perf_rec.record():
         match = upload.track.album in list(queryset)
     assert match is expected
     if expected:
@@ -342,6 +352,7 @@ def test_album_playable_by_instance_actor(privacy_level, expected, factories):
     queryset = models.Album.objects.playable_by(
         instance_actor
     ).annotate_playable_by_actor(instance_actor)
+    with django_perf_rec.record():
         match = upload.track.album in list(queryset)
     assert match is expected
     if expected:
@@ -356,6 +367,7 @@ def test_album_playable_by_anonymous(privacy_level, expected, factories):
         library__privacy_level=privacy_level, import_status="finished"
     )
     queryset = models.Album.objects.playable_by(None).annotate_playable_by_actor(None)
+    with django_perf_rec.record():
         match = upload.track.album in list(queryset)
     assert match is expected
     if expected:
@@ -371,6 +383,7 @@ def test_artist_playable_by_correct_actor(privacy_level, expected, factories):
     queryset = models.Artist.objects.playable_by(
         upload.library.actor
     ).annotate_playable_by_actor(upload.library.actor)
+    with django_perf_rec.record():
         match = upload.track.artist in list(queryset)
     assert match is expected
     if expected:
@@ -388,6 +401,7 @@ def test_artist_playable_by_instance_actor(privacy_level, expected, factories):
     queryset = models.Artist.objects.playable_by(
         instance_actor
     ).annotate_playable_by_actor(instance_actor)
+    with django_perf_rec.record():
         match = upload.track.artist in list(queryset)
     assert match is expected
     if expected:
@@ -402,6 +416,7 @@ def test_artist_playable_by_anonymous(privacy_level, expected, factories):
         library__privacy_level=privacy_level, import_status="finished"
     )
     queryset = models.Artist.objects.playable_by(None).annotate_playable_by_actor(None)
+    with django_perf_rec.record():
         match = upload.track.artist in list(queryset)
     assert match is expected
     if expected: