Merge branch 'master' into develop

api/config/settings/common.py

@@ -515,7 +515,6 @@ CACHES = {
         "LOCATION": "local-cache",
     },
 }
-
 CACHES["default"]["BACKEND"] = "django_redis.cache.RedisCache"
 
 CHANNEL_LAYERS = {
@@ -530,7 +529,20 @@ CACHES["default"]["OPTIONS"] = {
     "IGNORE_EXCEPTIONS": True,  # mimics memcache behavior.
     # http://niwinz.github.io/django-redis/latest/#_memcached_exceptions_behavior
 }
-
+CACHEOPS_DURATION = env("CACHEOPS_DURATION", default=0)
+CACHEOPS_ENABLED = bool(CACHEOPS_DURATION)
+
+if CACHEOPS_ENABLED:
+    INSTALLED_APPS += ("cacheops",)
+    CACHEOPS_REDIS = env("CACHE_URL", default=CACHE_DEFAULT)
+    CACHEOPS_PREFIX = lambda _: "cacheops"  # noqa
+    CACHEOPS_DEFAULTS = {"timeout": CACHEOPS_DURATION}
+    CACHEOPS = {
+        "users.user": {"ops": "get"},
+        "music.album": {"ops": "count"},
+        "music.artist": {"ops": "count"},
+        "music.track": {"ops": "count"},
+    }
 
 # CELERY
 INSTALLED_APPS += ("funkwhale_api.taskapp.celery.CeleryConfig",)

api/config/settings/local.py

@@ -38,10 +38,26 @@ EMAIL_PORT = 1025
 DEBUG_TOOLBAR_CONFIG = {
     "DISABLE_PANELS": ["debug_toolbar.panels.redirects.RedirectsPanel"],
     "SHOW_TEMPLATE_CONTEXT": True,
-    "SHOW_TOOLBAR_CALLBACK": lambda request: True,
+    "SHOW_TOOLBAR_CALLBACK": lambda request: "debug" in request.GET,
     "JQUERY_URL": "/staticfiles/admin/js/vendor/jquery/jquery.js",
 }
 
+DEBUG_TOOLBAR_PANELS = [
+    # 'debug_toolbar.panels.versions.VersionsPanel',
+    "debug_toolbar.panels.timer.TimerPanel",
+    "debug_toolbar.panels.settings.SettingsPanel",
+    "debug_toolbar.panels.headers.HeadersPanel",
+    # 'debug_toolbar.panels.request.RequestPanel',
+    "debug_toolbar.panels.sql.SQLPanel",
+    # 'debug_toolbar.panels.staticfiles.StaticFilesPanel',
+    # 'debug_toolbar.panels.templates.TemplatesPanel',
+    "debug_toolbar.panels.cache.CachePanel",
+    # 'debug_toolbar.panels.signals.SignalsPanel',
+    # 'debug_toolbar.panels.logging.LoggingPanel',
+    # 'debug_toolbar.panels.redirects.RedirectsPanel',
+    # 'debug_toolbar.panels.profiling.ProfilingPanel',
+]
+
 # django-extensions
 # ------------------------------------------------------------------------------
 # INSTALLED_APPS += ('django_extensions', )
@@ -69,4 +85,7 @@ if env.bool("WEAK_PASSWORDS", default=False):
     # Faster during tests
     PASSWORD_HASHERS = ("django.contrib.auth.hashers.MD5PasswordHasher",)
 
-MIDDLEWARE = ("funkwhale_api.common.middleware.DevHttpsMiddleware",) + MIDDLEWARE
+MIDDLEWARE = (
+    "funkwhale_api.common.middleware.DevHttpsMiddleware",
+    "funkwhale_api.common.middleware.ProfilerMiddleware",
+) + MIDDLEWARE

api/funkwhale_api/common/middleware.py

 import html
+import io
 import requests
 import time
 import xml.sax.saxutils
@@ -242,3 +243,53 @@ class ThrottleStatusMiddleware:
                 response["X-RateLimit-ResetSeconds"] = str(remaining)
 
         return response
+
+
+class ProfilerMiddleware:
+    """
+    from https://github.com/omarish/django-cprofile-middleware/blob/master/django_cprofile_middleware/middleware.py
+    Simple profile middleware to profile django views. To run it, add ?prof to
+    the URL like this:
+        http://localhost:8000/view/?prof
+    Optionally pass the following to modify the output:
+    ?sort => Sort the output by a given metric. Default is time.
+        See
+        http://docs.python.org/2/library/profile.html#pstats.Stats.sort_stats
+        for all sort options.
+    ?count => The number of rows to display. Default is 100.
+    ?download => Download profile file suitable for visualization. For example
+        in snakeviz or RunSnakeRun
+    This is adapted from an example found here:
+    http://www.slideshare.net/zeeg/django-con-high-performance-django-presentation.
+    """
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        if "prof" not in request.GET:
+            return self.get_response(request)
+        import profile
+        import pstats
+
+        profiler = profile.Profile()
+        response = profiler.runcall(self.get_response, request)
+        profiler.create_stats()
+        if "prof-download" in request.GET:
+            import marshal
+
+            output = marshal.dumps(profiler.stats)
+            response = http.HttpResponse(
+                output, content_type="application/octet-stream"
+            )
+            response["Content-Disposition"] = "attachment; filename=view.prof"
+            response["Content-Length"] = len(output)
+        stream = io.StringIO()
+        stats = pstats.Stats(profiler, stream=stream)
+
+        stats.sort_stats(request.GET.get("prof-sort", "cumtime"))
+        stats.print_stats(int(request.GET.get("count", 100)))
+
+        response = http.HttpResponse("<pre>%s</pre>" % stream.getvalue())
+
+        return response

api/funkwhale_api/music/models.py

@@ -436,7 +436,7 @@ class TrackQuerySet(common_models.LocalFromFidQuerySet, models.QuerySet):
             return self.exclude(pk__in=matches)
 
     def with_playable_uploads(self, actor):
-        uploads = Upload.objects.playable_by(actor).select_related("track")
+        uploads = Upload.objects.playable_by(actor)
         return self.prefetch_related(
             models.Prefetch("uploads", queryset=uploads, to_attr="playable_uploads")
         )
@@ -594,7 +594,8 @@ class Track(APIModelMixin):
 
     @property
     def listen_url(self):
-        return reverse("api:v1:listen-detail", kwargs={"uuid": self.uuid})
+        # Not using reverse because this is slow
+        return "/api/v1/listen/{}/".format(self.uuid)
 
     @property
     def local_license(self):

api/funkwhale_api/music/serializers.py

@@ -45,26 +45,21 @@ class LicenseSerializer(serializers.Serializer):
         return obj["identifiers"][0]
 
 
-class ArtistAlbumSerializer(serializers.ModelSerializer):
+class ArtistAlbumSerializer(serializers.Serializer):
     tracks_count = serializers.SerializerMethodField()
     cover = cover_field
     is_playable = serializers.SerializerMethodField()
+    is_local = serializers.BooleanField()
+    id = serializers.IntegerField()
+    fid = serializers.URLField()
+    mbid = serializers.UUIDField()
+    title = serializers.CharField()
+    artist = serializers.SerializerMethodField()
+    release_date = serializers.DateField()
+    creation_date = serializers.DateTimeField()
 
-    class Meta:
-        model = models.Album
-        fields = (
-            "id",
-            "fid",
-            "mbid",
-            "title",
-            "artist",
-            "release_date",
-            "cover",
-            "creation_date",
-            "tracks_count",
-            "is_playable",
-            "is_local",
-        )
+    def get_artist(self, o):
+        return o.artist_id
 
     def get_tracks_count(self, o):
         return o._tracks_count
@@ -76,26 +71,20 @@ class ArtistAlbumSerializer(serializers.ModelSerializer):
             return None
 
 
-class ArtistWithAlbumsSerializer(serializers.ModelSerializer):
-    albums = ArtistAlbumSerializer(many=True, read_only=True)
+DATETIME_FIELD = serializers.DateTimeField()
+
+
+class ArtistWithAlbumsSerializer(serializers.Serializer):
+    albums = ArtistAlbumSerializer(many=True)
     tags = serializers.SerializerMethodField()
     attributed_to = serializers.SerializerMethodField()
     tracks_count = serializers.SerializerMethodField()
-
-    class Meta:
-        model = models.Artist
-        fields = (
-            "id",
-            "fid",
-            "mbid",
-            "name",
-            "creation_date",
-            "albums",
-            "is_local",
-            "tags",
-            "attributed_to",
-            "tracks_count",
-        )
+    id = serializers.IntegerField()
+    fid = serializers.URLField()
+    mbid = serializers.UUIDField()
+    name = serializers.CharField()
+    creation_date = serializers.DateTimeField()
+    is_local = serializers.BooleanField()
 
     def get_tags(self, obj):
         tagged_items = getattr(obj, "_prefetched_tagged_items", [])
@@ -114,9 +103,7 @@ def serialize_artist_simple(artist):
         "fid": artist.fid,
         "mbid": str(artist.mbid),
         "name": artist.name,
-        "creation_date": serializers.DateTimeField().to_representation(
-            artist.creation_date
-        ),
+        "creation_date": DATETIME_FIELD.to_representation(artist.creation_date),
         "is_local": artist.is_local,
     }
 
@@ -129,9 +116,7 @@ def serialize_album_track(track):
         "title": track.title,
         "artist": serialize_artist_simple(track.artist),
         "album": track.album_id,
-        "creation_date": serializers.DateTimeField().to_representation(
-            track.creation_date
-        ),
+        "creation_date": DATETIME_FIELD.to_representation(track.creation_date),
         "position": track.position,
         "disc_number": track.disc_number,
         "uploads": [
@@ -145,31 +130,22 @@ def serialize_album_track(track):
     }
 
 
-class AlbumSerializer(serializers.ModelSerializer):
+class AlbumSerializer(serializers.Serializer):
     tracks = serializers.SerializerMethodField()
     artist = serializers.SerializerMethodField()
     cover = cover_field
     is_playable = serializers.SerializerMethodField()
     tags = serializers.SerializerMethodField()
     attributed_to = serializers.SerializerMethodField()
-
-    class Meta:
-        model = models.Album
-        fields = (
-            "id",
-            "fid",
-            "mbid",
-            "title",
-            "artist",
-            "tracks",
-            "release_date",
-            "cover",
-            "creation_date",
-            "is_playable",
-            "is_local",
-            "tags",
-            "attributed_to",
-        )
+    id = serializers.IntegerField()
+    fid = serializers.URLField()
+    mbid = serializers.UUIDField()
+    title = serializers.CharField()
+    artist = serializers.SerializerMethodField()
+    release_date = serializers.DateField()
+    creation_date = serializers.DateTimeField()
+    is_local = serializers.BooleanField()
+    is_playable = serializers.SerializerMethodField()
 
     get_attributed_to = serialize_attributed_to
 
@@ -227,7 +203,7 @@ def serialize_upload(upload):
     }
 
 
-class TrackSerializer(serializers.ModelSerializer):
+class TrackSerializer(serializers.Serializer):
     artist = serializers.SerializerMethodField()
     album = TrackAlbumSerializer(read_only=True)
     uploads = serializers.SerializerMethodField()
@@ -235,26 +211,17 @@ class TrackSerializer(serializers.ModelSerializer):
     tags = serializers.SerializerMethodField()
     attributed_to = serializers.SerializerMethodField()
 
-    class Meta:
-        model = models.Track
-        fields = (
-            "id",
-            "fid",
-            "mbid",
-            "title",
-            "album",
-            "artist",
-            "creation_date",
-            "position",
-            "disc_number",
-            "uploads",
-            "listen_url",
-            "copyright",
-            "license",
-            "is_local",
-            "tags",
-            "attributed_to",
-        )
+    id = serializers.IntegerField()
+    fid = serializers.URLField()
+    mbid = serializers.UUIDField()
+    title = serializers.CharField()
+    artist = serializers.SerializerMethodField()
+    creation_date = serializers.DateTimeField()
+    is_local = serializers.BooleanField()
+    position = serializers.IntegerField()
+    disc_number = serializers.IntegerField()
+    copyright = serializers.CharField()
+    license = serializers.SerializerMethodField()
 
     get_attributed_to = serialize_attributed_to
 
@@ -271,6 +238,9 @@ class TrackSerializer(serializers.ModelSerializer):
         tagged_items = getattr(obj, "_prefetched_tagged_items", [])
         return [ti.tag.name for ti in tagged_items]
 
+    def get_license(self, o):
+        return o.license_id
+
 
 @common_serializers.track_fields_for_update("name", "description", "privacy_level")
 class LibraryForOwnerSerializer(serializers.ModelSerializer):

api/requirements/base.txt

@@ -72,3 +72,4 @@ django-oauth-toolkit==1.2
 django-storages==1.7.1
 boto3<3
 unicode-slugify
+django-cacheops==4.2