Merge branch 'release/0.5.3'

CHANGELOG

 Changelog
 =========
 
-0.6 (Unreleased)
-----------------
+.. towncrier
+
+0.5.3 (2018-02-27)
+------------------
+
+Features:
+
+- Added admin interface for radios, track files, favorites and import requests (#80)
+- Added basic instance stats on /about (#82)
+- Search now unaccent letters for queries like "The Dø" or "Björk" yielding more results (#81)
+
+
+Bugfixes:
+
+- Always use username in sidebar (#89)
+- Click event outside of player icons (#83)
+- Fixed broken import because of missing transaction
+- Now always load next radio track on last queue track ended (#87)
+- Now exclude tracks without file from radio candidates (#88)
+- skip to next track properly on 40X errors (#86)
+
+
+Other:
+
+- Switched to towncrier for changelog management and compilation
 
 
 0.5.2 (2018-02-26)

api/config/settings/common.py

@@ -37,6 +37,7 @@ DJANGO_APPS = (
     'django.contrib.sites',
     'django.contrib.messages',
     'django.contrib.staticfiles',
+    'django.contrib.postgres',
 
     # Useful template tags:
     # 'django.contrib.humanize',

api/funkwhale_api/__init__.py

 # -*- coding: utf-8 -*-
-__version__ = '0.5.2'
+__version__ = '0.5.3'
 __version_info__ = tuple([int(num) if num.isdigit() else num for num in __version__.replace('-', '.', 1).split('.')])

api/funkwhale_api/common/migrations/0001_initial.py

+# Generated by Django 2.0.2 on 2018-02-27 18:43
+from django.db import migrations
+from django.contrib.postgres.operations import UnaccentExtension
+
+
+class Migration(migrations.Migration):
+
+    dependencies = []
+
+    operations = [
+        UnaccentExtension()
+    ]

api/funkwhale_api/common/utils.py

 import os
 import shutil
 
+from django.db import transaction
+
 
 def rename_file(instance, field_name, new_name, allow_missing_file=False):
     field = getattr(instance, field_name)
@@ -17,3 +19,9 @@ def rename_file(instance, field_name, new_name, allow_missing_file=False):
     field.name = os.path.join(initial_path, new_name_with_extension)
     instance.save()
     return new_name_with_extension
+
+
+def on_commit(f, *args, **kwargs):
+    return transaction.on_commit(
+        lambda: f(*args, **kwargs)
+    )

api/funkwhale_api/favorites/admin.py

+from django.contrib import admin
+
+from . import models
+
+
+@admin.register(models.TrackFavorite)
+class TrackFavoriteAdmin(admin.ModelAdmin):
+    list_display = ['user', 'track', 'creation_date']
+    list_select_related = [
+        'user',
+        'track'
+    ]

api/funkwhale_api/history/admin.py

@@ -6,3 +6,7 @@ from . import models
 class ListeningAdmin(admin.ModelAdmin):
     list_display = ['track', 'end_date', 'user', 'session_key']
     search_fields = ['track__name', 'user__username']
+    list_select_related = [
+        'user',
+        'track'
+    ]

api/funkwhale_api/instance/stats.py

+from django.db.models import Sum
+
+from funkwhale_api.favorites.models import TrackFavorite
+from funkwhale_api.history.models import Listening
+from funkwhale_api.music import models
+from funkwhale_api.users.models import User
+
+
+def get():
+    return {
+        'users': get_users(),
+        'tracks': get_tracks(),
+        'albums': get_albums(),
+        'artists': get_artists(),
+        'track_favorites': get_track_favorites(),
+        'listenings': get_listenings(),
+        'music_duration': get_music_duration(),
+    }
+
+
+def get_users():
+    return User.objects.count()
+
+
+def get_listenings():
+    return Listening.objects.count()
+
+
+def get_track_favorites():
+    return TrackFavorite.objects.count()
+
+
+def get_tracks():
+    return models.Track.objects.count()
+
+
+def get_albums():
+    return models.Album.objects.count()
+
+
+def get_artists():
+    return models.Artist.objects.count()
+
+
+def get_music_duration():
+    seconds = models.TrackFile.objects.aggregate(
+        d=Sum('duration'),
+    )['d']
+    if seconds:
+        return seconds / 3600
+    return 0

api/funkwhale_api/instance/urls.py

 from django.conf.urls import url
+from django.views.decorators.cache import cache_page
+
 from . import views
 
 
 urlpatterns = [
     url(r'^settings/$', views.InstanceSettings.as_view(), name='settings'),
+    url(r'^stats/$',
+        cache_page(60 * 5)(views.InstanceStats.as_view()), name='stats'),
 ]

api/funkwhale_api/instance/views.py

@@ -4,6 +4,8 @@ from rest_framework.response import Response
 from dynamic_preferences.api import serializers
 from dynamic_preferences.registries import global_preferences_registry
 
+from . import stats
+
 
 class InstanceSettings(views.APIView):
     permission_classes = []
@@ -23,3 +25,12 @@ class InstanceSettings(views.APIView):
         data = serializers.GlobalPreferenceSerializer(
             api_preferences, many=True).data
         return Response(data, status=200)
+
+
+class InstanceStats(views.APIView):
+    permission_classes = []
+    authentication_classes = []
+
+    def get(self, request, *args, **kwargs):
+        data = stats.get()
+        return Response(data, status=200)

api/funkwhale_api/music/admin.py

@@ -25,13 +25,26 @@ class TrackAdmin(admin.ModelAdmin):
 
 @admin.register(models.ImportBatch)
 class ImportBatchAdmin(admin.ModelAdmin):
-    list_display = ['creation_date', 'status']
-
+    list_display = [
+        'submitted_by',
+        'creation_date',
+        'import_request',
+        'status']
+    list_select_related = [
+        'submitted_by',
+        'import_request',
+    ]
+    list_filter = ['status']
+    search_fields = [
+        'import_request__name', 'source', 'batch__pk', 'mbid']
 
 @admin.register(models.ImportJob)
 class ImportJobAdmin(admin.ModelAdmin):
     list_display = ['source', 'batch', 'track_file', 'status', 'mbid']
-    list_select_related = True
+    list_select_related = [
+        'track_file',
+        'batch',
+    ]
     search_fields = ['source', 'batch__pk', 'mbid']
     list_filter = ['status']
 
@@ -50,3 +63,19 @@ class LyricsAdmin(admin.ModelAdmin):
     list_select_related = True
     search_fields = ['url', 'work__title']
     list_filter = ['work__language']
+
+
+@admin.register(models.TrackFile)
+class TrackFileAdmin(admin.ModelAdmin):
+    list_display = [
+        'track',
+        'audio_file',
+        'source',
+        'duration',
+        'mimetype',
+    ]
+    list_select_related = [
+        'track'
+    ]
+    search_fields = ['source', 'acoustid_track_id']
+    list_filter = ['mimetype']

api/funkwhale_api/music/tasks.py

@@ -73,7 +73,10 @@ def _do_import(import_job, replace):
 
 
 @celery.app.task(name='ImportJob.run', bind=True)
-@celery.require_instance(models.ImportJob, 'import_job')
+@celery.require_instance(
+    models.ImportJob.objects.filter(
+        status__in=['pending', 'errored']),
+    'import_job')
 def import_job_run(self, import_job, replace=False):
     def mark_errored():
         import_job.status = 'errored'

api/funkwhale_api/music/views.py

@@ -19,6 +19,7 @@ from musicbrainzngs import ResponseError
 from django.contrib.auth.decorators import login_required
 from django.utils.decorators import method_decorator
 
+from funkwhale_api.common import utils as funkwhale_utils
 from funkwhale_api.requests.models import ImportRequest
 from funkwhale_api.musicbrainz import api
 from funkwhale_api.common.permissions import (
@@ -62,7 +63,7 @@ class ArtistViewSet(SearchMixin, viewsets.ReadOnlyModelViewSet):
                                 'albums__tracks__tags'))
     serializer_class = serializers.ArtistSerializerNested
     permission_classes = [ConditionalAuthentication]
-    search_fields = ['name']
+    search_fields = ['name__unaccent']
     filter_class = filters.ArtistFilter
     ordering_fields = ('id', 'name', 'creation_date')
 
@@ -75,7 +76,7 @@ class AlbumViewSet(SearchMixin, viewsets.ReadOnlyModelViewSet):
                                               'tracks__files'))
     serializer_class = serializers.AlbumSerializerNested
     permission_classes = [ConditionalAuthentication]
-    search_fields = ['title']
+    search_fields = ['title__unaccent']
     ordering_fields = ('creation_date',)
 
 
@@ -116,7 +117,10 @@ class ImportJobViewSet(
     def perform_create(self, serializer):
         source = 'file://' + serializer.validated_data['audio_file'].name
         serializer.save(source=source)
-        tasks.import_job_run.delay(import_job_id=serializer.instance.pk)
+        funkwhale_utils.on_commit(
+            tasks.import_job_run.delay,
+            import_job_id=serializer.instance.pk
+        )
 
 
 class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet):
@@ -129,9 +133,9 @@ class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet):
     search_fields = ['title', 'artist__name']
     ordering_fields = (
         'creation_date',
-        'title',
-        'album__title',
-        'artist__name',
+        'title__unaccent',
+        'album__title__unaccent',
+        'artist__name__unaccent',
     )
 
     def get_queryset(self):
@@ -245,7 +249,11 @@ class Search(views.APIView):
         return Response(results, status=200)
 
     def get_tracks(self, query):
-        search_fields = ['mbid', 'title', 'album__title', 'artist__name']
+        search_fields = [
+            'mbid',
+            'title__unaccent',
+            'album__title__unaccent',
+            'artist__name__unaccent']
         query_obj = utils.get_query(query, search_fields)
         return (
             models.Track.objects.all()
@@ -259,7 +267,10 @@ class Search(views.APIView):
 
 
     def get_albums(self, query):
-        search_fields = ['mbid', 'title', 'artist__name']
+        search_fields = [
+            'mbid',
+            'title__unaccent',
+            'artist__name__unaccent']
         query_obj = utils.get_query(query, search_fields)
         return (
             models.Album.objects.all()
@@ -273,7 +284,7 @@ class Search(views.APIView):
 
 
     def get_artists(self, query):
-        search_fields = ['mbid', 'name']
+        search_fields = ['mbid', 'name__unaccent']
         query_obj = utils.get_query(query, search_fields)
         return (
             models.Artist.objects.all()
@@ -288,7 +299,7 @@ class Search(views.APIView):
 
 
     def get_tags(self, query):
-        search_fields = ['slug', 'name']
+        search_fields = ['slug', 'name__unaccent']
         query_obj = utils.get_query(query, search_fields)
 
         # We want the shortest tag first
@@ -336,6 +347,7 @@ class SubmitViewSet(viewsets.ViewSet):
             data, request, batch=None, import_request=import_request)
         return Response(import_data)
 
+    @transaction.atomic
     def _import_album(self, data, request, batch=None, import_request=None):
         # we import the whole album here to prevent race conditions that occurs
         # when using get_or_create_from_api in tasks
@@ -355,7 +367,11 @@ class SubmitViewSet(viewsets.ViewSet):
                 models.TrackFile.objects.get(track__mbid=row['mbid'])
             except models.TrackFile.DoesNotExist:
                 job = models.ImportJob.objects.create(mbid=row['mbid'], batch=batch, source=row['source'])
-                tasks.import_job_run.delay(import_job_id=job.pk)
+                funkwhale_utils.on_commit(
+                    tasks.import_job_run.delay,
+                    import_job_id=job.pk
+                )
+
         serializer = serializers.ImportBatchSerializer(batch)
         return serializer.data, batch
 

api/funkwhale_api/providers/audiofile/management/commands/import_files.py

@@ -3,6 +3,9 @@ import os
 
 from django.core.files import File
 from django.core.management.base import BaseCommand, CommandError
+from django.db import transaction
+
+from funkwhale_api.common import utils
 from funkwhale_api.music import tasks
 from funkwhale_api.users.models import User
 
@@ -86,6 +89,7 @@ class Command(BaseCommand):
         self.stdout.write(
             "For details, please refer to import batch #".format(batch.pk))
 
+    @transaction.atomic
     def do_import(self, matching, user, options):
         message = 'Importing {}...'
         if options['async']:
@@ -94,7 +98,7 @@ class Command(BaseCommand):
         # we create an import batch binded to the user
         batch = user.imports.create(source='shell')
         async = options['async']
-        handler = tasks.import_job_run.delay if async else tasks.import_job_run
+        import_handler = tasks.import_job_run.delay if async else tasks.import_job_run
         for path in matching:
             job = batch.jobs.create(
                 source='file://' + path,
@@ -105,7 +109,8 @@ class Command(BaseCommand):
 
             job.save()
             try:
-                handler(import_job_id=job.pk)
+                utils.on_commit(import_handler, import_job_id=job.pk)
             except Exception as e:
                 self.stdout.write('Error: {}'.format(e))
+
         return batch

api/funkwhale_api/radios/admin.py

+from django.contrib import admin
+
+from . import models
+
+
+@admin.register(models.Radio)
+class RadioAdmin(admin.ModelAdmin):
+    list_display = [
+        'user', 'name', 'is_public', 'creation_date', 'config']
+    list_select_related = [
+        'user',
+    ]
+    list_filter = [
+        'is_public',
+    ]
+    search_fields = ['name', 'description']
+
+
+@admin.register(models.RadioSession)
+class RadioSessionAdmin(admin.ModelAdmin):
+    list_display = [
+        'user',
+        'custom_radio',
+        'radio_type',
+        'creation_date',
+        'related_object']
+
+    list_select_related = [
+        'user',
+        'custom_radio'
+    ]
+    list_filter = [
+        'radio_type',
+    ]
+
+
+@admin.register(models.RadioSessionTrack)
+class RadioSessionTrackAdmin(admin.ModelAdmin):
+    list_display = [
+        'id',
+        'session',
+        'position',
+        'track',]
+
+    list_select_related = [
+        'track',
+        'session'
+    ]

api/funkwhale_api/radios/radios.py

 import random
 from rest_framework import serializers
+from django.db.models import Count
 from django.core.exceptions import ValidationError
 from taggit.models import Tag
 from funkwhale_api.users.models import User
@@ -39,8 +40,11 @@ class SessionRadio(SimpleRadio):
         self.session = models.RadioSession.objects.create(user=user, radio_type=self.radio_type, **kwargs)
         return self.session
 
-    def get_queryset(self):
-        raise NotImplementedError
+    def get_queryset(self, **kwargs):
+        qs = Track.objects.annotate(
+            files_count=Count('files')
+        )
+        return qs.filter(files_count__gt=0)
 
     def get_queryset_kwargs(self):
         return {}
@@ -75,7 +79,9 @@ class SessionRadio(SimpleRadio):
 @registry.register(name='random')
 class RandomRadio(SessionRadio):
     def get_queryset(self, **kwargs):
-        return Track.objects.all()
+        qs = super().get_queryset(**kwargs)
+        return qs.order_by('?')
+
 
 @registry.register(name='favorites')
 class FavoritesRadio(SessionRadio):
@@ -87,8 +93,9 @@ class FavoritesRadio(SessionRadio):
         return kwargs
 
     def get_queryset(self, **kwargs):
+        qs = super().get_queryset(**kwargs)
         track_ids = kwargs['user'].track_favorites.all().values_list('track', flat=True)
-        return Track.objects.filter(pk__in=track_ids)
+        return qs.filter(pk__in=track_ids)
 
 
 @registry.register(name='custom')
@@ -101,7 +108,11 @@ class CustomRadio(SessionRadio):
         return kwargs
 
     def get_queryset(self, **kwargs):
-        return filters.run(kwargs['custom_radio'].config)
+        qs = super().get_queryset(**kwargs)
+        return filters.run(
+            kwargs['custom_radio'].config,
+            candidates=qs,
+        )
 
     def validate_session(self, data, **context):
         data = super().validate_session(data, **context)
@@ -141,6 +152,7 @@ class TagRadio(RelatedObjectRadio):
     model = Tag
 
     def get_queryset(self, **kwargs):
+        qs = super().get_queryset(**kwargs)
         return Track.objects.filter(tags__in=[self.session.related_object])
 
 @registry.register(name='artist')
@@ -148,7 +160,8 @@ class ArtistRadio(RelatedObjectRadio):
     model = Artist
 
     def get_queryset(self, **kwargs):
-        return self.session.related_object.tracks.all()
+        qs = super().get_queryset(**kwargs)
+        return qs.filter(artist=self.session.related_object)
 
 
 @registry.register(name='less-listened')
@@ -160,5 +173,6 @@ class LessListenedRadio(RelatedObjectRadio):
         super().clean(instance)
 
     def get_queryset(self, **kwargs):
+        qs = super().get_queryset(**kwargs)
         listened = self.session.user.listenings.all().values_list('track', flat=True)
-        return Track.objects.exclude(pk__in=listened).order_by('?')
+        return qs.exclude(pk__in=listened).order_by('?')

api/funkwhale_api/requests/admin.py

+from django.contrib import admin
+
+from . import models
+
+
+@admin.register(models.ImportRequest)
+class ImportRequestAdmin(admin.ModelAdmin):
+    list_display = ['artist_name', 'user', 'status', 'creation_date']
+    list_select_related = [
+        'user',
+        'track'
+    ]
+    list_filter = [
+        'status',
+    ]
+    search_fields = ['artist_name', 'comment', 'albums']

api/tests/instance/test_stats.py

+from django.urls import reverse
+
+from funkwhale_api.instance import stats
+
+
+def test_can_get_stats_via_api(db, api_client, mocker):
+    stats = {
+        'foo': 'bar'
+    }
+    mocker.patch('funkwhale_api.instance.stats.get', return_value=stats)
+    url = reverse('api:v1:instance:stats')
+    response = api_client.get(url)
+    assert response.data == stats
+
+
+def test_get_users(mocker):
+    mocker.patch(
+        'funkwhale_api.users.models.User.objects.count', return_value=42)
+
+    assert stats.get_users() == 42
+
+
+def test_get_music_duration(factories):
+    factories['music.TrackFile'].create_batch(size=5, duration=360)
+
+    # duration is in hours
+    assert stats.get_music_duration() == 0.5
+
+
+def test_get_listenings(mocker):
+    mocker.patch(
+        'funkwhale_api.history.models.Listening.objects.count',
+         return_value=42)
+    assert stats.get_listenings() == 42
+
+
+def test_get_track_favorites(mocker):
+    mocker.patch(
+        'funkwhale_api.favorites.models.TrackFavorite.objects.count',
+         return_value=42)
+    assert stats.get_track_favorites() == 42
+
+
+def test_get_tracks(mocker):
+    mocker.patch(
+        'funkwhale_api.music.models.Track.objects.count',
+         return_value=42)
+    assert stats.get_tracks() == 42
+
+
+def test_get_albums(mocker):
+    mocker.patch(
+        'funkwhale_api.music.models.Album.objects.count',
+         return_value=42)
+    assert stats.get_albums() == 42
+
+
+def test_get_artists(mocker):
+    mocker.patch(
+        'funkwhale_api.music.models.Artist.objects.count',
+         return_value=42)
+    assert stats.get_artists() == 42
+
+
+def test_get(mocker):
+    keys = [
+        'users',
+        'tracks',
+        'albums',
+        'artists',
+        'track_favorites',
+        'listenings',
+        'music_duration',
+    ]
+    mocks = [
+        mocker.patch.object(stats, 'get_{}'.format(k), return_value=i)
+        for i, k in enumerate(keys)
+    ]
+
+    expected = {
+        k: i for i, k in enumerate(keys)
+    }
+
+    assert stats.get() == expected

api/tests/music/test_api.py

@@ -6,6 +6,7 @@ from django.urls import reverse
 from funkwhale_api.music import models
 from funkwhale_api.musicbrainz import api
 from funkwhale_api.music import serializers
+from funkwhale_api.music import tasks
 
 from . import data as api_data
 
@@ -208,7 +209,7 @@ def test_user_can_create_an_empty_batch(client, factories):
 
 def test_user_can_create_import_job_with_file(client, factories, mocker):
     path = os.path.join(DATA_DIR, 'test.ogg')
-    m = mocker.patch('funkwhale_api.music.tasks.import_job_run.delay')
+    m = mocker.patch('funkwhale_api.common.utils.on_commit')
     user = factories['users.SuperUser']()
     batch = factories['music.ImportBatch'](submitted_by=user)
     url = reverse('api:v1:import-jobs-list')
@@ -231,7 +232,9 @@ def test_user_can_create_import_job_with_file(client, factories, mocker):
     assert 'test.ogg' in job.source
     assert job.audio_file.read() == content
 
-    m.assert_called_once_with(import_job_id=job.pk)
+    m.assert_called_once_with(
+        tasks.import_job_run.delay,
+        import_job_id=job.pk)
 
 
 def test_can_search_artist(factories, client):