Merge branch 'release/0.13'

api/funkwhale_api/instance/views.py

@@ -2,9 +2,11 @@ from rest_framework import views
 from rest_framework.response import Response
 
 from dynamic_preferences.api import serializers
+from dynamic_preferences.api import viewsets as preferences_viewsets
 from dynamic_preferences.registries import global_preferences_registry
 
 from funkwhale_api.common import preferences
+from funkwhale_api.users.permissions import HasUserPermission
 
 from . import nodeinfo
 from . import stats
@@ -15,6 +17,11 @@ NODEINFO_2_CONTENT_TYPE = (
 )
 
 
+class AdminSettings(preferences_viewsets.GlobalPreferencesViewSet):
+    pagination_class = None
+    permission_classes = (HasUserPermission,)
+    required_permissions = ['settings']
+
 class InstanceSettings(views.APIView):
     permission_classes = []
     authentication_classes = []

api/funkwhale_api/music/admin.py

@@ -38,6 +38,7 @@ class ImportBatchAdmin(admin.ModelAdmin):
     search_fields = [
         'import_request__name', 'source', 'batch__pk', 'mbid']
 
+
 @admin.register(models.ImportJob)
 class ImportJobAdmin(admin.ModelAdmin):
     list_display = ['source', 'batch', 'track_file', 'status', 'mbid']
@@ -73,9 +74,16 @@ class TrackFileAdmin(admin.ModelAdmin):
         'source',
         'duration',
         'mimetype',
+        'size',
+        'bitrate'
     ]
     list_select_related = [
         'track'
     ]
-    search_fields = ['source', 'acoustid_track_id']
+    search_fields = [
+        'source',
+        'acoustid_track_id',
+        'track__title',
+        'track__album__title',
+        'track__artist__name']
     list_filter = ['mimetype']

api/funkwhale_api/music/factories.py

@@ -54,6 +54,10 @@ class TrackFileFactory(factory.django.DjangoModelFactory):
     audio_file = factory.django.FileField(
         from_path=os.path.join(SAMPLES_PATH, 'test.ogg'))
 
+    bitrate = None
+    size = None
+    duration = None
+
     class Meta:
         model = 'music.TrackFile'
 

api/funkwhale_api/music/management/commands/fix_track_files.py

@@ -2,6 +2,7 @@ import cacheops
 import os
 
 from django.db import transaction
+from django.db.models import Q
 from django.conf import settings
 from django.core.management.base import BaseCommand, CommandError
 
@@ -24,6 +25,8 @@ class Command(BaseCommand):
         if options['dry_run']:
             self.stdout.write('Dry-run on, will not commit anything')
         self.fix_mimetypes(**options)
+        self.fix_file_data(**options)
+        self.fix_file_size(**options)
         cacheops.invalidate_model(models.TrackFile)
 
     @transaction.atomic
@@ -43,3 +46,60 @@ class Command(BaseCommand):
             if not dry_run:
                 self.stdout.write('[mimetypes] commiting...')
                 qs.update(mimetype=mimetype)
+
+    def fix_file_data(self, dry_run, **kwargs):
+        self.stdout.write('Fixing missing bitrate or length...')
+        matching = models.TrackFile.objects.filter(
+            Q(bitrate__isnull=True) | Q(duration__isnull=True))
+        total = matching.count()
+        self.stdout.write(
+            '[bitrate/length] {} entries found with missing values'.format(
+                total))
+        if dry_run:
+            return
+        for i, tf in enumerate(matching.only('audio_file')):
+            self.stdout.write(
+                '[bitrate/length] {}/{} fixing file #{}'.format(
+                    i+1, total, tf.pk
+                ))
+
+            try:
+                audio_file = tf.get_audio_file()
+                if audio_file:
+                    with audio_file as f:
+                        data = utils.get_audio_file_data(audio_file)
+                    tf.bitrate = data['bitrate']
+                    tf.duration = data['length']
+                    tf.save(update_fields=['duration', 'bitrate'])
+                else:
+                    self.stderr.write('[bitrate/length] no file found')
+            except Exception as e:
+                self.stderr.write(
+                    '[bitrate/length] error with file #{}: {}'.format(
+                        tf.pk, str(e)
+                    )
+                )
+
+    def fix_file_size(self, dry_run, **kwargs):
+        self.stdout.write('Fixing missing size...')
+        matching = models.TrackFile.objects.filter(size__isnull=True)
+        total = matching.count()
+        self.stdout.write(
+            '[size] {} entries found with missing values'.format(total))
+        if dry_run:
+            return
+        for i, tf in enumerate(matching.only('size')):
+            self.stdout.write(
+                '[size] {}/{} fixing file #{}'.format(
+                    i+1, total, tf.pk
+                ))
+
+            try:
+                tf.size = tf.get_file_size()
+                tf.save(update_fields=['size'])
+            except Exception as e:
+                self.stderr.write(
+                    '[size] error with file #{}: {}'.format(
+                        tf.pk, str(e)
+                    )
+                )

api/funkwhale_api/music/metadata.py

@@ -28,6 +28,13 @@ def get_id3_tag(f, k):
         raise TagNotFound(k)
 
 
+def get_flac_tag(f, k):
+    try:
+        return f.get(k)[0]
+    except (KeyError, IndexError):
+        raise TagNotFound(k)
+
+
 def get_mp3_recording_id(f, k):
     try:
         return [
@@ -121,7 +128,38 @@ CONF = {
                 'getter': get_mp3_recording_id,
             },
         }
-    }
+    },
+    'FLAC': {
+        'getter': get_flac_tag,
+        'fields': {
+            'track_number': {
+                'field': 'tracknumber',
+                'to_application': convert_track_number
+            },
+            'title': {
+                'field': 'title'
+            },
+            'artist': {
+                'field': 'artist'
+            },
+            'album': {
+                'field': 'album'
+            },
+            'date': {
+                'field': 'date',
+                'to_application': lambda v: arrow.get(str(v)).date()
+            },
+            'musicbrainz_albumid': {
+                'field': 'musicbrainz_albumid'
+            },
+            'musicbrainz_artistid': {
+                'field': 'musicbrainz_artistid'
+            },
+            'musicbrainz_recordingid': {
+                'field': 'musicbrainz_trackid'
+            },
+        }
+    },
 }
 
 

api/funkwhale_api/music/migrations/0027_auto_20180515_1808.py

+# Generated by Django 2.0.3 on 2018-05-15 18:08
+
+from django.db import migrations, models
+import taggit.managers
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0026_trackfile_accessed_date'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='trackfile',
+            name='bitrate',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='trackfile',
+            name='size',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='track',
+            name='tags',
+            field=taggit.managers.TaggableManager(blank=True, help_text='A comma-separated list of tags.', through='taggit.TaggedItem', to='taggit.Tag', verbose_name='Tags'),
+        ),
+    ]

api/funkwhale_api/music/models.py

@@ -361,7 +361,7 @@ class Track(APIModelMixin):
         import_tags
     ]
     objects = TrackQuerySet.as_manager()
-    tags = TaggableManager()
+    tags = TaggableManager(blank=True)
 
     class Meta:
         ordering = ['album', 'position']
@@ -429,6 +429,8 @@ class TrackFile(models.Model):
     modification_date = models.DateTimeField(auto_now=True)
     accessed_date = models.DateTimeField(null=True, blank=True)
     duration = models.IntegerField(null=True, blank=True)
+    size = models.IntegerField(null=True, blank=True)
+    bitrate = models.IntegerField(null=True, blank=True)
     acoustid_track_id = models.UUIDField(null=True, blank=True)
     mimetype = models.CharField(null=True, blank=True, max_length=200)
 
@@ -467,7 +469,7 @@ class TrackFile(models.Model):
 
     @property
     def filename(self):
-        return '{}{}'.format(
+        return '{}.{}'.format(
             self.track.full_name,
             self.extension)
 
@@ -477,6 +479,41 @@ class TrackFile(models.Model):
             return
         return os.path.splitext(self.audio_file.name)[-1].replace('.', '', 1)
 
+    def get_file_size(self):
+        if self.audio_file:
+            return self.audio_file.size
+
+        if self.source.startswith('file://'):
+            return os.path.getsize(self.source.replace('file://', '', 1))
+
+        if self.library_track and self.library_track.audio_file:
+            return self.library_track.audio_file.size
+
+    def get_audio_file(self):
+        if self.audio_file:
+            return self.audio_file.open()
+        if self.source.startswith('file://'):
+            return open(self.source.replace('file://', '', 1), 'rb')
+        if self.library_track and self.library_track.audio_file:
+            return self.library_track.audio_file.open()
+
+    def set_audio_data(self):
+        audio_file = self.get_audio_file()
+        if audio_file:
+            with audio_file as f:
+                audio_data = utils.get_audio_file_data(f)
+            if not audio_data:
+                return
+            self.duration = int(audio_data['length'])
+            self.bitrate = audio_data['bitrate']
+            self.size = self.get_file_size()
+        else:
+            lt = self.library_track
+            if lt:
+                self.duration = lt.get_metadata('length')
+                self.size = lt.get_metadata('size')
+                self.bitrate = lt.get_metadata('bitrate')
+
     def save(self, **kwargs):
         if not self.mimetype and self.audio_file:
             self.mimetype = utils.guess_mimetype(self.audio_file)

api/funkwhale_api/music/serializers.py

@@ -27,6 +27,7 @@ class SimpleArtistSerializer(serializers.ModelSerializer):
 
 class ArtistSerializer(serializers.ModelSerializer):
     tags = TagSerializer(many=True, read_only=True)
+
     class Meta:
         model = models.Artist
         fields = ('id', 'mbid', 'name', 'tags', 'creation_date')
@@ -40,11 +41,21 @@ class TrackFileSerializer(serializers.ModelSerializer):
         fields = (
             'id',
             'path',
-            'duration',
             'source',
             'filename',
             'mimetype',
-            'track')
+            'track',
+            'duration',
+            'mimetype',
+            'bitrate',
+            'size',
+        )
+        read_only_fields = [
+            'duration',
+            'mimetype',
+            'bitrate',
+            'size',
+        ]
 
     def get_path(self, o):
         url = o.path

api/funkwhale_api/music/tasks.py

@@ -134,6 +134,7 @@ def _do_import(import_job, replace=False, use_acoustid=True):
         # in place import, we set mimetype from extension
         path, ext = os.path.splitext(import_job.source)
         track_file.mimetype = music_utils.get_type_from_ext(ext)
+    track_file.set_audio_data()
     track_file.save()
     import_job.status = 'finished'
     import_job.track_file = track_file

api/funkwhale_api/music/utils.py

 import magic
 import mimetypes
+import mutagen
 import re
 
 from django.db.models import Q
@@ -66,6 +67,7 @@ def compute_status(jobs):
 AUDIO_EXTENSIONS_AND_MIMETYPE = [
     ('ogg', 'audio/ogg'),
     ('mp3', 'audio/mpeg'),
+    ('flac', 'audio/x-flac'),
 ]
 
 EXTENSION_TO_MIMETYPE = {ext: mt for ext, mt in AUDIO_EXTENSIONS_AND_MIMETYPE}
@@ -81,3 +83,14 @@ def get_type_from_ext(extension):
         # we remove leading dot
         extension = extension[1:]
     return EXTENSION_TO_MIMETYPE.get(extension)
+
+
+def get_audio_file_data(f):
+    data = mutagen.File(f)
+    if not data:
+        return
+    d = {}
+    d['bitrate'] = data.info.bitrate
+    d['length'] = data.info.length
+
+    return d

api/funkwhale_api/music/views.py

@@ -25,8 +25,8 @@ from rest_framework import permissions
 from musicbrainzngs import ResponseError
 
 from funkwhale_api.common import utils as funkwhale_utils
-from funkwhale_api.common.permissions import (
-    ConditionalAuthentication, HasModelPermission)
+from funkwhale_api.common.permissions import ConditionalAuthentication
+from funkwhale_api.users.permissions import HasUserPermission
 from taggit.models import Tag
 from funkwhale_api.federation import actors
 from funkwhale_api.federation.authentication import SignatureAuthentication
@@ -107,25 +107,22 @@ class ImportBatchViewSet(
               .annotate(job_count=Count('jobs'))
     )
     serializer_class = serializers.ImportBatchSerializer
-    permission_classes = (permissions.DjangoModelPermissions, )
+    permission_classes = (HasUserPermission,)
+    required_permissions = ['library']
     filter_class = filters.ImportBatchFilter
 
     def perform_create(self, serializer):
         serializer.save(submitted_by=self.request.user)
 
 
-class ImportJobPermission(HasModelPermission):
-    # not a typo, perms on import job is proxied to import batch
-    model = models.ImportBatch
-
-
 class ImportJobViewSet(
         mixins.CreateModelMixin,
         mixins.ListModelMixin,
         viewsets.GenericViewSet):
     queryset = (models.ImportJob.objects.all().select_related())
     serializer_class = serializers.ImportJobSerializer
-    permission_classes = (ImportJobPermission, )
+    permission_classes = (HasUserPermission,)
+    required_permissions = ['library']
     filter_class = filters.ImportJobFilter
 
     @list_route(methods=['get'])
@@ -230,7 +227,7 @@ def get_file_path(audio_file):
                     'MUSIC_DIRECTORY_PATH to serve in-place imported files'
                 )
             path = '/music' + audio_file.replace(prefix, '', 1)
-        return settings.PROTECT_FILES_PATH + path
+        return (settings.PROTECT_FILES_PATH + path).encode('utf-8')
     if t == 'apache2':
         try:
             path = audio_file.path
@@ -241,7 +238,7 @@ def get_file_path(audio_file):
                     'You need to specify MUSIC_DIRECTORY_SERVE_PATH and '
                     'MUSIC_DIRECTORY_PATH to serve in-place imported files'
                 )
-            path = audio_file.replace(prefix, serve_path, 1)
+            path = audio_file.replace(prefix, serve_path, 1).encode('utf-8')
         return path
 
 
@@ -268,6 +265,10 @@ def handle_serve(track_file):
                 qs = LibraryTrack.objects.select_for_update()
                 library_track = qs.get(pk=library_track.pk)
                 library_track.download_audio()
+            track_file.library_track = library_track
+            track_file.set_audio_data()
+            track_file.save(update_fields=['bitrate', 'duration', 'size'])
+
         audio_file = library_track.audio_file
         file_path = get_file_path(audio_file)
         mt = library_track.audio_mimetype
@@ -275,7 +276,10 @@ def handle_serve(track_file):
         file_path = get_file_path(audio_file)
     elif f.source and f.source.startswith('file://'):
         file_path = get_file_path(f.source.replace('file://', '', 1))
-    response = Response()
+    if mt:
+        response = Response(content_type=mt)
+    else:
+        response = Response()
     filename = f.filename
     mapping = {
         'nginx': 'X-Accel-Redirect',
@@ -293,7 +297,11 @@ def handle_serve(track_file):
 
 
 class TrackFileViewSet(viewsets.ReadOnlyModelViewSet):
-    queryset = (models.TrackFile.objects.all().order_by('-id'))
+    queryset = (
+        models.TrackFile.objects.all()
+            .select_related('track__artist', 'track__album')
+            .order_by('-id')
+    )
     serializer_class = serializers.TrackFileSerializer
     authentication_classes = rest_settings.api_settings.DEFAULT_AUTHENTICATION_CLASSES + [
         SignatureAuthentication
@@ -431,7 +439,8 @@ class Search(views.APIView):
 
 class SubmitViewSet(viewsets.ViewSet):
     queryset = models.ImportBatch.objects.none()
-    permission_classes = (permissions.DjangoModelPermissions, )
+    permission_classes = (HasUserPermission,)
+    required_permissions = ['library']
 
     @list_route(methods=['post'])
     @transaction.non_atomic_requests

api/funkwhale_api/playlists/dynamic_preferences_registry.py

@@ -13,3 +13,6 @@ class MaxTracks(preferences.DefaultFromSettingMixin, types.IntegerPreference):
     name = 'max_tracks'
     verbose_name = 'Max tracks per playlist'
     setting = 'PLAYLISTS_MAX_TRACKS'
+    field_kwargs = {
+        'required': False,
+    }

api/funkwhale_api/providers/acoustid/dynamic_preferences_registry.py

+from django import forms
+
 from dynamic_preferences.types import StringPreference, Section
 from dynamic_preferences.registries import global_preferences_registry
 
@@ -11,3 +13,7 @@ class APIKey(StringPreference):
     default = ''
     verbose_name = 'Acoustid API key'
     help_text = 'The API key used to query AcoustID. Get one at https://acoustid.org/new-application.'
+    widget = forms.PasswordInput
+    field_kwargs = {
+        'required': False,
+    }

api/funkwhale_api/providers/youtube/dynamic_preferences_registry.py

+from django import forms
+
 from dynamic_preferences.types import StringPreference, Section
 from dynamic_preferences.registries import global_preferences_registry
 
@@ -11,3 +13,7 @@ class APIKey(StringPreference):
     default = 'CHANGEME'
     verbose_name = 'YouTube API key'
     help_text = 'The API key used to query YouTube. Get one at https://console.developers.google.com/.'
+    widget = forms.PasswordInput
+    field_kwargs = {
+        'required': False,
+    }

api/funkwhale_api/subsonic/renderers.py

@@ -15,6 +15,9 @@ class SubsonicJSONRenderer(renderers.JSONRenderer):
             }
         }
         final['subsonic-response'].update(data)
+        if 'error' in final:
+            # an error was returned
+            final['subsonic-response']['status'] = 'failed'
         return super().render(final, accepted_media_type, renderer_context)
 
 
@@ -31,6 +34,9 @@ class SubsonicXMLRenderer(renderers.JSONRenderer):
             'version': '1.16.0',
         }
         final.update(data)
+        if 'error' in final:
+            # an error was returned
+            final['status'] = 'failed'
         tree = dict_to_xml_tree('subsonic-response', final)
         return b'<?xml version="1.0" encoding="UTF-8"?>\n' + ET.tostring(tree, encoding='utf-8')
 

api/funkwhale_api/subsonic/serializers.py

@@ -81,6 +81,10 @@ def get_track_data(album, track, tf):
         'artistId': album.artist.pk,
         'type': 'music',
     }
+    if tf.bitrate:
+        data['bitrate'] = int(tf.bitrate/1000)
+    if tf.size:
+        data['size'] = tf.size
     if album.release_date:
         data['year'] = album.release_date.year
     return data
@@ -211,5 +215,9 @@ def get_music_directory_data(artist):
             'parent': artist.id,
             'type': 'music',
         }
+        if tf.bitrate:
+            td['bitrate'] = int(tf.bitrate/1000)
+        if tf.size:
+            td['size'] = tf.size
         data['child'].append(td)
     return data

api/funkwhale_api/subsonic/views.py

@@ -31,15 +31,19 @@ def find_object(queryset, model_field='pk', field='id', cast=int):
                 raw_value = data[field]
             except KeyError:
                 return response.Response({
-                    'code': 10,
-                    'message': "required parameter '{}' not present".format(field)
+                    'error': {
+                        'code': 10,
+                        'message': "required parameter '{}' not present".format(field)
+                    }
                 })
             try:
                 value = cast(raw_value)
             except (TypeError, ValidationError):
                 return response.Response({
-                    'code': 0,
-                    'message': 'For input string "{}"'.format(raw_value)
+                    'error': {
+                        'code': 0,
+                        'message': 'For input string "{}"'.format(raw_value)
+                    }
                 })
             qs = queryset
             if hasattr(qs, '__call__'):
@@ -48,9 +52,11 @@ def find_object(queryset, model_field='pk', field='id', cast=int):
                 obj = qs.get(**{model_field: value})
             except qs.model.DoesNotExist:
                 return response.Response({
-                    'code': 70,
-                    'message': '{} not found'.format(
-                        qs.model.__class__.__name__)
+                    'error': {
+                        'code': 70,
+                        'message': '{} not found'.format(
+                            qs.model.__class__.__name__)
+                    }
                 })
             kwargs['obj'] = obj
             return func(self, request, *args, **kwargs)
@@ -83,15 +89,14 @@ class SubsonicViewSet(viewsets.GenericViewSet):
         payload = {
             'status': 'failed'
         }
-        try:
+        if exc.__class__ in mapping:
             code, message = mapping[exc.__class__]
-        except KeyError:
-            return super().handle_exception(exc)
         else:
-            payload['error'] = {
-                'code': code,
-                'message': message
-            }
+            return super().handle_exception(exc)
+        payload['error'] = {
+            'code': code,
+            'message': message
+        }
 
         return response.Response(payload, status=200)
 
@@ -450,8 +455,10 @@ class SubsonicViewSet(viewsets.GenericViewSet):
         name = data.get('name', '')
         if not name:
             return response.Response({
-                'code': 10,
-                'message': 'Playlist ID or name must be specified.'
+                'error': {
+                    'code': 10,
+                    'message': 'Playlist ID or name must be specified.'
+                }
             }, data)
 
         playlist = request.user.playlists.create(

api/funkwhale_api/users/admin.py

@@ -5,6 +5,7 @@ from django import forms
 from django.contrib import admin
 from django.contrib.auth.admin import UserAdmin as AuthUserAdmin
 from django.contrib.auth.forms import UserChangeForm, UserCreationForm
+from django.utils.translation import ugettext_lazy as _
 
 from .models import User
 
@@ -41,8 +42,33 @@ class UserAdmin(AuthUserAdmin):
         'email',
         'date_joined',
         'last_login',
-        'privacy_level',
+        'is_staff',
+        'is_superuser',
     ]
     list_filter = [
+        'is_superuser',
+        'is_staff',
         'privacy_level',
+        'permission_settings',
+        'permission_library',
+        'permission_federation',
     ]
+
+    fieldsets = (
+        (None, {'fields': ('username', 'password', 'privacy_level')}),
+        (_('Personal info'), {'fields': ('first_name', 'last_name', 'email')}),
+        (_('Permissions'), {
+            'fields': (
+                'is_active',
+                'is_staff',
+                'is_superuser',
+                'permission_library',
+                'permission_settings',
+                'permission_federation')}),
+        (_('Important dates'), {'fields': ('last_login', 'date_joined')}),
+        (_('Useless fields'), {
+            'fields': (
+                'user_permissions',
+                'groups',
+            )})
+        )

api/funkwhale_api/users/dynamic_preferences_registry.py

@@ -10,6 +10,7 @@ class RegistrationEnabled(types.BooleanPreference):
     section = users
     name = 'registration_enabled'
     default = False
-    verbose_name = (
-        'Can visitors open a new account on this instance?'
+    verbose_name = 'Open registrations to new users'
+    help_text = (
+        'When enabled, new users will be able to register on this instance.'
     )

api/funkwhale_api/users/factories.py

 import factory
 
-from funkwhale_api.factories import registry
+from funkwhale_api.factories import registry, ManyToManyFromList
 from django.contrib.auth.models import Permission
 
 
+@registry.register
+class GroupFactory(factory.django.DjangoModelFactory):
+    name = factory.Sequence(lambda n: 'group-{0}'.format(n))
+
+    class Meta:
+        model = 'auth.Group'
+
+    @factory.post_generation
+    def perms(self, create, extracted, **kwargs):
+        if not create:
+            # Simple build, do nothing.
+            return
+
+        if extracted:
+            perms = [
+                Permission.objects.get(
+                    content_type__app_label=p.split('.')[0],
+                    codename=p.split('.')[1],
+                )
+                for p in extracted
+            ]
+            # A list of permissions were passed in, use them
+            self.permissions.add(*perms)
+
+
 @registry.register
 class UserFactory(factory.django.DjangoModelFactory):
     username = factory.Sequence(lambda n: 'user-{0}'.format(n))
     email = factory.Sequence(lambda n: 'user-{0}@example.com'.format(n))
     password = factory.PostGenerationMethodCall('set_password', 'test')
     subsonic_api_token = None
+    groups = ManyToManyFromList('groups')
 
     class Meta:
         model = 'users.User'