Merge branch 'release/0.5'

.gitignore

@@ -72,7 +72,7 @@ api/music
 api/media
 api/staticfiles
 api/static
-
+api/.pytest_cache
 
 # Front
 front/node_modules/

CHANGELOG

 Changelog
 =========
 
+0.6 (Unreleased)
+----------------
+
 
-0.5 (Unreleased)
+0.5 (2018-02-24)
 ----------------
 
+- Front: Now reset player colors when track has no cover (#46)
+- Front: play button now disabled for unplayable tracks
+- API: You can now enable or disable registration on the fly, via a preference (#58)
+- Front: can now signup via the web interface (#35)
+- Front: Fixed broken redirection on login
+- Front: Fixed broken error handling on settings and login form
+
+About page:
+
+There is a brand new about page on instances (/about), and instance
+owner can now provide a name, a short and a long description for their instance via the admin (/api/admin/dynamic_preferences/globalpreferencemodel/).
+
+Transcoding:
+
+Basic transcoding is now available to/from the following formats : ogg and mp3.
+
+*This is still an alpha feature at the moment, please report any bug.*
+
+This relies internally on FFMPEG and can put some load on your server.
+It's definitely recommended you setup some caching for the transcoded files
+at your webserver level. Check the the exemple nginx file at deploy/nginx.conf
+for an implementation.
+
+On the frontend, usage of transcoding should be transparent in the player.
+
+Music Requests:
+
+This release includes a new feature, music requests, which allows users
+to request music they'd like to see imported.
+Admins can browse those requests and mark them as completed when
+an import is made.
 
 0.4 (2018-02-18)
 ----------------

api/Dockerfile

@@ -3,7 +3,7 @@ FROM python:3.5
 ENV PYTHONUNBUFFERED 1
 
 # Requirements have to be pulled and installed here, otherwise caching won't work
-
+RUN echo 'deb http://httpredir.debian.org/debian/ jessie-backports main' > /etc/apt/sources.list.d/ffmpeg.list
 COPY ./requirements.apt /requirements.apt
 RUN apt-get update -qq && grep "^[^#;]" requirements.apt | xargs apt-get install -y
 RUN curl -L https://github.com/acoustid/chromaprint/releases/download/v1.4.2/chromaprint-fpcalc-1.4.2-linux-x86_64.tar.gz | tar -xz -C /usr/local/bin --strip 1

api/config/api_urls.py

@@ -52,6 +52,10 @@ v1_patterns += [
         include(
             ('funkwhale_api.users.api_urls', 'users'),
             namespace='users')),
+    url(r'^requests/',
+        include(
+            ('funkwhale_api.requests.api_urls', 'requests'),
+            namespace='requests')),
     url(r'^token/$', jwt_views.obtain_jwt_token, name='token'),
     url(r'^token/refresh/$', jwt_views.refresh_jwt_token, name='token_refresh'),
 ]

api/config/settings/common.py

@@ -80,10 +80,12 @@ if RAVEN_ENABLED:
 
 # Apps specific for this project go here.
 LOCAL_APPS = (
+    'funkwhale_api.common',
     'funkwhale_api.users',  # custom users app
     # Your stuff: custom apps go here
     'funkwhale_api.instance',
     'funkwhale_api.music',
+    'funkwhale_api.requests',
     'funkwhale_api.favorites',
     'funkwhale_api.radios',
     'funkwhale_api.history',
@@ -262,7 +264,7 @@ AUTHENTICATION_BACKENDS = (
 )
 
 # Some really nice defaults
-ACCOUNT_AUTHENTICATION_METHOD = 'username'
+ACCOUNT_AUTHENTICATION_METHOD = 'username_email'
 ACCOUNT_EMAIL_REQUIRED = True
 ACCOUNT_EMAIL_VERIFICATION = 'mandatory'
 
@@ -315,7 +317,6 @@ CORS_ORIGIN_ALLOW_ALL = True
 # )
 CORS_ALLOW_CREDENTIALS = True
 API_AUTHENTICATION_REQUIRED = env.bool("API_AUTHENTICATION_REQUIRED", True)
-REGISTRATION_MODE = env('REGISTRATION_MODE', default='disabled')
 REST_FRAMEWORK = {
     'DEFAULT_PERMISSION_CLASSES': (
         'rest_framework.permissions.IsAuthenticated',

api/config/urls.py

@@ -13,8 +13,8 @@ urlpatterns = [
     url(settings.ADMIN_URL, admin.site.urls),
 
     url(r'^api/', include(("config.api_urls", 'api'), namespace="api")),
-    url(r'^api/auth/', include('rest_auth.urls')),
-    url(r'^api/auth/registration/', include('funkwhale_api.users.rest_auth_urls')),
+    url(r'^api/v1/auth/', include('rest_auth.urls')),
+    url(r'^api/v1/auth/registration/', include('funkwhale_api.users.rest_auth_urls')),
     url(r'^accounts/', include('allauth.urls')),
 
     # Your stuff: custom urls includes go here

api/docker/Dockerfile.test

 FROM python:3.5
 
 ENV PYTHONUNBUFFERED 1
-ENV PYTHONDONTWRITEBYTECODE  1
+ENV PYTHONDONTWRITEBYTECODE 1
 
 # Requirements have to be pulled and installed here, otherwise caching won't work
+RUN echo 'deb http://httpredir.debian.org/debian/ jessie-backports main' > /etc/apt/sources.list.d/ffmpeg.list
 COPY ./requirements.apt /requirements.apt
 COPY ./install_os_dependencies.sh /install_os_dependencies.sh
 RUN bash install_os_dependencies.sh install

api/funkwhale_api/__init__.py

 # -*- coding: utf-8 -*-
-__version__ = '0.4'
+__version__ = '0.5'
 __version_info__ = tuple([int(num) if num.isdigit() else num for num in __version__.replace('-', '.', 1).split('.')])

api/funkwhale_api/history/views.py

@@ -17,9 +17,6 @@ class ListeningViewSet(mixins.CreateModelMixin,
     queryset = models.Listening.objects.all()
     permission_classes = [ConditionalAuthentication]
 
-    def create(self, request, *args, **kwargs):
-        return super().create(request, *args, **kwargs)
-
     def get_queryset(self):
         queryset = super().get_queryset()
         if self.request.user.is_authenticated:

api/funkwhale_api/instance/dynamic_preferences_registry.py

+from django.forms import widgets
+
 from dynamic_preferences import types
 from dynamic_preferences.registries import global_preferences_registry
 
 raven = types.Section('raven')
+instance = types.Section('instance')
+
+
+@global_preferences_registry.register
+class InstanceName(types.StringPreference):
+    show_in_api = True
+    section = instance
+    name = 'name'
+    default = ''
+    help_text = 'Instance public name'
+    verbose_name = 'The public name of your instance'
+
 
+@global_preferences_registry.register
+class InstanceShortDescription(types.StringPreference):
+    show_in_api = True
+    section = instance
+    name = 'short_description'
+    default = ''
+    verbose_name = 'Instance succinct description'
+
+
+@global_preferences_registry.register
+class InstanceLongDescription(types.StringPreference):
+    show_in_api = True
+    section = instance
+    name = 'long_description'
+    default = ''
+    help_text = 'Instance long description (markdown allowed)'
+    field_kwargs = {
+        'widget': widgets.Textarea
+    }
 
 @global_preferences_registry.register
 class RavenDSN(types.StringPreference):

api/funkwhale_api/music/forms.py

+from django import forms
+
+from . import models
+
+
+class TranscodeForm(forms.Form):
+    FORMAT_CHOICES = [
+        ('ogg', 'ogg'),
+        ('mp3', 'mp3'),
+    ]
+
+    to = forms.ChoiceField(choices=FORMAT_CHOICES)
+    BITRATE_CHOICES = [
+        (64, '64'),
+        (128, '128'),
+        (256, '256'),
+    ]
+    bitrate = forms.ChoiceField(
+        choices=BITRATE_CHOICES, required=False)
+
+    track_file = forms.ModelChoiceField(
+        queryset=models.TrackFile.objects.all()
+    )

api/funkwhale_api/music/migrations/0018_auto_20180218_1554.py

+# Generated by Django 2.0.2 on 2018-02-18 15:54
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0017_auto_20171227_1728'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='trackfile',
+            name='mimetype',
+            field=models.CharField(blank=True, max_length=200, null=True),
+        ),
+        migrations.AlterField(
+            model_name='importjob',
+            name='source',
+            field=models.CharField(max_length=500),
+        ),
+        migrations.AlterField(
+            model_name='importjob',
+            name='status',
+            field=models.CharField(choices=[('pending', 'Pending'), ('finished', 'Finished'), ('errored', 'Errored'), ('skipped', 'Skipped')], default='pending', max_length=30),
+        ),
+    ]

api/funkwhale_api/music/migrations/0019_populate_mimetypes.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+import os
+
+from django.db import migrations, models
+from funkwhale_api.music.utils import guess_mimetype
+
+
+def populate_mimetype(apps, schema_editor):
+    TrackFile = apps.get_model("music", "TrackFile")
+
+    for tf in TrackFile.objects.filter(audio_file__isnull=False, mimetype__isnull=True).only('audio_file'):
+        try:
+            tf.mimetype = guess_mimetype(tf.audio_file)
+        except Exception as e:
+            print('Error on track file {}: {}'.format(tf.pk, e))
+            continue
+        print('Track file {}: {}'.format(tf.pk, tf.mimetype))
+        tf.save(update_fields=['mimetype'])
+
+
+def rewind(apps, schema_editor):
+    pass
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0018_auto_20180218_1554'),
+    ]
+
+    operations = [
+        migrations.RunPython(populate_mimetype, rewind),
+    ]

api/funkwhale_api/music/migrations/0020_importbatch_status.py

+# Generated by Django 2.0.2 on 2018-02-20 19:12
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0019_populate_mimetypes'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='importbatch',
+            name='status',
+            field=models.CharField(choices=[('pending', 'Pending'), ('finished', 'Finished'), ('errored', 'Errored'), ('skipped', 'Skipped')], default='pending', max_length=30),
+        ),
+    ]

api/funkwhale_api/music/migrations/0021_populate_batch_status.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+import os
+
+from django.db import migrations, models
+
+
+def populate_status(apps, schema_editor):
+    from funkwhale_api.music.utils import compute_status
+    ImportBatch = apps.get_model("music", "ImportBatch")
+
+    for ib in ImportBatch.objects.prefetch_related('jobs'):
+        ib.status = compute_status(ib.jobs.all())
+        ib.save(update_fields=['status'])
+
+
+def rewind(apps, schema_editor):
+    pass
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0020_importbatch_status'),
+    ]
+
+    operations = [
+        migrations.RunPython(populate_status, rewind),
+    ]

api/funkwhale_api/music/migrations/0022_importbatch_import_request.py

+# Generated by Django 2.0.2 on 2018-02-20 22:48
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('requests', '__first__'),
+        ('music', '0021_populate_batch_status'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='importbatch',
+            name='import_request',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='import_batches', to='requests.ImportRequest'),
+        ),
+    ]

api/funkwhale_api/music/models.py

@@ -10,14 +10,18 @@ from django.conf import settings
 from django.db import models
 from django.core.files.base import ContentFile
 from django.core.files import File
+from django.db.models.signals import post_save
+from django.dispatch import receiver
 from django.urls import reverse
 from django.utils import timezone
+
 from taggit.managers import TaggableManager
 from versatileimagefield.fields import VersatileImageField
 
 from funkwhale_api import downloader
 from funkwhale_api import musicbrainz
 from . import importers
+from . import utils
 
 
 class APIModelMixin(models.Model):
@@ -364,6 +368,7 @@ class TrackFile(models.Model):
     source = models.URLField(null=True, blank=True)
     duration = models.IntegerField(null=True, blank=True)
     acoustid_track_id = models.UUIDField(null=True, blank=True)
+    mimetype = models.CharField(null=True, blank=True, max_length=200)
 
     def download_file(self):
         # import the track file, since there is not any
@@ -393,6 +398,18 @@ class TrackFile(models.Model):
             self.track.full_name,
             os.path.splitext(self.audio_file.name)[-1])
 
+    def save(self, **kwargs):
+        if not self.mimetype and self.audio_file:
+            self.mimetype = utils.guess_mimetype(self.audio_file)
+        return super().save(**kwargs)
+
+
+IMPORT_STATUS_CHOICES = (
+    ('pending', 'Pending'),
+    ('finished', 'Finished'),
+    ('errored', 'Errored'),
+    ('skipped', 'Skipped'),
+)
 
 class ImportBatch(models.Model):
     IMPORT_BATCH_SOURCES = [
@@ -406,22 +423,24 @@ class ImportBatch(models.Model):
         'users.User',
         related_name='imports',
         on_delete=models.CASCADE)
-
+    status = models.CharField(
+        choices=IMPORT_STATUS_CHOICES, default='pending', max_length=30)
+    import_request = models.ForeignKey(
+        'requests.ImportRequest',
+        related_name='import_batches',
+        null=True,
+        blank=True,
+        on_delete=models.CASCADE)
     class Meta:
         ordering = ['-creation_date']
 
     def __str__(self):
         return str(self.pk)
 
-    @property
-    def status(self):
-        pending = any([job.status == 'pending' for job in self.jobs.all()])
-        errored = any([job.status == 'errored' for job in self.jobs.all()])
-        if pending:
-            return 'pending'
-        if errored:
-            return 'errored'
-        return 'finished'
+    def update_status(self):
+        self.status = utils.compute_status(self.jobs.all())
+        self.save(update_fields=['status'])
+
 
 class ImportJob(models.Model):
     batch = models.ForeignKey(
@@ -434,15 +453,39 @@ class ImportJob(models.Model):
         on_delete=models.CASCADE)
     source = models.CharField(max_length=500)
     mbid = models.UUIDField(editable=False, null=True, blank=True)
-    STATUS_CHOICES = (
-        ('pending', 'Pending'),
-        ('finished', 'Finished'),
-        ('errored', 'Errored'),
-        ('skipped', 'Skipped'),
-    )
-    status = models.CharField(choices=STATUS_CHOICES, default='pending', max_length=30)
+
+    status = models.CharField(
+        choices=IMPORT_STATUS_CHOICES, default='pending', max_length=30)
     audio_file = models.FileField(
         upload_to='imports/%Y/%m/%d', max_length=255, null=True, blank=True)
 
     class Meta:
         ordering = ('id', )
+
+
+@receiver(post_save, sender=ImportJob)
+def update_batch_status(sender, instance, **kwargs):
+    instance.batch.update_status()
+
+
+@receiver(post_save, sender=ImportBatch)
+def update_request_status(sender, instance, created, **kwargs):
+    update_fields = kwargs.get('update_fields', []) or []
+    if not instance.import_request:
+        return
+
+    if not created and not 'status' in update_fields:
+        return
+
+    r_status = instance.import_request.status
+    status = instance.status
+
+    if status == 'pending' and r_status == 'pending':
+        # let's mark the request as accepted since we started an import
+        instance.import_request.status = 'accepted'
+        return instance.import_request.save(update_fields=['status'])
+
+    if status == 'finished' and r_status == 'accepted':
+        # let's mark the request as imported since the import is over
+        instance.import_request.status = 'imported'
+        return instance.import_request.save(update_fields=['status'])

api/funkwhale_api/music/serializers.py

@@ -28,7 +28,14 @@ class TrackFileSerializer(serializers.ModelSerializer):
 
     class Meta:
         model = models.TrackFile
-        fields = ('id', 'path', 'duration', 'source', 'filename', 'track')
+        fields = (
+            'id',
+            'path',
+            'duration',
+            'source',
+            'filename',
+            'mimetype',
+            'track')
 
     def get_path(self, o):
         url = o.path
@@ -118,5 +125,5 @@ class ImportBatchSerializer(serializers.ModelSerializer):
     jobs = ImportJobSerializer(many=True, read_only=True)
     class Meta:
         model = models.ImportBatch
-        fields = ('id', 'jobs', 'status', 'creation_date')
+        fields = ('id', 'jobs', 'status', 'creation_date', 'import_request')
         read_only_fields = ('creation_date',)

api/funkwhale_api/music/utils.py

+import magic
 import re
 
 from django.db.models import Q
 
+
 def normalize_query(query_string,
                     findterms=re.compile(r'"([^"]+)"|(\S+)').findall,
                     normspace=re.compile(r'\s{2,}').sub):
@@ -15,6 +17,7 @@ def normalize_query(query_string,
     '''
     return [normspace(' ', (t[0] or t[1]).strip()) for t in findterms(query_string)]
 
+
 def get_query(query_string, search_fields):
     ''' Returns a query, that is a combination of Q objects. That combination
         aims to search keywords within a model by testing the given search fields.
@@ -35,3 +38,18 @@ def get_query(query_string, search_fields):
         else:
             query = query & or_query
     return query
+
+
+def guess_mimetype(f):
+    b = min(100000, f.size)
+    return magic.from_buffer(f.read(b), mime=True)
+
+
+def compute_status(jobs):
+    errored = any([job.status == 'errored' for job in jobs])
+    if errored:
+        return 'errored'
+    pending = any([job.status == 'pending' for job in jobs])
+    if pending:
+        return 'pending'
+    return 'finished'

api/funkwhale_api/music/views.py

+import ffmpeg
 import os
 import json
+import subprocess
 import unicodedata
 import urllib
+
 from django.urls import reverse
 from django.db import models, transaction
 from django.db.models.functions import Length
 from django.conf import settings
+from django.http import StreamingHttpResponse
+
 from rest_framework import viewsets, views, mixins
 from rest_framework.decorators import detail_route, list_route
 from rest_framework.response import Response
@@ -14,11 +19,13 @@ from musicbrainzngs import ResponseError
 from django.contrib.auth.decorators import login_required
 from django.utils.decorators import method_decorator
 
+from funkwhale_api.requests.models import ImportRequest
 from funkwhale_api.musicbrainz import api
 from funkwhale_api.common.permissions import (
     ConditionalAuthentication, HasModelPermission)
 from taggit.models import Tag
 
+from . import forms
 from . import models
 from . import serializers
 from . import importers
@@ -183,6 +190,40 @@ class TrackFileViewSet(viewsets.ReadOnlyModelViewSet):
             f.audio_file.url)
         return response
 
+    @list_route(methods=['get'])
+    def viewable(self, request, *args, **kwargs):
+        return Response({}, status=200)
+
+    @list_route(methods=['get'])
+    def transcode(self, request, *args, **kwargs):
+        form = forms.TranscodeForm(request.GET)
+        if not form.is_valid():
+            return Response(form.errors, status=400)
+
+        f = form.cleaned_data['track_file']
+        output_kwargs = {
+            'format': form.cleaned_data['to']
+        }
+        args = (ffmpeg
+            .input(f.audio_file.path)
+            .output('pipe:', **output_kwargs)
+            .get_args()
+        )
+        # we use a generator here so the view return immediatly and send
+        # file chunk to the browser, instead of blocking a few seconds
+        def _transcode():
+            p = subprocess.Popen(
+                ['ffmpeg'] + args,
+                stdout=subprocess.PIPE)
+            for line in p.stdout:
+                yield line
+
+        response = StreamingHttpResponse(
+            _transcode(), status=200,
+            content_type=form.cleaned_data['to'])
+
+        return response
+
 
 class TagViewSet(viewsets.ReadOnlyModelViewSet):
     queryset = Tag.objects.all().order_by('name')
@@ -274,14 +315,28 @@ class SubmitViewSet(viewsets.ViewSet):
         serializer = serializers.ImportBatchSerializer(batch)
         return Response(serializer.data)
 
+    def get_import_request(self, data):
+        try:
+            raw = data['importRequest']
+        except KeyError:
+            return
+
+        pk = int(raw)
+        try:
+            return ImportRequest.objects.get(pk=pk)
+        except ImportRequest.DoesNotExist:
+            pass
+
     @list_route(methods=['post'])
     @transaction.non_atomic_requests
     def album(self, request, *args, **kwargs):
         data = json.loads(request.body.decode('utf-8'))
-        import_data, batch = self._import_album(data, request, batch=None)
+        import_request = self.get_import_request(data)
+        import_data, batch = self._import_album(
+            data, request, batch=None, import_request=import_request)
         return Response(import_data)
 
-    def _import_album(self, data, request, batch=None):
+    def _import_album(self, data, request, batch=None, import_request=None):
         # we import the whole album here to prevent race conditions that occurs
         # when using get_or_create_from_api in tasks
         album_data = api.releases.get(id=data['releaseId'], includes=models.Album.api_includes)['release']
@@ -292,7 +347,9 @@ class SubmitViewSet(viewsets.ViewSet):
         except ResponseError:
             pass
         if not batch:
-            batch = models.ImportBatch.objects.create(submitted_by=request.user)
+            batch = models.ImportBatch.objects.create(
+                submitted_by=request.user,
+                import_request=import_request)
         for row in data['tracks']:
             try:
                 models.TrackFile.objects.get(track__mbid=row['mbid'])
@@ -306,6 +363,7 @@ class SubmitViewSet(viewsets.ViewSet):
     @transaction.non_atomic_requests
     def artist(self, request, *args, **kwargs):
         data = json.loads(request.body.decode('utf-8'))
+        import_request = self.get_import_request(data)
         artist_data = api.artists.get(id=data['artistId'])['artist']
         cleaned_data = models.Artist.clean_musicbrainz_data(artist_data)
         artist = importers.load(models.Artist, cleaned_data, artist_data, import_hooks=[])
@@ -313,7 +371,8 @@ class SubmitViewSet(viewsets.ViewSet):
         import_data = []
         batch = None
         for row in data['albums']:
-            row_data, batch = self._import_album(row, request, batch=batch)
+            row_data, batch = self._import_album(
+                row, request, batch=batch, import_request=import_request)
             import_data.append(row_data)
 
         return Response(import_data[0])