Merge branch 'release/0.10'

.env.dev

@@ -9,3 +9,4 @@ FUNKWHALE_HOSTNAME=localhost
 FUNKWHALE_PROTOCOL=http
 PYTHONDONTWRITEBYTECODE=true
 WEBPACK_DEVSERVER_PORT=8080
+MUSIC_DIRECTORY_PATH=/music

.gitlab-ci.yml

@@ -70,7 +70,9 @@ build_front:
     - yarn install
     - yarn run i18n-extract
     - yarn run i18n-compile
-    - yarn run build
+    # this is to ensure we don't have any errors in the output,
+    # cf https://code.eliotberriot.com/funkwhale/funkwhale/issues/169
+    - yarn run build | tee /dev/stderr | (! grep -i 'ERROR in')
   cache:
     key: "$CI_PROJECT_ID__front_dependencies"
     paths:
@@ -81,9 +83,9 @@ build_front:
     paths:
       - front/dist/
   only:
-    - tags
-    - master
-    - develop
+    - tags@funkwhale/funkwhale
+    - master@funkwhale/funkwhale
+    - develop@funkwhale/funkwhale
   tags:
     - docker
 
@@ -100,7 +102,7 @@ pages:
     paths:
       - public
   only:
-    - develop
+    - develop@funkwhale/funkwhale
   tags:
     - docker
 
@@ -114,7 +116,7 @@ docker_develop:
     - docker build -t $IMAGE .
     - docker push $IMAGE
   only:
-    - develop
+    - develop@funkwhale/funkwhale
   tags:
     - dind
 
@@ -128,9 +130,9 @@ build_api:
       - api
   script: echo Done!
   only:
-    - tags
-    - master
-    - develop
+    - tags@funkwhale/funkwhale
+    - master@funkwhale/funkwhale
+    - develop@funkwhale/funkwhale
 
 
 docker_release:
@@ -144,6 +146,6 @@ docker_release:
     - docker push $IMAGE
     - docker push $IMAGE_LATEST
   only:
-    - tags
+    - tags@funkwhale/funkwhale
   tags:
     - dind

CHANGELOG

@@ -3,6 +3,88 @@ Changelog
 
 .. towncrier
 
+0.10 (2018-04-23)
+-----------------
+
+Features:
+
+- Can now import files in-place from the CLI importer (#155)
+
+
+Enhancements:
+
+- Avoid downloading audio files multiple times from remote libraries (#163)
+- Better file import performance and error handling (#144)
+- Import job and batch API and front-end have been improved with better
+  performance, pagination and additional filters (#171)
+- Increased max_length on TrackFile.source, this will help when importing files
+  with a really long path (#142)
+- Player is back in Queue tab (#150)
+
+
+Bugfixes:
+
+- Fail graciously when AP representation includes a null_value for mediaType
+- Fix sidebar tabs not showing under small resolution under Chrome (#173)
+- Fixed broken login due to badly configured Axios (#172)
+- Fixed broken playlist modal after login (#155)
+- Fixed queue reorder or track deletion restarting currently playing track
+  (#151)
+- Radio will now append new track if you delete the last track in queue (#145)
+- Reset all sensitive front-end data on logout (#124)
+- Typos/not showing text due to i18n work (#175)
+
+
+Documentation:
+
+- Better documentation for hardware requirements and memory usage (#165)
+
+
+In-place import
+^^^^^^^^^^^^^^^
+
+This release includes in-place imports for the CLI import. This means you can
+load gigabytes of music into funkwhale without worrying about about Funkwhale
+copying those music files in its internal storage and eating your disk space.
+
+`This new feature is documented here <https://docs.funkwhale.audio/importing-music.html#in-place-import>`_
+and require additional configuration to ensure funkwhale and your webserver can
+serve those files properly.
+
+**Non-docker users:**
+
+Assuming your music is stored in ``/srv/funkwhale/data/music``, add the following
+block to your nginx configuration::
+
+    location /_protected/music {
+        internal;
+        alias   /srv/funkwhale/data/music;
+    }
+
+And the following to your .env file::
+
+    MUSIC_DIRECTORY_PATH=/srv/funkwhale/data/music
+
+**Docker users:**
+
+Assuming your music is stored in ``/srv/funkwhale/data/music``, add the following
+block to your nginx configuration::
+
+    location /_protected/music {
+        internal;
+        alias   /srv/funkwhale/data/music;
+    }
+
+Assuming you have the following volume directive in your ``docker-compose.yml``
+(it's the default): ``/srv/funkwhale/data/music:/music:ro``, then add
+the following to your .env file::
+
+    # this is the path in the container
+    MUSIC_DIRECTORY_PATH=/music
+    # this is the path on the host
+    MUSIC_DIRECTORY_SERVE_PATH=/srv/funkwhale/data/music
+
+
 0.9.1 (2018-04-17)
 ------------------
 

api/config/settings/common.py

@@ -390,6 +390,12 @@ REST_FRAMEWORK = {
 ATOMIC_REQUESTS = False
 USE_X_FORWARDED_HOST = True
 USE_X_FORWARDED_PORT = True
+
+# Wether we should use Apache, Nginx (or other) headers when serving audio files
+# Default to Nginx
+REVERSE_PROXY_TYPE = env('REVERSE_PROXY_TYPE', default='nginx')
+assert REVERSE_PROXY_TYPE in ['apache2', 'nginx'], 'Unsupported REVERSE_PROXY_TYPE'
+
 # Wether we should check user permission before serving audio files (meaning
 # return an obfuscated url)
 # This require a special configuration on the reverse proxy side
@@ -441,3 +447,9 @@ EXTERNAL_REQUESTS_VERIFY_SSL = env.bool(
     'EXTERNAL_REQUESTS_VERIFY_SSL',
     default=True
 )
+
+MUSIC_DIRECTORY_PATH = env('MUSIC_DIRECTORY_PATH', default=None)
+# on Docker setup, the music directory may not match the host path,
+# and we need to know it for it to serve stuff properly
+MUSIC_DIRECTORY_SERVE_PATH = env(
+    'MUSIC_DIRECTORY_SERVE_PATH', default=MUSIC_DIRECTORY_PATH)

api/funkwhale_api/__init__.py

 # -*- coding: utf-8 -*-
-__version__ = '0.9.1'
+__version__ = '0.10'
 __version_info__ = tuple([int(num) if num.isdigit() else num for num in __version__.replace('-', '.', 1).split('.')])

api/funkwhale_api/federation/serializers.py

@@ -708,23 +708,7 @@ class AudioSerializer(serializers.Serializer):
         except (KeyError, TypeError):
             raise serializers.ValidationError('Missing mediaType')
 
-        if not media_type.startswith('audio/'):
-            raise serializers.ValidationError('Invalid mediaType')
-
-        return url
-
-    def validate_url(self, v):
-        try:
-            url = v['href']
-        except (KeyError, TypeError):
-            raise serializers.ValidationError('Missing href')
-
-        try:
-            media_type = v['mediaType']
-        except (KeyError, TypeError):
-            raise serializers.ValidationError('Missing mediaType')
-
-        if not media_type.startswith('audio/'):
+        if not media_type or not media_type.startswith('audio/'):
             raise serializers.ValidationError('Invalid mediaType')
 
         return v

api/funkwhale_api/music/factories.py

@@ -43,6 +43,7 @@ class TrackFactory(factory.django.DjangoModelFactory):
     artist = factory.SelfAttribute('album.artist')
     position = 1
     tags = ManyToManyFromList('tags')
+
     class Meta:
         model = 'music.Track'
 
@@ -57,6 +58,9 @@ class TrackFileFactory(factory.django.DjangoModelFactory):
         model = 'music.TrackFile'
 
     class Params:
+        in_place = factory.Trait(
+            audio_file=None,
+        )
         federation = factory.Trait(
             audio_file=None,
             library_track=factory.SubFactory(LibraryTrackFactory),
@@ -105,6 +109,10 @@ class ImportJobFactory(factory.django.DjangoModelFactory):
             status='finished',
             track_file=factory.SubFactory(TrackFileFactory),
         )
+        in_place = factory.Trait(
+            status='finished',
+            audio_file=None,
+        )
 
 
 @registry.register(name='music.FileImportJob')

api/funkwhale_api/music/filters.py

@@ -2,6 +2,7 @@ from django.db.models import Count
 
 from django_filters import rest_framework as filters
 
+from funkwhale_api.common import fields
 from . import models
 
 
@@ -28,6 +29,39 @@ class ArtistFilter(ListenableMixin):
         }
 
 
+class ImportBatchFilter(filters.FilterSet):
+    q = fields.SearchFilter(search_fields=[
+        'submitted_by__username',
+        'source',
+    ])
+
+    class Meta:
+        model = models.ImportBatch
+        fields = {
+            'status': ['exact'],
+            'source': ['exact'],
+            'submitted_by': ['exact'],
+        }
+
+
+class ImportJobFilter(filters.FilterSet):
+    q = fields.SearchFilter(search_fields=[
+        'batch__submitted_by__username',
+        'source',
+    ])
+
+    class Meta:
+        model = models.ImportJob
+        fields = {
+            'batch': ['exact'],
+            'batch__status': ['exact'],
+            'batch__source': ['exact'],
+            'batch__submitted_by': ['exact'],
+            'status': ['exact'],
+            'source': ['exact'],
+        }
+
+
 class AlbumFilter(ListenableMixin):
     listenable = filters.BooleanFilter(name='_', method='filter_listenable')
 

api/funkwhale_api/music/metadata.py

-import mutagen
+from django import forms
 import arrow
+import mutagen
 
 NODEFAULT = object()
 
@@ -50,6 +51,13 @@ def convert_track_number(v):
     except (ValueError, AttributeError, IndexError):
         pass
 
+
+VALIDATION = {
+    'musicbrainz_artistid': forms.UUIDField(),
+    'musicbrainz_albumid': forms.UUIDField(),
+    'musicbrainz_recordingid': forms.UUIDField(),
+}
+
 CONF = {
     'OggVorbis': {
         'getter': lambda f, k: f[k][0],
@@ -146,4 +154,7 @@ class Metadata(object):
         converter = field_conf.get('to_application')
         if converter:
             v = converter(v)
+        field = VALIDATION.get(key)
+        if field:
+            v = field.to_python(v)
         return v

api/funkwhale_api/music/migrations/0025_auto_20180419_2023.py

+# Generated by Django 2.0.3 on 2018-04-19 20:23
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0024_populate_uuid'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='trackfile',
+            name='source',
+            field=models.URLField(blank=True, max_length=500, null=True),
+        ),
+    ]

api/funkwhale_api/music/models.py

@@ -412,7 +412,7 @@ class TrackFile(models.Model):
     track = models.ForeignKey(
         Track, related_name='files', on_delete=models.CASCADE)
     audio_file = models.FileField(upload_to='tracks/%Y/%m/%d', max_length=255)
-    source = models.URLField(null=True, blank=True)
+    source = models.URLField(null=True, blank=True, max_length=500)
     creation_date = models.DateTimeField(default=timezone.now)
     modification_date = models.DateTimeField(auto_now=True)
     duration = models.IntegerField(null=True, blank=True)
@@ -463,6 +463,26 @@ class TrackFile(models.Model):
             self.mimetype = utils.guess_mimetype(self.audio_file)
         return super().save(**kwargs)
 
+    @property
+    def serve_from_source_path(self):
+        if not self.source or not self.source.startswith('file://'):
+            raise ValueError('Cannot serve this file from source')
+        serve_path = settings.MUSIC_DIRECTORY_SERVE_PATH
+        prefix = settings.MUSIC_DIRECTORY_PATH
+        if not serve_path or not prefix:
+            raise ValueError(
+                'You need to specify MUSIC_DIRECTORY_SERVE_PATH and '
+                'MUSIC_DIRECTORY_PATH to serve in-place imported files'
+            )
+        file_path = self.source.replace('file://', '', 1)
+        parts = os.path.split(file_path.replace(prefix, '', 1))
+        if parts[0] == '/':
+            parts = parts[1:]
+        return os.path.join(
+            serve_path,
+            *parts
+        )
+
 
 IMPORT_STATUS_CHOICES = (
     ('pending', 'Pending'),
@@ -507,6 +527,8 @@ class ImportBatch(models.Model):
     def update_status(self):
         old_status = self.status
         self.status = utils.compute_status(self.jobs.all())
+        if self.status == old_status:
+            return
         self.save(update_fields=['status'])
         if self.status != old_status and self.status == 'finished':
             from . import tasks

api/funkwhale_api/music/serializers.py

@@ -6,6 +6,7 @@ from funkwhale_api.activity import serializers as activity_serializers
 from funkwhale_api.federation import utils as federation_utils
 from funkwhale_api.federation.models import LibraryTrack
 from funkwhale_api.federation.serializers import AP_CONTEXT
+from funkwhale_api.users.serializers import UserBasicSerializer
 
 from . import models
 
@@ -90,6 +91,7 @@ class TrackSerializerNested(LyricsMixin):
     files = TrackFileSerializer(many=True, read_only=True)
     album = SimpleAlbumSerializer(read_only=True)
     tags = TagSerializer(many=True, read_only=True)
+
     class Meta:
         model = models.Track
         fields = ('id', 'mbid', 'title', 'artist', 'files', 'album', 'tags', 'lyrics')
@@ -108,6 +110,7 @@ class AlbumSerializerNested(serializers.ModelSerializer):
 class ArtistSerializerNested(serializers.ModelSerializer):
     albums = AlbumSerializerNested(many=True, read_only=True)
     tags = TagSerializer(many=True, read_only=True)
+
     class Meta:
         model = models.Artist
         fields = ('id', 'mbid', 'name', 'albums', 'tags')
@@ -121,18 +124,43 @@ class LyricsSerializer(serializers.ModelSerializer):
 
 class ImportJobSerializer(serializers.ModelSerializer):
     track_file = TrackFileSerializer(read_only=True)
+
     class Meta:
         model = models.ImportJob
-        fields = ('id', 'mbid', 'batch', 'source', 'status', 'track_file', 'audio_file')
+        fields = (
+            'id',
+            'mbid',
+            'batch',
+            'source',
+            'status',
+            'track_file',
+            'audio_file')
         read_only_fields = ('status', 'track_file')
 
 
 class ImportBatchSerializer(serializers.ModelSerializer):
-    jobs = ImportJobSerializer(many=True, read_only=True)
+    submitted_by = UserBasicSerializer(read_only=True)
+
     class Meta:
         model = models.ImportBatch
-        fields = ('id', 'jobs', 'status', 'creation_date', 'import_request')
-        read_only_fields = ('creation_date',)
+        fields = (
+            'id',
+            'submitted_by',
+            'source',
+            'status',
+            'creation_date',
+            'import_request')
+        read_only_fields = (
+            'creation_date', 'submitted_by', 'source')
+
+    def to_representation(self, instance):
+        repr = super().to_representation(instance)
+        try:
+            repr['job_count'] = instance.job_count
+        except AttributeError:
+            # Queryset was not annotated
+            pass
+        return repr
 
 
 class TrackActivitySerializer(activity_serializers.ModelSerializer):

api/funkwhale_api/music/tasks.py

@@ -71,7 +71,7 @@ def import_track_from_remote(library_track):
         library_track.title, artist=artist, album=album)
 
 
-def _do_import(import_job, replace, use_acoustid=True):
+def _do_import(import_job, replace=False, use_acoustid=True):
     from_file = bool(import_job.audio_file)
     mbid = import_job.mbid
     acoustid_track_id = None
@@ -93,6 +93,9 @@ def _do_import(import_job, replace, use_acoustid=True):
         track = import_track_data_from_path(import_job.audio_file.path)
     elif import_job.library_track:
         track = import_track_from_remote(import_job.library_track)
+    elif import_job.source.startswith('file://'):
+        track = import_track_data_from_path(
+            import_job.source.replace('file://', '', 1))
     else:
         raise ValueError(
             'Not enough data to process import, '
@@ -123,7 +126,8 @@ def _do_import(import_job, replace, use_acoustid=True):
         else:
             # no downloading, we hotlink
             pass
-    else:
+    elif not import_job.audio_file and not import_job.source.startswith('file://'):
+        # not an implace import, and we have a source, so let's download it
         track_file.download_file()
     track_file.save()
     import_job.status = 'finished'
@@ -133,7 +137,7 @@ def _do_import(import_job, replace, use_acoustid=True):
         import_job.audio_file.delete()
     import_job.save()
 
-    return track.pk
+    return track_file
 
 
 @celery.app.task(name='ImportJob.run', bind=True)
@@ -147,7 +151,8 @@ def import_job_run(self, import_job, replace=False, use_acoustid=True):
         import_job.save(update_fields=['status'])
 
     try:
-        return _do_import(import_job, replace, use_acoustid=use_acoustid)
+        tf = _do_import(import_job, replace, use_acoustid=use_acoustid)
+        return tf.pk if tf else None
     except Exception as exc:
         if not settings.DEBUG:
             try:

api/funkwhale_api/music/utils.py

@@ -53,10 +53,11 @@ def guess_mimetype(f):
 
 
 def compute_status(jobs):
-    errored = any([job.status == 'errored' for job in jobs])
+    statuses = jobs.order_by().values_list('status', flat=True).distinct()
+    errored = any([status == 'errored' for status in statuses])
     if errored:
         return 'errored'
-    pending = any([job.status == 'pending' for job in jobs])
+    pending = any([status == 'pending' for status in statuses])
     if pending:
         return 'pending'
     return 'finished'

api/funkwhale_api/music/views.py

@@ -11,6 +11,7 @@ from django.core.exceptions import ObjectDoesNotExist
 from django.conf import settings
 from django.db import models, transaction
 from django.db.models.functions import Length
+from django.db.models import Count
 from django.http import StreamingHttpResponse
 from django.urls import reverse
 from django.utils.decorators import method_decorator
@@ -23,13 +24,14 @@ from rest_framework import permissions
 from musicbrainzngs import ResponseError
 
 from funkwhale_api.common import utils as funkwhale_utils
-from funkwhale_api.federation import actors
-from funkwhale_api.requests.models import ImportRequest
-from funkwhale_api.musicbrainz import api
 from funkwhale_api.common.permissions import (
     ConditionalAuthentication, HasModelPermission)
 from taggit.models import Tag
+from funkwhale_api.federation import actors
 from funkwhale_api.federation.authentication import SignatureAuthentication
+from funkwhale_api.federation.models import LibraryTrack
+from funkwhale_api.musicbrainz import api
+from funkwhale_api.requests.models import ImportRequest
 
 from . import filters
 from . import forms
@@ -98,14 +100,14 @@ class ImportBatchViewSet(
         mixins.RetrieveModelMixin,
         viewsets.GenericViewSet):
     queryset = (
-        models.ImportBatch.objects.all()
-                          .prefetch_related('jobs__track_file')
-                          .order_by('-creation_date'))
+        models.ImportBatch.objects
+              .select_related()
+              .order_by('-creation_date')
+              .annotate(job_count=Count('jobs'))
+    )
     serializer_class = serializers.ImportBatchSerializer
     permission_classes = (permissions.DjangoModelPermissions, )
-
-    def get_queryset(self):
-        return super().get_queryset().filter(submitted_by=self.request.user)
+    filter_class = filters.ImportBatchFilter
 
     def perform_create(self, serializer):
         serializer.save(submitted_by=self.request.user)
@@ -118,13 +120,30 @@ class ImportJobPermission(HasModelPermission):
 
 class ImportJobViewSet(
         mixins.CreateModelMixin,
+        mixins.ListModelMixin,
         viewsets.GenericViewSet):
-    queryset = (models.ImportJob.objects.all())
+    queryset = (models.ImportJob.objects.all().select_related())
     serializer_class = serializers.ImportJobSerializer
     permission_classes = (ImportJobPermission, )
+    filter_class = filters.ImportJobFilter
 
-    def get_queryset(self):
-        return super().get_queryset().filter(batch__submitted_by=self.request.user)
+    @list_route(methods=['get'])
+    def stats(self, request, *args, **kwargs):
+        qs = models.ImportJob.objects.all()
+        filterset = filters.ImportJobFilter(request.GET, queryset=qs)
+        qs = filterset.qs
+        qs = qs.values('status').order_by('status')
+        qs = qs.annotate(status_count=Count('status'))
+
+        data = {}
+        for row in qs:
+            data[row['status']] = row['status_count']
+
+        for s, _ in models.IMPORT_STATUS_CHOICES:
+            data.setdefault(s, 0)
+
+        data['count'] = sum([v for v in data.values()])
+        return Response(data)
 
     def perform_create(self, serializer):
         source = 'file://' + serializer.validated_data['audio_file'].name
@@ -135,7 +154,8 @@ class ImportJobViewSet(
         )
 
 
-class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet):
+class TrackViewSet(
+        TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet):
     """
     A simple ViewSet for viewing and editing accounts.
     """
@@ -185,6 +205,25 @@ class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet):
         return Response(serializer.data)
 
 
+def get_file_path(audio_file):
+    t = settings.REVERSE_PROXY_TYPE
+    if t == 'nginx':
+        # we have to use the internal locations
+        try:
+            path = audio_file.url
+        except AttributeError:
+            # a path was given
+            path = '/music' + audio_file
+        return settings.PROTECT_FILES_PATH + path