Verified Commit 0087ae4c authored by Eliot Berriot's avatar Eliot Berriot 💬

Merge branch 'release/0.10'

parents 22cc2048 7e8972a3
Pipeline #801 passed with stages
in 5 minutes and 35 seconds
......@@ -9,3 +9,4 @@ FUNKWHALE_HOSTNAME=localhost
FUNKWHALE_PROTOCOL=http
PYTHONDONTWRITEBYTECODE=true
WEBPACK_DEVSERVER_PORT=8080
MUSIC_DIRECTORY_PATH=/music
......@@ -70,7 +70,9 @@ build_front:
- yarn install
- yarn run i18n-extract
- yarn run i18n-compile
- yarn run build
# this is to ensure we don't have any errors in the output,
# cf https://code.eliotberriot.com/funkwhale/funkwhale/issues/169
- yarn run build | tee /dev/stderr | (! grep -i 'ERROR in')
cache:
key: "$CI_PROJECT_ID__front_dependencies"
paths:
......@@ -81,9 +83,9 @@ build_front:
paths:
- front/dist/
only:
- tags
- master
- develop
- tags@funkwhale/funkwhale
- master@funkwhale/funkwhale
- develop@funkwhale/funkwhale
tags:
- docker
......@@ -100,7 +102,7 @@ pages:
paths:
- public
only:
- develop
- develop@funkwhale/funkwhale
tags:
- docker
......@@ -114,7 +116,7 @@ docker_develop:
- docker build -t $IMAGE .
- docker push $IMAGE
only:
- develop
- develop@funkwhale/funkwhale
tags:
- dind
......@@ -128,9 +130,9 @@ build_api:
- api
script: echo Done!
only:
- tags
- master
- develop
- tags@funkwhale/funkwhale
- master@funkwhale/funkwhale
- develop@funkwhale/funkwhale
docker_release:
......@@ -144,6 +146,6 @@ docker_release:
- docker push $IMAGE
- docker push $IMAGE_LATEST
only:
- tags
- tags@funkwhale/funkwhale
tags:
- dind
......@@ -3,6 +3,88 @@ Changelog
.. towncrier
0.10 (2018-04-23)
-----------------
Features:
- Can now import files in-place from the CLI importer (#155)
Enhancements:
- Avoid downloading audio files multiple times from remote libraries (#163)
- Better file import performance and error handling (#144)
- Import job and batch API and front-end have been improved with better
performance, pagination and additional filters (#171)
- Increased max_length on TrackFile.source, this will help when importing files
with a really long path (#142)
- Player is back in Queue tab (#150)
Bugfixes:
- Fail graciously when AP representation includes a null_value for mediaType
- Fix sidebar tabs not showing under small resolution under Chrome (#173)
- Fixed broken login due to badly configured Axios (#172)
- Fixed broken playlist modal after login (#155)
- Fixed queue reorder or track deletion restarting currently playing track
(#151)
- Radio will now append new track if you delete the last track in queue (#145)
- Reset all sensitive front-end data on logout (#124)
- Typos/not showing text due to i18n work (#175)
Documentation:
- Better documentation for hardware requirements and memory usage (#165)
In-place import
^^^^^^^^^^^^^^^
This release includes in-place imports for the CLI import. This means you can
load gigabytes of music into funkwhale without worrying about about Funkwhale
copying those music files in its internal storage and eating your disk space.
`This new feature is documented here <https://docs.funkwhale.audio/importing-music.html#in-place-import>`_
and require additional configuration to ensure funkwhale and your webserver can
serve those files properly.
**Non-docker users:**
Assuming your music is stored in ``/srv/funkwhale/data/music``, add the following
block to your nginx configuration::
location /_protected/music {
internal;
alias /srv/funkwhale/data/music;
}
And the following to your .env file::
MUSIC_DIRECTORY_PATH=/srv/funkwhale/data/music
**Docker users:**
Assuming your music is stored in ``/srv/funkwhale/data/music``, add the following
block to your nginx configuration::
location /_protected/music {
internal;
alias /srv/funkwhale/data/music;
}
Assuming you have the following volume directive in your ``docker-compose.yml``
(it's the default): ``/srv/funkwhale/data/music:/music:ro``, then add
the following to your .env file::
# this is the path in the container
MUSIC_DIRECTORY_PATH=/music
# this is the path on the host
MUSIC_DIRECTORY_SERVE_PATH=/srv/funkwhale/data/music
0.9.1 (2018-04-17)
------------------
......
......@@ -390,6 +390,12 @@ REST_FRAMEWORK = {
ATOMIC_REQUESTS = False
USE_X_FORWARDED_HOST = True
USE_X_FORWARDED_PORT = True
# Wether we should use Apache, Nginx (or other) headers when serving audio files
# Default to Nginx
REVERSE_PROXY_TYPE = env('REVERSE_PROXY_TYPE', default='nginx')
assert REVERSE_PROXY_TYPE in ['apache2', 'nginx'], 'Unsupported REVERSE_PROXY_TYPE'
# Wether we should check user permission before serving audio files (meaning
# return an obfuscated url)
# This require a special configuration on the reverse proxy side
......@@ -441,3 +447,9 @@ EXTERNAL_REQUESTS_VERIFY_SSL = env.bool(
'EXTERNAL_REQUESTS_VERIFY_SSL',
default=True
)
MUSIC_DIRECTORY_PATH = env('MUSIC_DIRECTORY_PATH', default=None)
# on Docker setup, the music directory may not match the host path,
# and we need to know it for it to serve stuff properly
MUSIC_DIRECTORY_SERVE_PATH = env(
'MUSIC_DIRECTORY_SERVE_PATH', default=MUSIC_DIRECTORY_PATH)
# -*- coding: utf-8 -*-
__version__ = '0.9.1'
__version__ = '0.10'
__version_info__ = tuple([int(num) if num.isdigit() else num for num in __version__.replace('-', '.', 1).split('.')])
......@@ -708,23 +708,7 @@ class AudioSerializer(serializers.Serializer):
except (KeyError, TypeError):
raise serializers.ValidationError('Missing mediaType')
if not media_type.startswith('audio/'):
raise serializers.ValidationError('Invalid mediaType')
return url
def validate_url(self, v):
try:
url = v['href']
except (KeyError, TypeError):
raise serializers.ValidationError('Missing href')
try:
media_type = v['mediaType']
except (KeyError, TypeError):
raise serializers.ValidationError('Missing mediaType')
if not media_type.startswith('audio/'):
if not media_type or not media_type.startswith('audio/'):
raise serializers.ValidationError('Invalid mediaType')
return v
......
......@@ -43,6 +43,7 @@ class TrackFactory(factory.django.DjangoModelFactory):
artist = factory.SelfAttribute('album.artist')
position = 1
tags = ManyToManyFromList('tags')
class Meta:
model = 'music.Track'
......@@ -57,6 +58,9 @@ class TrackFileFactory(factory.django.DjangoModelFactory):
model = 'music.TrackFile'
class Params:
in_place = factory.Trait(
audio_file=None,
)
federation = factory.Trait(
audio_file=None,
library_track=factory.SubFactory(LibraryTrackFactory),
......@@ -105,6 +109,10 @@ class ImportJobFactory(factory.django.DjangoModelFactory):
status='finished',
track_file=factory.SubFactory(TrackFileFactory),
)
in_place = factory.Trait(
status='finished',
audio_file=None,
)
@registry.register(name='music.FileImportJob')
......
......@@ -2,6 +2,7 @@ from django.db.models import Count
from django_filters import rest_framework as filters
from funkwhale_api.common import fields
from . import models
......@@ -28,6 +29,39 @@ class ArtistFilter(ListenableMixin):
}
class ImportBatchFilter(filters.FilterSet):
q = fields.SearchFilter(search_fields=[
'submitted_by__username',
'source',
])
class Meta:
model = models.ImportBatch
fields = {
'status': ['exact'],
'source': ['exact'],
'submitted_by': ['exact'],
}
class ImportJobFilter(filters.FilterSet):
q = fields.SearchFilter(search_fields=[
'batch__submitted_by__username',
'source',
])
class Meta:
model = models.ImportJob
fields = {
'batch': ['exact'],
'batch__status': ['exact'],
'batch__source': ['exact'],
'batch__submitted_by': ['exact'],
'status': ['exact'],
'source': ['exact'],
}
class AlbumFilter(ListenableMixin):
listenable = filters.BooleanFilter(name='_', method='filter_listenable')
......
import mutagen
from django import forms
import arrow
import mutagen
NODEFAULT = object()
......@@ -50,6 +51,13 @@ def convert_track_number(v):
except (ValueError, AttributeError, IndexError):
pass
VALIDATION = {
'musicbrainz_artistid': forms.UUIDField(),
'musicbrainz_albumid': forms.UUIDField(),
'musicbrainz_recordingid': forms.UUIDField(),
}
CONF = {
'OggVorbis': {
'getter': lambda f, k: f[k][0],
......@@ -146,4 +154,7 @@ class Metadata(object):
converter = field_conf.get('to_application')
if converter:
v = converter(v)
field = VALIDATION.get(key)
if field:
v = field.to_python(v)
return v
# Generated by Django 2.0.3 on 2018-04-19 20:23
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('music', '0024_populate_uuid'),
]
operations = [
migrations.AlterField(
model_name='trackfile',
name='source',
field=models.URLField(blank=True, max_length=500, null=True),
),
]
......@@ -412,7 +412,7 @@ class TrackFile(models.Model):
track = models.ForeignKey(
Track, related_name='files', on_delete=models.CASCADE)
audio_file = models.FileField(upload_to='tracks/%Y/%m/%d', max_length=255)
source = models.URLField(null=True, blank=True)
source = models.URLField(null=True, blank=True, max_length=500)
creation_date = models.DateTimeField(default=timezone.now)
modification_date = models.DateTimeField(auto_now=True)
duration = models.IntegerField(null=True, blank=True)
......@@ -463,6 +463,26 @@ class TrackFile(models.Model):
self.mimetype = utils.guess_mimetype(self.audio_file)
return super().save(**kwargs)
@property
def serve_from_source_path(self):
if not self.source or not self.source.startswith('file://'):
raise ValueError('Cannot serve this file from source')
serve_path = settings.MUSIC_DIRECTORY_SERVE_PATH
prefix = settings.MUSIC_DIRECTORY_PATH
if not serve_path or not prefix:
raise ValueError(
'You need to specify MUSIC_DIRECTORY_SERVE_PATH and '
'MUSIC_DIRECTORY_PATH to serve in-place imported files'
)
file_path = self.source.replace('file://', '', 1)
parts = os.path.split(file_path.replace(prefix, '', 1))
if parts[0] == '/':
parts = parts[1:]
return os.path.join(
serve_path,
*parts
)
IMPORT_STATUS_CHOICES = (
('pending', 'Pending'),
......@@ -507,6 +527,8 @@ class ImportBatch(models.Model):
def update_status(self):
old_status = self.status
self.status = utils.compute_status(self.jobs.all())
if self.status == old_status:
return
self.save(update_fields=['status'])
if self.status != old_status and self.status == 'finished':
from . import tasks
......
......@@ -6,6 +6,7 @@ from funkwhale_api.activity import serializers as activity_serializers
from funkwhale_api.federation import utils as federation_utils
from funkwhale_api.federation.models import LibraryTrack
from funkwhale_api.federation.serializers import AP_CONTEXT
from funkwhale_api.users.serializers import UserBasicSerializer
from . import models
......@@ -90,6 +91,7 @@ class TrackSerializerNested(LyricsMixin):
files = TrackFileSerializer(many=True, read_only=True)
album = SimpleAlbumSerializer(read_only=True)
tags = TagSerializer(many=True, read_only=True)
class Meta:
model = models.Track
fields = ('id', 'mbid', 'title', 'artist', 'files', 'album', 'tags', 'lyrics')
......@@ -108,6 +110,7 @@ class AlbumSerializerNested(serializers.ModelSerializer):
class ArtistSerializerNested(serializers.ModelSerializer):
albums = AlbumSerializerNested(many=True, read_only=True)
tags = TagSerializer(many=True, read_only=True)
class Meta:
model = models.Artist
fields = ('id', 'mbid', 'name', 'albums', 'tags')
......@@ -121,18 +124,43 @@ class LyricsSerializer(serializers.ModelSerializer):
class ImportJobSerializer(serializers.ModelSerializer):
track_file = TrackFileSerializer(read_only=True)
class Meta:
model = models.ImportJob
fields = ('id', 'mbid', 'batch', 'source', 'status', 'track_file', 'audio_file')
fields = (
'id',
'mbid',
'batch',
'source',
'status',
'track_file',
'audio_file')
read_only_fields = ('status', 'track_file')
class ImportBatchSerializer(serializers.ModelSerializer):
jobs = ImportJobSerializer(many=True, read_only=True)
submitted_by = UserBasicSerializer(read_only=True)
class Meta:
model = models.ImportBatch
fields = ('id', 'jobs', 'status', 'creation_date', 'import_request')
read_only_fields = ('creation_date',)
fields = (
'id',
'submitted_by',
'source',
'status',
'creation_date',
'import_request')
read_only_fields = (
'creation_date', 'submitted_by', 'source')
def to_representation(self, instance):
repr = super().to_representation(instance)
try:
repr['job_count'] = instance.job_count
except AttributeError:
# Queryset was not annotated
pass
return repr
class TrackActivitySerializer(activity_serializers.ModelSerializer):
......
......@@ -71,7 +71,7 @@ def import_track_from_remote(library_track):
library_track.title, artist=artist, album=album)
def _do_import(import_job, replace, use_acoustid=True):
def _do_import(import_job, replace=False, use_acoustid=True):
from_file = bool(import_job.audio_file)
mbid = import_job.mbid
acoustid_track_id = None
......@@ -93,6 +93,9 @@ def _do_import(import_job, replace, use_acoustid=True):
track = import_track_data_from_path(import_job.audio_file.path)
elif import_job.library_track:
track = import_track_from_remote(import_job.library_track)
elif import_job.source.startswith('file://'):
track = import_track_data_from_path(
import_job.source.replace('file://', '', 1))
else:
raise ValueError(
'Not enough data to process import, '
......@@ -123,7 +126,8 @@ def _do_import(import_job, replace, use_acoustid=True):
else:
# no downloading, we hotlink
pass
else:
elif not import_job.audio_file and not import_job.source.startswith('file://'):
# not an implace import, and we have a source, so let's download it
track_file.download_file()
track_file.save()
import_job.status = 'finished'
......@@ -133,7 +137,7 @@ def _do_import(import_job, replace, use_acoustid=True):
import_job.audio_file.delete()
import_job.save()
return track.pk
return track_file
@celery.app.task(name='ImportJob.run', bind=True)
......@@ -147,7 +151,8 @@ def import_job_run(self, import_job, replace=False, use_acoustid=True):
import_job.save(update_fields=['status'])
try:
return _do_import(import_job, replace, use_acoustid=use_acoustid)
tf = _do_import(import_job, replace, use_acoustid=use_acoustid)
return tf.pk if tf else None
except Exception as exc:
if not settings.DEBUG:
try:
......
......@@ -53,10 +53,11 @@ def guess_mimetype(f):
def compute_status(jobs):
errored = any([job.status == 'errored' for job in jobs])
statuses = jobs.order_by().values_list('status', flat=True).distinct()
errored = any([status == 'errored' for status in statuses])
if errored:
return 'errored'
pending = any([job.status == 'pending' for job in jobs])
pending = any([status == 'pending' for status in statuses])
if pending:
return 'pending'
return 'finished'
......
......@@ -11,6 +11,7 @@ from django.core.exceptions import ObjectDoesNotExist
from django.conf import settings
from django.db import models, transaction
from django.db.models.functions import Length
from django.db.models import Count
from django.http import StreamingHttpResponse
from django.urls import reverse
from django.utils.decorators import method_decorator
......@@ -23,13 +24,14 @@ from rest_framework import permissions
from musicbrainzngs import ResponseError
from funkwhale_api.common import utils as funkwhale_utils
from funkwhale_api.federation import actors
from funkwhale_api.requests.models import ImportRequest
from funkwhale_api.musicbrainz import api
from funkwhale_api.common.permissions import (
ConditionalAuthentication, HasModelPermission)
from taggit.models import Tag
from funkwhale_api.federation import actors
from funkwhale_api.federation.authentication import SignatureAuthentication
from funkwhale_api.federation.models import LibraryTrack
from funkwhale_api.musicbrainz import api
from funkwhale_api.requests.models import ImportRequest
from . import filters
from . import forms
......@@ -98,14 +100,14 @@ class ImportBatchViewSet(
mixins.RetrieveModelMixin,
viewsets.GenericViewSet):
queryset = (
models.ImportBatch.objects.all()
.prefetch_related('jobs__track_file')
.order_by('-creation_date'))
models.ImportBatch.objects
.select_related()
.order_by('-creation_date')
.annotate(job_count=Count('jobs'))
)
serializer_class = serializers.ImportBatchSerializer
permission_classes = (permissions.DjangoModelPermissions, )
def get_queryset(self):
return super().get_queryset().filter(submitted_by=self.request.user)
filter_class = filters.ImportBatchFilter
def perform_create(self, serializer):
serializer.save(submitted_by=self.request.user)
......@@ -118,13 +120,30 @@ class ImportJobPermission(HasModelPermission):
class ImportJobViewSet(
mixins.CreateModelMixin,
mixins.ListModelMixin,
viewsets.GenericViewSet):
queryset = (models.ImportJob.objects.all())
queryset = (models.ImportJob.objects.all().select_related())
serializer_class = serializers.ImportJobSerializer
permission_classes = (ImportJobPermission, )
filter_class = filters.ImportJobFilter
def get_queryset(self):
return super().get_queryset().filter(batch__submitted_by=self.request.user)
@list_route(methods=['get'])
def stats(self, request, *args, **kwargs):
qs = models.ImportJob.objects.all()
filterset = filters.ImportJobFilter(request.GET, queryset=qs)
qs = filterset.qs
qs = qs.values('status').order_by('status')
qs = qs.annotate(status_count=Count('status'))
data = {}
for row in qs:
data[row['status']] = row['status_count']
for s, _ in models.IMPORT_STATUS_CHOICES:
data.setdefault(s, 0)
data['count'] = sum([v for v in data.values()])
return Response(data)
def perform_create(self, serializer):
source = 'file://' + serializer.validated_data['audio_file'].name
......@@ -135,7 +154,8 @@ class ImportJobViewSet(
)
class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet):
class TrackViewSet(
TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet):
"""
A simple ViewSet for viewing and editing accounts.
"""
......@@ -185,6 +205,25 @@ class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet):
return Response(serializer.data)
def get_file_path(audio_file):
t = settings.REVERSE_PROXY_TYPE
if t == 'nginx':
# we have to use the internal locations
try:
path = audio_file.url
except AttributeError:
# a path was given
path = '/music' + audio_file
return settings.PROTECT_FILES_PATH + path
if t == 'apache2':
try:
path = audio_file.path
except AttributeError:
# a path was given
path = audio_file
return path
class TrackFileViewSet(viewsets.ReadOnlyModelViewSet):
queryset = (models.TrackFile.objects.all().order_by('-id'))
serializer_class = serializers.TrackFileSerializer
......@@ -195,12 +234,13 @@ class TrackFileViewSet(viewsets.ReadOnlyModelViewSet):
@detail_route(methods=['get'])
def serve(self, request, *args, **kwargs):
queryset = models.TrackFile.objects.select_related(
'library_track',
'track__album__artist',
'track__artist',
)
try:
f = models.TrackFile.objects.select_related(
'library_track',
'track__album__artist',
'track__artist',
).get(pk=kwargs['pk'])
f = queryset.get(pk=kwargs['pk'])
except models.TrackFile.DoesNotExist:
return Response(status=404)
......@@ -213,14 +253,29 @@ class TrackFileViewSet(viewsets.ReadOnlyModelViewSet):
if library_track and not audio_file:
if not library_track.audio_file:
# we need to populate from cache
library_track.download_audio()
with transaction.atomic():
# why the transaction/select_for_update?
# this is because browsers may send multiple requests
# in a short time range, for partial content,
# thus resulting in multiple downloads from the remote
qs = LibraryTrack.objects.select_for_update()
library_track = qs.get(pk=library_track.pk)
library_track.download_audio()
audio_file = library_track.audio_file
file_path = get_file_path(audio_file)
mt = library_track.audio_mimetype
elif audio_file:
file_path = get_file_path(audio_file)
elif f.source and f.source.startswith('file://'):
file_path = get_file_path(f.serve_from_source_path)
response = Response()
filename = f.filename
response['X-Accel-Redirect'] = "{}{}".format(
settings.PROTECT_FILES_PATH,
audio_file.url)
mapping = {
'nginx': 'X-Accel-Redirect',
'apache2': 'X-Sendfile',
}
file_header = mapping[settings.REVERSE_PROXY_TYPE]
response[file_header] = file_path
filename = "filename*=UTF-8''{}".format(
urllib.parse.quote(filename))
response["Content-Disposition"] = "attachment; {}".format(filename)
......
import glob
import os
from django.conf import settings
from django.core.files import File
from django.core.management.base import BaseCommand, CommandError
from django.db import transaction
from funkwhale_api.common import utils
from funkwhale_api.music import models
from funkwhale_api.music import tasks
from funkwhale_api.users.models import User
......@@ -39,7 +39,20 @@ class Command(BaseCommand):
action='store_true',
dest='exit_on_failure',
default=False,
help='use this flag to disable error catching',
help='Use this flag to disable error catching',
)
parser.add_argument(
'--in-place', '-i',
action='store_true',