Merge branch 'release/0.3.1'

CHANGELOG

 Changelog
 =========
 
+0.3.2 (Unreleased)
+------------------
+
+
+0.3.1
+------------------
+
+- Revamped all import logic, everything is more tested and consistend
+- Can now use Acoustid in file imports to automatically grab metadata from musicbrainz
+- Brand new file import wizard
+
 
-0.2.7 (Unreleased)
+0.2.7
 ------------------
 
 - Shortcuts: can now use the ``f`` shortcut to toggle the currently playing track

api/Dockerfile

@@ -6,8 +6,8 @@ ENV PYTHONUNBUFFERED 1
 
 COPY ./requirements.apt /requirements.apt
 RUN apt-get update -qq && grep "^[^#;]" requirements.apt | xargs apt-get install -y
-
-
+RUN curl -L https://github.com/acoustid/chromaprint/releases/download/v1.4.2/chromaprint-fpcalc-1.4.2-linux-x86_64.tar.gz | tar -xz -C /usr/local/bin --strip 1
+RUN fcalc yolofkjdssdhf
 COPY ./requirements/base.txt /requirements/base.txt
 RUN pip install -r /requirements/base.txt
 COPY ./requirements/production.txt /requirements/production.txt

api/config/api_urls.py

@@ -15,6 +15,7 @@ router.register(r'trackfiles', views.TrackFileViewSet, 'trackfiles')
 router.register(r'artists', views.ArtistViewSet, 'artists')
 router.register(r'albums', views.AlbumViewSet, 'albums')
 router.register(r'import-batches', views.ImportBatchViewSet, 'import-batches')
+router.register(r'import-jobs', views.ImportJobViewSet, 'import-jobs')
 router.register(r'submit', views.SubmitViewSet, 'submit')
 router.register(r'playlists', playlists_views.PlaylistViewSet, 'playlists')
 router.register(

api/config/settings/common.py

@@ -47,7 +47,6 @@ THIRD_PARTY_APPS = (
     'corsheaders',
     'rest_framework',
     'rest_framework.authtoken',
-    'djcelery',
     'taggit',
     'cachalot',
     'rest_auth',
@@ -68,6 +67,7 @@ LOCAL_APPS = (
     'funkwhale_api.playlists',
     'funkwhale_api.providers.audiofile',
     'funkwhale_api.providers.youtube',
+    'funkwhale_api.providers.acoustid',
 )
 
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
@@ -266,14 +266,14 @@ CACHES["default"]["OPTIONS"] = {
 
 ########## CELERY
 INSTALLED_APPS += ('funkwhale_api.taskapp.celery.CeleryConfig',)
-BROKER_URL = env(
+CELERY_BROKER_URL = env(
     "CELERY_BROKER_URL", default=env('CACHE_URL', default=CACHE_DEFAULT))
 ########## END CELERY
 # Location of root django.contrib.admin URL, use {% url 'admin:index' %}
 ADMIN_URL = r'^admin/'
 # Your common stuff: Below this line define 3rd party library settings
-CELERY_DEFAULT_RATE_LIMIT = 1
-CELERYD_TASK_TIME_LIMIT = 300
+CELERY_TASK_DEFAULT_RATE_LIMIT = 1
+CELERY_TASK_TIME_LIMIT = 300
 import datetime
 JWT_AUTH = {
     'JWT_ALLOW_REFRESH': True,

api/config/settings/local.py

@@ -54,7 +54,7 @@ TEST_RUNNER = 'django.test.runner.DiscoverRunner'
 
 ########## CELERY
 # In development, all tasks will be executed locally by blocking until the task returns
-CELERY_ALWAYS_EAGER = False
+CELERY_TASK_ALWAYS_EAGER = False
 ########## END CELERY
 
 # Your local stuff: Below this line define 3rd party library settings

api/config/settings/test.py

@@ -23,7 +23,7 @@ CACHES = {
     }
 }
 
-BROKER_URL = 'memory://'
+CELERY_BROKER_URL = 'memory://'
 
 # TESTING
 # ------------------------------------------------------------------------------
@@ -31,7 +31,7 @@ TEST_RUNNER = 'django.test.runner.DiscoverRunner'
 
 ########## CELERY
 # In development, all tasks will be executed locally by blocking until the task returns
-CELERY_ALWAYS_EAGER = True
+CELERY_TASK_ALWAYS_EAGER = True
 ########## END CELERY
 
 # Your local stuff: Below this line define 3rd party library settings

api/docker/Dockerfile.test

@@ -7,6 +7,7 @@ ENV PYTHONDONTWRITEBYTECODE  1
 COPY ./requirements.apt /requirements.apt
 COPY ./install_os_dependencies.sh /install_os_dependencies.sh
 RUN bash install_os_dependencies.sh install
+RUN curl -L https://github.com/acoustid/chromaprint/releases/download/v1.4.2/chromaprint-fpcalc-1.4.2-linux-x86_64.tar.gz | tar -xz -C /usr/local/bin --strip 1
 
 RUN mkdir /requirements
 COPY ./requirements/base.txt /requirements/base.txt

api/funkwhale_api/__init__.py

 # -*- coding: utf-8 -*-
-__version__ = '0.3'
+__version__ = '0.3.1'
 __version_info__ = tuple([int(num) if num.isdigit() else num for num in __version__.replace('-', '.', 1).split('.')])

api/funkwhale_api/common/permissions.py

 from django.conf import settings
 
-from rest_framework.permissions import BasePermission
+from rest_framework.permissions import BasePermission, DjangoModelPermissions
 
 
 class ConditionalAuthentication(BasePermission):
@@ -9,3 +9,14 @@ class ConditionalAuthentication(BasePermission):
         if settings.API_AUTHENTICATION_REQUIRED:
             return request.user and request.user.is_authenticated
         return True
+
+
+class HasModelPermission(DjangoModelPermissions):
+    """
+    Same as DjangoModelPermissions, but we pin the model:
+
+        class MyModelPermission(HasModelPermission):
+            model = User
+    """
+    def get_required_permissions(self, method, model_cls):
+        return super().get_required_permissions(method, self.model)

api/funkwhale_api/music/factories.py

@@ -72,6 +72,14 @@ class ImportJobFactory(factory.django.DjangoModelFactory):
         model = 'music.ImportJob'
 
 
+@registry.register(name='music.FileImportJob')
+class FileImportJobFactory(ImportJobFactory):
+    source = 'file://'
+    mbid = None
+    audio_file = factory.django.FileField(
+        from_path=os.path.join(SAMPLES_PATH, 'test.ogg'))
+
+
 @registry.register
 class WorkFactory(factory.django.DjangoModelFactory):
     mbid = factory.Faker('uuid4')

api/funkwhale_api/music/migrations/0016_trackfile_acoustid_track_id.py

+# Generated by Django 2.0 on 2017-12-26 16:39
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0015_bind_track_file_to_import_job'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='trackfile',
+            name='acoustid_track_id',
+            field=models.UUIDField(blank=True, null=True),
+        ),
+    ]

api/funkwhale_api/music/migrations/0017_auto_20171227_1728.py

+# Generated by Django 2.0 on 2017-12-27 17:28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0016_trackfile_acoustid_track_id'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='importbatch',
+            name='source',
+            field=models.CharField(choices=[('api', 'api'), ('shell', 'shell')], default='api', max_length=30),
+        ),
+        migrations.AddField(
+            model_name='importjob',
+            name='audio_file',
+            field=models.FileField(blank=True, max_length=255, null=True, upload_to='imports/%Y/%m/%d'),
+        ),
+        migrations.AlterField(
+            model_name='importjob',
+            name='mbid',
+            field=models.UUIDField(blank=True, editable=False, null=True),
+        ),
+    ]

api/funkwhale_api/music/models.py

@@ -15,11 +15,9 @@ from django.utils import timezone
 from taggit.managers import TaggableManager
 from versatileimagefield.fields import VersatileImageField
 
-from funkwhale_api.taskapp import celery
 from funkwhale_api import downloader
 from funkwhale_api import musicbrainz
 from . import importers
-from . import lyrics as lyrics_utils
 
 
 class APIModelMixin(models.Model):
@@ -255,14 +253,6 @@ class Lyrics(models.Model):
     url = models.URLField(unique=True)
     content = models.TextField(null=True, blank=True)
 
-    @celery.app.task(name='Lyrics.fetch_content', filter=celery.task_method)
-    def fetch_content(self):
-        html = lyrics_utils._get_html(self.url)
-        content = lyrics_utils.extract_content(html)
-        cleaned_content = lyrics_utils.clean_content(content)
-        self.content = cleaned_content
-        self.save()
-
     @property
     def content_rendered(self):
         return markdown.markdown(
@@ -362,6 +352,7 @@ class TrackFile(models.Model):
     audio_file = models.FileField(upload_to='tracks/%Y/%m/%d', max_length=255)
     source = models.URLField(null=True, blank=True)
     duration = models.IntegerField(null=True, blank=True)
+    acoustid_track_id = models.UUIDField(null=True, blank=True)
 
     def download_file(self):
         # import the track file, since there is not any
@@ -393,9 +384,17 @@ class TrackFile(models.Model):
 
 
 class ImportBatch(models.Model):
+    IMPORT_BATCH_SOURCES = [
+        ('api', 'api'),
+        ('shell', 'shell')
+    ]
+    source = models.CharField(
+        max_length=30, default='api', choices=IMPORT_BATCH_SOURCES)
     creation_date = models.DateTimeField(default=timezone.now)
     submitted_by = models.ForeignKey(
-        'users.User', related_name='imports', on_delete=models.CASCADE)
+        'users.User',
+        related_name='imports',
+        on_delete=models.CASCADE)
 
     class Meta:
         ordering = ['-creation_date']
@@ -406,8 +405,11 @@ class ImportBatch(models.Model):
     @property
     def status(self):
         pending = any([job.status == 'pending' for job in self.jobs.all()])
+        errored = any([job.status == 'errored' for job in self.jobs.all()])
         if pending:
             return 'pending'
+        if errored:
+            return 'errored'
         return 'finished'
 
 class ImportJob(models.Model):
@@ -419,36 +421,17 @@ class ImportJob(models.Model):
         null=True,
         blank=True,
         on_delete=models.CASCADE)
-    source = models.URLField()
-    mbid = models.UUIDField(editable=False)
+    source = models.CharField(max_length=500)
+    mbid = models.UUIDField(editable=False, null=True, blank=True)
     STATUS_CHOICES = (
         ('pending', 'Pending'),
-        ('finished', 'finished'),
+        ('finished', 'Finished'),
+        ('errored', 'Errored'),
+        ('skipped', 'Skipped'),
     )
     status = models.CharField(choices=STATUS_CHOICES, default='pending', max_length=30)
+    audio_file = models.FileField(
+        upload_to='imports/%Y/%m/%d', max_length=255, null=True, blank=True)
 
     class Meta:
         ordering = ('id', )
-    @celery.app.task(name='ImportJob.run', filter=celery.task_method)
-    def run(self, replace=False):
-        try:
-            track, created = Track.get_or_create_from_api(mbid=self.mbid)
-            track_file = None
-            if replace:
-                track_file = track.files.first()
-            elif track.files.count() > 0:
-                return
-
-            track_file = track_file or TrackFile(
-                track=track, source=self.source)
-            track_file.download_file()
-            track_file.save()
-            self.status = 'finished'
-            self.track_file = track_file
-            self.save()
-            return track.pk
-
-        except Exception as exc:
-            if not settings.DEBUG:
-                raise ImportJob.run.retry(args=[self], exc=exc, countdown=30, max_retries=3)
-            raise

api/funkwhale_api/music/serializers.py

@@ -113,7 +113,8 @@ class ImportJobSerializer(serializers.ModelSerializer):
     track_file = TrackFileSerializer(read_only=True)
     class Meta:
         model = models.ImportJob
-        fields = ('id', 'mbid', 'source', 'status', 'track_file')
+        fields = ('id', 'mbid', 'batch', 'source', 'status', 'track_file', 'audio_file')
+        read_only_fields = ('status', 'track_file')
 
 
 class ImportBatchSerializer(serializers.ModelSerializer):
@@ -121,3 +122,4 @@ class ImportBatchSerializer(serializers.ModelSerializer):
     class Meta:
         model = models.ImportBatch
         fields = ('id', 'jobs', 'status', 'creation_date')
+        read_only_fields = ('creation_date',)

api/funkwhale_api/music/tasks.py

+from django.core.files.base import ContentFile
+
+from funkwhale_api.taskapp import celery
+from funkwhale_api.providers.acoustid import get_acoustid_client
+from funkwhale_api.providers.audiofile.tasks import import_track_data_from_path
+
+from django.conf import settings
+from . import models
+from . import lyrics as lyrics_utils
+
+
+@celery.app.task(name='acoustid.set_on_track_file')
+@celery.require_instance(models.TrackFile, 'track_file')
+def set_acoustid_on_track_file(track_file):
+    client = get_acoustid_client()
+    result = client.get_best_match(track_file.audio_file.path)
+
+    def update(id):
+        track_file.acoustid_track_id = id
+        track_file.save(update_fields=['acoustid_track_id'])
+        return id
+    if result:
+        return update(result['id'])
+
+
+def _do_import(import_job, replace):
+    from_file = bool(import_job.audio_file)
+    mbid = import_job.mbid
+    acoustid_track_id = None
+    duration = None
+    track = None
+    if not mbid and from_file:
+        # we try to deduce mbid from acoustid
+        client = get_acoustid_client()
+        match = client.get_best_match(import_job.audio_file.path)
+        if not match:
+            raise ValueError('Cannot get match')
+        duration = match['recordings'][0]['duration']
+        mbid = match['recordings'][0]['id']
+        acoustid_track_id = match['id']
+    if mbid:
+        track, _ = models.Track.get_or_create_from_api(mbid=mbid)
+    else:
+        track = import_track_data_from_path(import_job.audio_file.path)
+
+    track_file = None
+    if replace:
+        track_file = track.files.first()
+    elif track.files.count() > 0:
+        if import_job.audio_file:
+            import_job.audio_file.delete()
+        import_job.status = 'skipped'
+        import_job.save()
+        return
+
+    track_file = track_file or models.TrackFile(
+        track=track, source=import_job.source)
+    track_file.acoustid_track_id = acoustid_track_id
+    if from_file:
+        track_file.audio_file = ContentFile(import_job.audio_file.read())
+        track_file.audio_file.name = import_job.audio_file.name
+        track_file.duration = duration
+    else:
+        track_file.download_file()
+    track_file.save()
+    import_job.status = 'finished'
+    import_job.track_file = track_file
+    if import_job.audio_file:
+        # it's imported on the track, we don't need it anymore
+        import_job.audio_file.delete()
+    import_job.save()
+    return track.pk
+
+
+@celery.app.task(name='ImportJob.run', bind=True)
+@celery.require_instance(models.ImportJob, 'import_job')
+def import_job_run(self, import_job, replace=False):
+    def mark_errored():
+        import_job.status = 'errored'
+        import_job.save()
+
+    try:
+        return _do_import(import_job, replace)
+    except Exception as exc:
+        if not settings.DEBUG:
+            try:
+                self.retry(exc=exc, countdown=30, max_retries=3)
+            except:
+                mark_errored()
+                raise
+        mark_errored()
+        raise
+
+
+@celery.app.task(name='Lyrics.fetch_content')
+@celery.require_instance(models.Lyrics, 'lyrics')
+def fetch_content(lyrics):
+    html = lyrics_utils._get_html(lyrics.url)
+    content = lyrics_utils.extract_content(html)
+    cleaned_content = lyrics_utils.clean_content(content)
+    lyrics.content = cleaned_content
+    lyrics.save(update_fields=['content'])

api/funkwhale_api/music/views.py

@@ -6,7 +6,7 @@ from django.urls import reverse
 from django.db import models, transaction
 from django.db.models.functions import Length
 from django.conf import settings
-from rest_framework import viewsets, views
+from rest_framework import viewsets, views, mixins
 from rest_framework.decorators import detail_route, list_route
 from rest_framework.response import Response
 from rest_framework import permissions
@@ -15,13 +15,15 @@ from django.contrib.auth.decorators import login_required
 from django.utils.decorators import method_decorator
 
 from funkwhale_api.musicbrainz import api
-from funkwhale_api.common.permissions import ConditionalAuthentication
+from funkwhale_api.common.permissions import (
+    ConditionalAuthentication, HasModelPermission)
 from taggit.models import Tag
 
 from . import models
 from . import serializers
 from . import importers
 from . import filters
+from . import tasks
 from . import utils
 
 
@@ -70,16 +72,45 @@ class AlbumViewSet(SearchMixin, viewsets.ReadOnlyModelViewSet):
     ordering_fields = ('creation_date',)
 
 
-class ImportBatchViewSet(viewsets.ReadOnlyModelViewSet):
+class ImportBatchViewSet(
+        mixins.CreateModelMixin,
+        mixins.ListModelMixin,
+        mixins.RetrieveModelMixin,
+        viewsets.GenericViewSet):
     queryset = (
         models.ImportBatch.objects.all()
                           .prefetch_related('jobs__track_file')
                           .order_by('-creation_date'))
     serializer_class = serializers.ImportBatchSerializer
+    permission_classes = (permissions.DjangoModelPermissions, )
 
     def get_queryset(self):
         return super().get_queryset().filter(submitted_by=self.request.user)
 
+    def perform_create(self, serializer):
+        serializer.save(submitted_by=self.request.user)
+
+
+class ImportJobPermission(HasModelPermission):
+    # not a typo, perms on import job is proxied to import batch
+    model = models.ImportBatch
+
+
+class ImportJobViewSet(
+        mixins.CreateModelMixin,
+        viewsets.GenericViewSet):
+    queryset = (models.ImportJob.objects.all())
+    serializer_class = serializers.ImportJobSerializer
+    permission_classes = (ImportJobPermission, )
+
+    def get_queryset(self):
+        return super().get_queryset().filter(batch__submitted_by=self.request.user)
+
+    def perform_create(self, serializer):
+        source = 'file://' + serializer.validated_data['audio_file'].name
+        serializer.save(source=source)
+        tasks.import_job_run.delay(import_job_id=serializer.instance.pk)
+
 
 class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet):
     """
@@ -129,7 +160,8 @@ class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet):
         lyrics = work.fetch_lyrics()
         try:
             if not lyrics.content:
-                lyrics.fetch_content()
+                tasks.fetch_content(lyrics_id=lyrics.pk)
+                lyrics.refresh_from_db()
         except AttributeError:
             return Response({'error': 'unavailable lyrics'}, status=404)
         serializer = serializers.LyricsSerializer(lyrics)
@@ -244,7 +276,7 @@ class SubmitViewSet(viewsets.ViewSet):
             pass
         batch = models.ImportBatch.objects.create(submitted_by=request.user)
         job = models.ImportJob.objects.create(mbid=request.POST['mbid'], batch=batch, source=request.POST['import_url'])
-        job.run.delay()
+        tasks.import_job_run.delay(import_job_id=job.pk)
         serializer = serializers.ImportBatchSerializer(batch)
         return Response(serializer.data)
 
@@ -272,7 +304,7 @@ class SubmitViewSet(viewsets.ViewSet):
                 models.TrackFile.objects.get(track__mbid=row['mbid'])
             except models.TrackFile.DoesNotExist:
                 job = models.ImportJob.objects.create(mbid=row['mbid'], batch=batch, source=row['source'])
-                job.run.delay()
+                tasks.import_job_run.delay(import_job_id=job.pk)
         serializer = serializers.ImportBatchSerializer(batch)
         return serializer.data, batch
 

api/funkwhale_api/providers/acoustid/__init__.py

+import acoustid
+
+from dynamic_preferences.registries import global_preferences_registry
+
+
+class Client(object):
+    def __init__(self, api_key):
+        self.api_key = api_key
+
+    def match(self, file_path):
+        return acoustid.match(self.api_key, file_path, parse=False)
+
+    def get_best_match(self, file_path):
+        results = self.match(file_path=file_path)
+        MIN_SCORE_FOR_MATCH = 0.8
+        try:
+            rows = results['results']
+        except KeyError:
+            return
+        for row in rows:
+            if row['score'] >= MIN_SCORE_FOR_MATCH:
+                return row
+
+
+def get_acoustid_client():
+    manager = global_preferences_registry.manager()
+    return Client(api_key=manager['providers_acoustid__api_key'])

api/funkwhale_api/providers/acoustid/dynamic_preferences_registry.py

+from dynamic_preferences.types import StringPreference, Section
+from dynamic_preferences.registries import global_preferences_registry
+
+acoustid = Section('providers_acoustid')
+
+
+@global_preferences_registry.register
+class APIKey(StringPreference):
+    section = acoustid
+    name = 'api_key'
+    default = ''
+    verbose_name = 'Acoustid API key'
+    help_text = 'The API key used to query AcoustID. Get one at https://acoustid.org/new-application.'

api/funkwhale_api/providers/audiofile/management/commands/import_files.py

 import glob
+import os
+
+from django.core.files import File
 from django.core.management.base import BaseCommand, CommandError
-from funkwhale_api.providers.audiofile import tasks
+from funkwhale_api.music import tasks
+from funkwhale_api.users.models import User
 
 
 class Command(BaseCommand):
@@ -15,6 +19,11 @@ class Command(BaseCommand):
             default=False,
             help='Will match the pattern recursively (including subdirectories)',
         )
+        parser.add_argument(
+            '--username',
+            dest='username',
+            help='The username of the user you want to be bound to the import',
+        )
         parser.add_argument(
             '--async',
             action='store_true',
@@ -46,6 +55,20 @@ class Command(BaseCommand):
         if not matching: