Merge branch 'release/0.2.4'

.gitlab-ci.yml

@@ -18,6 +18,8 @@ test_api:
     - pip install -r requirements/test.txt
   script:
     - pytest
+  variables:
+    DATABASE_URL: "sqlite://"
 
   tags:
     - docker

CHANGELOG

+Changelog
+=========
+
+
+0.2.5 (unreleased)
+------------------
+
+
+0.2.4 (2017-12-14)
+------------------
+
+Features:
+
+- Models: now store relese group mbid on Album model (#7)
+- Models: now bind import job to track files (#44)
+
+Bugfixes:
+
+- Library: fixen broken "play all albums" button on artist cards in Artist browsing view (#43)

api/compose/django/entrypoint.sh

@@ -4,7 +4,7 @@ set -e
 # Since docker-compose relies heavily on environment variables itself for configuration, we'd have to define multiple
 # environment variables just to support cookiecutter out of the box. That makes no sense, so this little entrypoint
 # does all this for us.
-export REDIS_URL=redis://redis:6379/0
+export CACHE_URL=redis://redis:6379/0
 
 # the official postgres image uses 'postgres' as default user if not set explictly.
 if [ -z "$POSTGRES_ENV_POSTGRES_USER" ]; then
@@ -13,7 +13,7 @@ fi
 
 export DATABASE_URL=postgres://$POSTGRES_ENV_POSTGRES_USER:$POSTGRES_ENV_POSTGRES_PASSWORD@postgres:5432/$POSTGRES_ENV_POSTGRES_USER
 
-export CELERY_BROKER_URL=$REDIS_URL
+export CELERY_BROKER_URL=$CACHE_URL
 
 # we copy the frontend files, if any so we can serve them from the outside
 if [ -d "frontend" ]; then

api/config/settings/common.py

@@ -124,7 +124,7 @@ MANAGERS = ADMINS
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#databases
 DATABASES = {
     # Raises ImproperlyConfigured exception if DATABASE_URL not in os.environ
-    'default': env.db("DATABASE_URL", default="postgresql://postgres@postgres/postgres"),
+    'default': env.db("DATABASE_URL"),
 }
 DATABASES['default']['ATOMIC_REQUESTS'] = True
 #
@@ -199,7 +199,7 @@ CRISPY_TEMPLATE_PACK = 'bootstrap3'
 # STATIC FILE CONFIGURATION
 # ------------------------------------------------------------------------------
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#static-root
-STATIC_ROOT = str(ROOT_DIR('staticfiles'))
+STATIC_ROOT = env("STATIC_ROOT", default=str(ROOT_DIR('staticfiles')))
 
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#static-url
 STATIC_URL = env("STATIC_URL", default='/staticfiles/')
@@ -218,12 +218,10 @@ STATICFILES_FINDERS = (
 # MEDIA CONFIGURATION
 # ------------------------------------------------------------------------------
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#media-root
-MEDIA_ROOT = str(APPS_DIR('media'))
-
-
+MEDIA_ROOT = env("MEDIA_ROOT", default=str(APPS_DIR('media')))
 
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#media-url
-MEDIA_URL = '/media/'
+MEDIA_URL = env("MEDIA_URL", default='/media/')
 
 # URL Configuration
 # ------------------------------------------------------------------------------
@@ -253,26 +251,24 @@ LOGIN_URL = 'account_login'
 # SLUGLIFIER
 AUTOSLUG_SLUGIFY_FUNCTION = 'slugify.slugify'
 
-########## CELERY
-INSTALLED_APPS += ('funkwhale_api.taskapp.celery.CeleryConfig',)
-# if you are not using the django database broker (e.g. rabbitmq, redis, memcached), you can remove the next line.
-INSTALLED_APPS += ('kombu.transport.django',)
-BROKER_URL = env("CELERY_BROKER_URL", default='django://')
-########## END CELERY
-
-
+CACHE_DEFAULT = "redis://127.0.0.1:6379/0"
 CACHES = {
-    "default": {
-        "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": "{0}/{1}".format(env.cache_url('REDIS_URL', default="redis://127.0.0.1:6379"), 0),
-        "OPTIONS": {
-            "CLIENT_CLASS": "django_redis.client.DefaultClient",
-            "IGNORE_EXCEPTIONS": True,  # mimics memcache behavior.
-                                        # http://niwinz.github.io/django-redis/latest/#_memcached_exceptions_behavior
-        }
-    }
+    "default": env.cache_url('CACHE_URL', default=CACHE_DEFAULT)
 }
 
+CACHES["default"]["BACKEND"] = "django_redis.cache.RedisCache"
+CACHES["default"]["OPTIONS"] = {
+    "CLIENT_CLASS": "django_redis.client.DefaultClient",
+    "IGNORE_EXCEPTIONS": True,  # mimics memcache behavior.
+                                # http://niwinz.github.io/django-redis/latest/#_memcached_exceptions_behavior
+}
+
+
+########## CELERY
+INSTALLED_APPS += ('funkwhale_api.taskapp.celery.CeleryConfig',)
+BROKER_URL = env(
+    "CELERY_BROKER_URL", default=env('CACHE_URL', default=CACHE_DEFAULT))
+########## END CELERY
 # Location of root django.contrib.admin URL, use {% url 'admin:index' %}
 ADMIN_URL = r'^admin/'
 # Your common stuff: Below this line define 3rd party library settings
@@ -336,3 +332,8 @@ MUSICBRAINZ_CACHE_DURATION = env.int(
 )
 
 CACHALOT_ENABLED = env.bool('CACHALOT_ENABLED', default=True)
+
+
+# Custom Admin URL, use {% url 'admin:index' %}
+ADMIN_URL = env('DJANGO_ADMIN_URL', default='^api/admin/')
+CSRF_USE_SESSIONS = True

api/config/settings/production.py

@@ -54,7 +54,7 @@ SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
 # ------------------------------------------------------------------------------
 # Hosts/domain names that are valid for this site
 # See https://docs.djangoproject.com/en/1.6/ref/settings/#allowed-hosts
-ALLOWED_HOSTS = env.list('DJANGO_ALLOWED_HOSTS', default=['funkwhale.io'])
+ALLOWED_HOSTS = env.list('DJANGO_ALLOWED_HOSTS')
 # END SITE CONFIGURATION
 
 INSTALLED_APPS += ("gunicorn", )
@@ -65,10 +65,6 @@ INSTALLED_APPS += ("gunicorn", )
 # ------------------------
 DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage'
 
-# URL that handles the media served from MEDIA_ROOT, used for managing
-# stored files.
-MEDIA_URL = '/media/'
-
 # Static Assets
 # ------------------------
 STATICFILES_STORAGE = 'django.contrib.staticfiles.storage.StaticFilesStorage'
@@ -92,11 +88,6 @@ TEMPLATES[0]['OPTIONS']['loaders'] = [
         'django.template.loaders.filesystem.Loader', 'django.template.loaders.app_directories.Loader', ]),
 ]
 
-# DATABASE CONFIGURATION
-# ------------------------------------------------------------------------------
-# Raises ImproperlyConfigured exception if DATABASE_URL not in os.environ
-DATABASES['default'] = env.db("DATABASE_URL")
-
 # CACHING
 # ------------------------------------------------------------------------------
 # Heroku URL does not pass the DB number, so we parse it in
@@ -151,7 +142,5 @@ LOGGING = {
     }
 }
 
-# Custom Admin URL, use {% url 'admin:index' %}
-ADMIN_URL = env('DJANGO_ADMIN_URL')
 
 # Your production stuff: Below this line define 3rd party library settings

api/config/settings/test.py

@@ -22,6 +22,9 @@ CACHES = {
         'LOCATION': ''
     }
 }
+INSTALLED_APPS += ('kombu.transport.django',)
+BROKER_URL = 'django://'
+
 # TESTING
 # ------------------------------------------------------------------------------
 TEST_RUNNER = 'django.test.runner.DiscoverRunner'

api/funkwhale_api/__init__.py

 # -*- coding: utf-8 -*-
-__version__ = '0.2.3'
+__version__ = '0.2.4'
 __version_info__ = tuple([int(num) if num.isdigit() else num for num in __version__.replace('-', '.', 1).split('.')])

api/funkwhale_api/contrib/sites/migrations/0003_auto_20171214_2205.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11 on 2017-12-14 22:05
+from __future__ import unicode_literals
+
+import django.contrib.sites.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sites', '0002_set_site_domain_and_name'),
+    ]
+
+    operations = [
+        migrations.AlterModelManagers(
+            name='site',
+            managers=[
+                ('objects', django.contrib.sites.models.SiteManager()),
+            ],
+        ),
+        migrations.AlterField(
+            model_name='site',
+            name='domain',
+            field=models.CharField(max_length=100, unique=True, validators=[django.contrib.sites.models._simple_domain_name_validator], verbose_name='domain name'),
+        ),
+    ]

api/funkwhale_api/music/admin.py

@@ -2,30 +2,35 @@ from django.contrib import admin
 
 from . import models
 
+
 @admin.register(models.Artist)
 class ArtistAdmin(admin.ModelAdmin):
     list_display = ['name', 'mbid', 'creation_date']
     search_fields = ['name', 'mbid']
 
+
 @admin.register(models.Album)
 class AlbumAdmin(admin.ModelAdmin):
     list_display = ['title', 'artist', 'mbid', 'release_date', 'creation_date']
     search_fields = ['title', 'artist__name', 'mbid']
     list_select_related = True
 
+
 @admin.register(models.Track)
 class TrackAdmin(admin.ModelAdmin):
     list_display = ['title', 'artist', 'album', 'mbid']
     search_fields = ['title', 'artist__name', 'album__title', 'mbid']
     list_select_related = True
 
+
 @admin.register(models.ImportBatch)
 class ImportBatchAdmin(admin.ModelAdmin):
     list_display = ['creation_date', 'status']
 
+
 @admin.register(models.ImportJob)
 class ImportJobAdmin(admin.ModelAdmin):
-    list_display = ['source', 'batch', 'status', 'mbid']
+    list_display = ['source', 'batch', 'track_file', 'status', 'mbid']
     list_select_related = True
     search_fields = ['source', 'batch__pk', 'mbid']
     list_filter = ['status']

api/funkwhale_api/music/migrations/0013_auto_20171213_2211.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11 on 2017-12-13 22:11
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0012_auto_20161122_1905'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='importjob',
+            options={'ordering': ('id',)},
+        ),
+        migrations.AlterModelOptions(
+            name='track',
+            options={'ordering': ['album', 'position']},
+        ),
+        migrations.AddField(
+            model_name='album',
+            name='release_group_id',
+            field=models.UUIDField(blank=True, null=True),
+        ),
+    ]

api/funkwhale_api/music/migrations/0014_importjob_track_file.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11 on 2017-12-14 21:14
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0013_auto_20171213_2211'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='importjob',
+            name='track_file',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='jobs', to='music.TrackFile'),
+        ),
+    ]

api/funkwhale_api/music/migrations/0015_bind_track_file_to_import_job.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+import os
+
+from django.db import migrations, models
+from funkwhale_api.common.utils import rename_file
+
+
+def bind_jobs(apps, schema_editor):
+    TrackFile = apps.get_model("music", "TrackFile")
+    ImportJob = apps.get_model("music", "ImportJob")
+
+    for job in ImportJob.objects.all().only('mbid'):
+        f = TrackFile.objects.filter(track__mbid=job.mbid).first()
+        if not f:
+            print('No file for mbid {}'.format(job.mbid))
+            continue
+        job.track_file = f
+        job.save(update_fields=['track_file'])
+
+
+def rewind(apps, schema_editor):
+    pass
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0014_importjob_track_file'),
+    ]
+
+    operations = [
+        migrations.RunPython(bind_jobs, rewind),
+    ]

api/funkwhale_api/music/models.py

@@ -110,13 +110,14 @@ class Album(APIModelMixin):
     title = models.CharField(max_length=255)
     artist = models.ForeignKey(Artist, related_name='albums')
     release_date = models.DateField(null=True)
+    release_group_id = models.UUIDField(null=True, blank=True)
     cover = VersatileImageField(upload_to='albums/covers/%Y/%m/%d', null=True, blank=True)
     TYPE_CHOICES = (
         ('album', 'Album'),
     )
     type = models.CharField(choices=TYPE_CHOICES, max_length=30, default='album')
 
-    api_includes = ['artist-credits', 'recordings', 'media']
+    api_includes = ['artist-credits', 'recordings', 'media', 'release-groups']
     api = musicbrainz.api.releases
     musicbrainz_model = 'release'
     musicbrainz_mapping = {
@@ -127,6 +128,10 @@ class Album(APIModelMixin):
             'musicbrainz_field_name': 'release-list',
             'converter': lambda v: int(v[0]['medium-list'][0]['position']),
         },
+        'release_group_id': {
+            'musicbrainz_field_name': 'release-group',
+            'converter': lambda v: v['id'],
+        },
         'title': {
             'musicbrainz_field_name': 'title',
         },
@@ -388,6 +393,8 @@ class ImportBatch(models.Model):
 
 class ImportJob(models.Model):
     batch = models.ForeignKey(ImportBatch, related_name='jobs')
+    track_file = models.ForeignKey(
+        TrackFile, related_name='jobs', null=True, blank=True)
     source = models.URLField()
     mbid = models.UUIDField(editable=False)
     STATUS_CHOICES = (
@@ -408,10 +415,12 @@ class ImportJob(models.Model):
             elif track.files.count() > 0:
                 return
 
-            track_file = track_file or TrackFile(track=track, source=self.source)
+            track_file = track_file or TrackFile(
+                track=track, source=self.source)
             track_file.download_file()
             track_file.save()
             self.status = 'finished'
+            self.track_file = track_file
             self.save()
             return track.pk
 

api/funkwhale_api/music/serializers.py

@@ -9,35 +9,26 @@ class TagSerializer(serializers.ModelSerializer):
         model = Tag
         fields = ('id', 'name', 'slug')
 
+
 class SimpleArtistSerializer(serializers.ModelSerializer):
     class Meta:
         model = models.Artist
         fields = ('id', 'mbid', 'name')
 
+
 class ArtistSerializer(serializers.ModelSerializer):
     tags = TagSerializer(many=True, read_only=True)
     class Meta:
         model = models.Artist
         fields = ('id', 'mbid', 'name', 'tags')
 
-class ImportJobSerializer(serializers.ModelSerializer):
-    class Meta:
-        model = models.ImportJob
-        fields = ('id', 'mbid', 'source', 'status')
-
-class ImportBatchSerializer(serializers.ModelSerializer):
-    jobs = ImportJobSerializer(many=True, read_only=True)
-    class Meta:
-        model = models.ImportBatch
-        fields = ('id', 'jobs', 'status', 'creation_date')
-
 
 class TrackFileSerializer(serializers.ModelSerializer):
     path = serializers.SerializerMethodField()
 
     class Meta:
         model = models.TrackFile
-        fields = ('id', 'path', 'duration', 'source', 'filename')
+        fields = ('id', 'path', 'duration', 'source', 'filename', 'track')
 
     def get_path(self, o):
         request = self.context.get('request')
@@ -46,12 +37,14 @@ class TrackFileSerializer(serializers.ModelSerializer):
             url = request.build_absolute_uri(url)
         return url
 
+
 class SimpleAlbumSerializer(serializers.ModelSerializer):
 
     class Meta:
         model = models.Album
         fields = ('id', 'mbid', 'title', 'release_date', 'cover')
 
+
 class AlbumSerializer(serializers.ModelSerializer):
     tags = TagSerializer(many=True, read_only=True)
     class Meta:
@@ -81,6 +74,7 @@ class TrackSerializer(LyricsMixin):
             'position',
             'lyrics')
 
+
 class TrackSerializerNested(LyricsMixin):
     artist = ArtistSerializer()
     files = TrackFileSerializer(many=True, read_only=True)
@@ -90,6 +84,7 @@ class TrackSerializerNested(LyricsMixin):
         model = models.Track
         fields = ('id', 'mbid', 'title', 'artist', 'files', 'album', 'tags', 'lyrics')
 
+
 class AlbumSerializerNested(serializers.ModelSerializer):
     tracks = TrackSerializer(many=True, read_only=True)
     artist = SimpleArtistSerializer()
@@ -99,6 +94,7 @@ class AlbumSerializerNested(serializers.ModelSerializer):
         model = models.Album
         fields = ('id', 'mbid', 'title', 'cover', 'artist', 'release_date', 'tracks', 'tags')
 
+
 class ArtistSerializerNested(serializers.ModelSerializer):
     albums = AlbumSerializerNested(many=True, read_only=True)
     tags = TagSerializer(many=True, read_only=True)
@@ -111,3 +107,17 @@ class LyricsSerializer(serializers.ModelSerializer):
     class Meta:
         model = models.Lyrics
         fields = ('id', 'work', 'content', 'content_rendered')
+
+
+class ImportJobSerializer(serializers.ModelSerializer):
+    track_file = TrackFileSerializer(read_only=True)
+    class Meta:
+        model = models.ImportJob
+        fields = ('id', 'mbid', 'source', 'status', 'track_file')
+
+
+class ImportBatchSerializer(serializers.ModelSerializer):
+    jobs = ImportJobSerializer(many=True, read_only=True)
+    class Meta:
+        model = models.ImportBatch
+        fields = ('id', 'jobs', 'status', 'creation_date')

api/funkwhale_api/music/tests/factories.py

 import factory
 import os
 
+from funkwhale_api.users.tests.factories import UserFactory
+
 SAMPLES_PATH = os.path.dirname(os.path.abspath(__file__))
 
 
@@ -18,6 +20,7 @@ class AlbumFactory(factory.django.DjangoModelFactory):
     release_date = factory.Faker('date')
     cover = factory.django.ImageField()
     artist = factory.SubFactory(ArtistFactory)
+    release_group_id = factory.Faker('uuid4')
 
     class Meta:
         model = 'music.Album'
@@ -41,3 +44,18 @@ class TrackFileFactory(factory.django.DjangoModelFactory):
 
     class Meta:
         model = 'music.TrackFile'
+
+
+class ImportBatchFactory(factory.django.DjangoModelFactory):
+    submitted_by = factory.SubFactory(UserFactory)
+
+    class Meta:
+        model = 'music.ImportBatch'
+
+
+class ImportJobFactory(factory.django.DjangoModelFactory):
+    batch = factory.SubFactory(ImportBatchFactory)
+    source = factory.Faker('url')
+
+    class Meta:
+        model = 'music.ImportJob'

api/funkwhale_api/music/tests/test_models.py

+import pytest
+
+from funkwhale_api.music import models
+from funkwhale_api.music import importers
+from . import factories
+
+
+def test_can_store_release_group_id_on_album(db):
+    album = factories.AlbumFactory()
+    assert album.release_group_id is not None
+
+
+def test_import_album_stores_release_group(db):
+
+    album_data = {
+        "artist-credit": [
+            {
+                "artist": {
+                    "disambiguation": "George Shaw",
+                    "id": "62c3befb-6366-4585-b256-809472333801",
+                    "name": "Adhesive Wombat",
+                    "sort-name": "Wombat, Adhesive"
+                }
+            }
+        ],
+        "artist-credit-phrase": "Adhesive Wombat",
+        "country": "XW",
+        "date": "2013-06-05",
+        "id": "a50d2a81-2a50-484d-9cb4-b9f6833f583e",
+        "status": "Official",
+        "title": "Marsupial Madness",
+        'release-group': {'id': '447b4979-2178-405c-bfe6-46bf0b09e6c7'}
+    }
+    artist = factories.ArtistFactory(
+        mbid=album_data['artist-credit'][0]['artist']['id']
+    )
+    cleaned_data = models.Album.clean_musicbrainz_data(album_data)
+    album = importers.load(models.Album, cleaned_data, album_data, import_hooks=[])
+
+    assert album.release_group_id == album_data['release-group']['id']
+    assert album.artist == artist
+
+
+def test_import_job_is_bound_to_track_file(db, mocker):
+    track = factories.TrackFactory()
+    job = factories.ImportJobFactory(mbid=track.mbid)
+
+    mocker.patch('funkwhale_api.music.models.TrackFile.download_file')
+    job.run()
+    job.refresh_from_db()
+    assert job.track_file.track == track

api/funkwhale_api/music/views.py

@@ -72,7 +72,10 @@ class AlbumViewSet(SearchMixin, viewsets.ReadOnlyModelViewSet):
 
 
 class ImportBatchViewSet(viewsets.ReadOnlyModelViewSet):
-    queryset = models.ImportBatch.objects.all().order_by('-creation_date')
+    queryset = (
+        models.ImportBatch.objects.all()
+                          .prefetch_related('jobs__track_file')
+                          .order_by('-creation_date'))
     serializer_class = serializers.ImportBatchSerializer
 
     def get_queryset(self):

api/funkwhale_api/providers/audiofile/management/commands/import_files.py

 import glob
 from django.core.management.base import BaseCommand, CommandError
-from funkwhale_api.providers.audiofile import importer
+from funkwhale_api.providers.audiofile import tasks
 
 
 class Command(BaseCommand):
@@ -61,7 +61,7 @@ class Command(BaseCommand):
         for path in matching:
             self.stdout.write(message.format(path))
             try:
-                importer.from_path(path)
+                tasks.from_path(path)
             except Exception as e:
                 self.stdout.write('Error: {}'.format(e))