Initial commit that merge both the front end and the API in the same repository

api/funkwhale_api/history/tests/test_history.py

+import random
+import json
+from test_plus.test import TestCase
+from django.core.urlresolvers import reverse
+from django.core.exceptions import ValidationError
+from django.utils import timezone
+
+from model_mommy import mommy
+
+from funkwhale_api.users.models import User
+from funkwhale_api.history import models
+
+class TestHistory(TestCase):
+
+    def setUp(self):
+        super().setUp()
+        self.user = User.objects.create_user(username='test', email='test@test.com', password='test')
+
+    def test_can_create_listening(self):
+        track = mommy.make('music.Track')
+        now = timezone.now()
+        l = models.Listening.objects.create(user=self.user, track=track)
+
+    def test_anonymous_user_can_create_listening_via_api(self):
+        track = mommy.make('music.Track')
+        url = self.reverse('api:history:listenings-list')
+        response = self.client.post(url, {
+            'track': track.pk,
+        })
+
+        listening = models.Listening.objects.latest('id')
+
+        self.assertEqual(listening.track, track)
+        self.assertIsNotNone(listening.session_key)
+
+    def test_logged_in_user_can_create_listening_via_api(self):
+        track = mommy.make('music.Track')
+
+        self.client.login(username=self.user.username, password='test')
+
+        url = self.reverse('api:history:listenings-list')
+        response = self.client.post(url, {
+            'track': track.pk,
+        })
+
+        listening = models.Listening.objects.latest('id')
+
+        self.assertEqual(listening.track, track)
+        self.assertEqual(listening.user, self.user)

api/funkwhale_api/history/urls.py

+from django.conf.urls import include, url
+from . import views
+
+from rest_framework import routers
+router = routers.SimpleRouter()
+router.register(r'listenings', views.ListeningViewSet, 'listenings')
+
+urlpatterns = router.urls

api/funkwhale_api/history/views.py

+from rest_framework import generics, mixins, viewsets
+from rest_framework import status
+from rest_framework.response import Response
+from rest_framework.decorators import detail_route
+
+from funkwhale_api.music.serializers import TrackSerializerNested
+from funkwhale_api.common.permissions import ConditionalAuthentication
+
+from . import models
+from . import serializers
+
+class ListeningViewSet(mixins.CreateModelMixin,
+                          mixins.RetrieveModelMixin,
+                          viewsets.GenericViewSet):
+
+    serializer_class = serializers.ListeningSerializer
+    queryset = models.Listening.objects.all()
+    permission_classes = [ConditionalAuthentication]
+
+    def create(self, request, *args, **kwargs):
+        return super().create(request, *args, **kwargs)
+
+    def get_queryset(self):
+        queryset = super().get_queryset()
+        if self.request.user.is_authenticated():
+            return queryset.filter(user=self.request.user)
+        else:
+            return queryset.filter(session_key=self.request.session.session_key)
+
+    def get_serializer_context(self):
+        context = super().get_serializer_context()
+        if self.request.user.is_authenticated():
+            context['user'] = self.request.user
+        else:
+            context['session_key'] = self.request.session.session_key
+        return context

api/funkwhale_api/music/admin.py

+from django.contrib import admin
+
+from . import models
+
+@admin.register(models.Artist)
+class ArtistAdmin(admin.ModelAdmin):
+    list_display = ['name', 'mbid', 'creation_date']
+    search_fields = ['name', 'mbid']
+
+@admin.register(models.Album)
+class AlbumAdmin(admin.ModelAdmin):
+    list_display = ['title', 'artist', 'mbid', 'release_date', 'creation_date']
+    search_fields = ['title', 'artist__name', 'mbid']
+    list_select_related = True
+
+@admin.register(models.Track)
+class TrackAdmin(admin.ModelAdmin):
+    list_display = ['title', 'artist', 'album', 'mbid']
+    search_fields = ['title', 'artist__name', 'album__title', 'mbid']
+    list_select_related = True
+
+@admin.register(models.ImportBatch)
+class ImportBatchAdmin(admin.ModelAdmin):
+    list_display = ['creation_date', 'status']
+
+@admin.register(models.ImportJob)
+class ImportJobAdmin(admin.ModelAdmin):
+    list_display = ['source', 'batch', 'status', 'mbid']
+    list_select_related = True
+    search_fields = ['source', 'batch__pk', 'mbid']
+    list_filter = ['status']
+
+
+@admin.register(models.Work)
+class WorkAdmin(admin.ModelAdmin):
+    list_display = ['title', 'mbid', 'language', 'nature']
+    list_select_related = True
+    search_fields = ['title']
+    list_filter = ['language', 'nature']
+
+
+@admin.register(models.Lyrics)
+class LyricsAdmin(admin.ModelAdmin):
+    list_display = ['url', 'id', 'url']
+    list_select_related = True
+    search_fields = ['url', 'work__title']
+    list_filter = ['work__language']

api/funkwhale_api/music/importers.py

+
+
+def load(model, *args, **kwargs):
+    importer = registry[model.__name__](model=model)
+    return importer.load(*args, **kwargs)
+
+class Importer(object):
+    def __init__(self, model):
+        self.model = model
+
+    def load(self, cleaned_data, raw_data, import_hooks):
+        mbid = cleaned_data.pop('mbid')
+        m = self.model.objects.update_or_create(mbid=mbid, defaults=cleaned_data)[0]
+        for hook in import_hooks:
+            hook(m, cleaned_data, raw_data)
+        return m
+
+class Mapping(object):
+    """Cast musicbrainz data to funkwhale data and vice-versa"""
+    def __init__(self, musicbrainz_mapping):
+        self.musicbrainz_mapping = musicbrainz_mapping
+
+        self._from_musicbrainz = {}
+        self._to_musicbrainz = {}
+        for field_name, conf in self.musicbrainz_mapping.items():
+            self._from_musicbrainz[conf['musicbrainz_field_name']] = {
+                'field_name': field_name,
+                'converter': conf.get('converter', lambda v: v)
+            }
+            self._to_musicbrainz[field_name] = {
+                'field_name': conf['musicbrainz_field_name'],
+                'converter': conf.get('converter', lambda v: v)
+            }
+    def from_musicbrainz(self, key, value):
+        return self._from_musicbrainz[key]['field_name'], self._from_musicbrainz[key]['converter'](value)
+
+registry = {
+    'Artist': Importer,
+    'Track': Importer,
+    'Album': Importer,
+    'Work': Importer,
+}

api/funkwhale_api/music/lyrics.py

+import urllib.request
+import html.parser
+from bs4 import BeautifulSoup
+
+
+def _get_html(url):
+    with urllib.request.urlopen(url) as response:
+        html = response.read()
+    return html.decode('utf-8')
+
+
+def extract_content(html):
+    soup = BeautifulSoup(html, "html.parser")
+    return soup.find_all("div", class_='lyricbox')[0].contents
+
+
+def clean_content(contents):
+    final_content = ""
+    for e in contents:
+        if e == '\n':
+            continue
+        if e.name == 'script':
+            continue
+        if e.name == 'br':
+            final_content += "\n"
+            continue
+        try:
+            final_content += e.text
+        except AttributeError:
+            final_content += str(e)
+    return final_content

api/funkwhale_api/music/metadata.py

+import mutagen
+
+NODEFAULT = object()
+
+class Metadata(object):
+    ALIASES = {
+        'release': 'musicbrainz_albumid',
+        'artist': 'musicbrainz_artistid',
+        'recording': 'musicbrainz_trackid',
+    }
+
+    def __init__(self, path):
+        self._file = mutagen.File(path)
+
+    def get(self, key, default=NODEFAULT, single=True):
+        try:
+            v = self._file[key]
+        except KeyError:
+            if default == NODEFAULT:
+                raise
+            return default
+
+        # Some tags are returned as lists of string
+        if single:
+            return v[0]
+        return v
+
+    def __getattr__(self, key):
+        try:
+            alias = self.ALIASES[key]
+        except KeyError:
+            raise ValueError('Invalid alias {}'.format(key))
+
+        return self.get(alias, single=True)

api/funkwhale_api/music/migrations/0001_initial.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+from django.conf import settings
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Album',
+            fields=[
+                ('id', models.AutoField(primary_key=True, auto_created=True, serialize=False, verbose_name='ID')),
+                ('mbid', models.UUIDField(editable=False, blank=True, null=True)),
+                ('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
+                ('title', models.CharField(max_length=255)),
+                ('release_date', models.DateField()),
+                ('type', models.CharField(default='album', choices=[('album', 'Album')], max_length=30)),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='Artist',
+            fields=[
+                ('id', models.AutoField(primary_key=True, auto_created=True, serialize=False, verbose_name='ID')),
+                ('mbid', models.UUIDField(editable=False, blank=True, null=True)),
+                ('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
+                ('name', models.CharField(max_length=255)),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='ImportBatch',
+            fields=[
+                ('id', models.AutoField(primary_key=True, auto_created=True, serialize=False, verbose_name='ID')),
+                ('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
+                ('submitted_by', models.ForeignKey(related_name='imports', to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='ImportJob',
+            fields=[
+                ('id', models.AutoField(primary_key=True, auto_created=True, serialize=False, verbose_name='ID')),
+                ('source', models.URLField()),
+                ('mbid', models.UUIDField(editable=False)),
+                ('status', models.CharField(default='pending', choices=[('pending', 'Pending'), ('finished', 'finished')], max_length=30)),
+                ('batch', models.ForeignKey(related_name='jobs', to='music.ImportBatch')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Track',
+            fields=[
+                ('id', models.AutoField(primary_key=True, auto_created=True, serialize=False, verbose_name='ID')),
+                ('mbid', models.UUIDField(editable=False, blank=True, null=True)),
+                ('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
+                ('title', models.CharField(max_length=255)),
+                ('album', models.ForeignKey(related_name='tracks', blank=True, null=True, to='music.Album')),
+                ('artist', models.ForeignKey(related_name='tracks', to='music.Artist')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='TrackFile',
+            fields=[
+                ('id', models.AutoField(primary_key=True, auto_created=True, serialize=False, verbose_name='ID')),
+                ('audio_file', models.FileField(upload_to='tracks')),
+                ('source', models.URLField(blank=True, null=True)),
+                ('duration', models.IntegerField(blank=True, null=True)),
+                ('track', models.ForeignKey(related_name='files', to='music.Track')),
+            ],
+        ),
+        migrations.AddField(
+            model_name='album',
+            name='artist',
+            field=models.ForeignKey(related_name='albums', to='music.Artist'),
+        ),
+    ]

api/funkwhale_api/music/migrations/0002_auto_20151215_1645.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='album',
+            options={'ordering': ['-creation_date']},
+        ),
+        migrations.AlterModelOptions(
+            name='artist',
+            options={'ordering': ['-creation_date']},
+        ),
+        migrations.AlterModelOptions(
+            name='importbatch',
+            options={'ordering': ['-creation_date']},
+        ),
+        migrations.AlterModelOptions(
+            name='track',
+            options={'ordering': ['-creation_date']},
+        ),
+        migrations.AddField(
+            model_name='album',
+            name='cover',
+            field=models.ImageField(upload_to='albums/covers/%Y/%m/%d', null=True, blank=True),
+        ),
+        migrations.AlterField(
+            model_name='trackfile',
+            name='audio_file',
+            field=models.FileField(upload_to='tracks/%Y/%m/%d'),
+        ),
+    ]

api/funkwhale_api/music/migrations/0003_auto_20151222_2233.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0002_auto_20151215_1645'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='album',
+            name='release_date',
+            field=models.DateField(null=True),
+        ),
+    ]

api/funkwhale_api/music/migrations/0004_track_tags.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import taggit.managers
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('taggit', '0002_auto_20150616_2121'),
+        ('music', '0003_auto_20151222_2233'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='track',
+            name='tags',
+            field=taggit.managers.TaggableManager(verbose_name='Tags', help_text='A comma-separated list of tags.', through='taggit.TaggedItem', to='taggit.Tag'),
+        ),
+    ]

api/funkwhale_api/music/migrations/0005_deduplicate.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+def get_duplicates(model):
+    return [i['mbid'] for i in model.objects.values('mbid').annotate(idcount=models.Count('mbid')).order_by('-idcount') if i['idcount'] > 1]
+
+def deduplicate(apps, schema_editor):
+    Artist = apps.get_model("music", "Artist")
+    Album = apps.get_model("music", "Album")
+    Track = apps.get_model("music", "Track")
+
+    for mbid in get_duplicates(Artist):
+        ref = Artist.objects.filter(mbid=mbid).order_by('pk').first()
+        duplicates = Artist.objects.filter(mbid=mbid).exclude(pk=ref.pk)
+        Album.objects.filter(artist__in=duplicates).update(artist=ref)
+        Track.objects.filter(artist__in=duplicates).update(artist=ref)
+        duplicates.delete()
+
+    for mbid in get_duplicates(Album):
+        ref = Album.objects.filter(mbid=mbid).order_by('pk').first()
+        duplicates = Album.objects.filter(mbid=mbid).exclude(pk=ref.pk)
+        Track.objects.filter(album__in=duplicates).update(album=ref)
+        duplicates.delete()
+
+def rewind(*args, **kwargs):
+    pass
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0004_track_tags'),
+    ]
+
+    operations = [
+        migrations.RunPython(deduplicate, rewind),
+    ]

api/funkwhale_api/music/migrations/0006_unique_mbid.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0005_deduplicate'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='album',
+            name='mbid',
+            field=models.UUIDField(null=True, editable=False, unique=True, blank=True, db_index=True),
+        ),
+        migrations.AlterField(
+            model_name='artist',
+            name='mbid',
+            field=models.UUIDField(null=True, editable=False, unique=True, blank=True, db_index=True),
+        ),
+        migrations.AlterField(
+            model_name='track',
+            name='mbid',
+            field=models.UUIDField(null=True, editable=False, unique=True, blank=True, db_index=True),
+        ),
+    ]

api/funkwhale_api/music/migrations/0007_track_position.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0006_unique_mbid'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='track',
+            name='position',
+            field=models.PositiveIntegerField(blank=True, null=True),
+        ),
+    ]

api/funkwhale_api/music/migrations/0008_auto_20160529_1456.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0007_track_position'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='album',
+            name='mbid',
+            field=models.UUIDField(null=True, db_index=True, unique=True, blank=True),
+        ),
+        migrations.AlterField(
+            model_name='artist',
+            name='mbid',
+            field=models.UUIDField(null=True, db_index=True, unique=True, blank=True),
+        ),
+        migrations.AlterField(
+            model_name='track',
+            name='mbid',
+            field=models.UUIDField(null=True, db_index=True, unique=True, blank=True),
+        ),
+    ]

api/funkwhale_api/music/migrations/0009_auto_20160920_1614.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.utils.timezone
+import versatileimagefield.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0008_auto_20160529_1456'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Lyrics',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, verbose_name='ID', serialize=False)),
+                ('url', models.URLField()),
+                ('content', models.TextField(null=True, blank=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Work',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, verbose_name='ID', serialize=False)),
+                ('mbid', models.UUIDField(unique=True, null=True, db_index=True, blank=True)),
+                ('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
+                ('language', models.CharField(max_length=20)),
+                ('nature', models.CharField(max_length=50)),
+                ('title', models.CharField(max_length=255)),
+            ],
+            options={
+                'ordering': ['-creation_date'],
+                'abstract': False,
+            },
+        ),
+        migrations.AddField(
+            model_name='lyrics',
+            name='work',
+            field=models.ForeignKey(related_name='lyrics', to='music.Work', blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='track',
+            name='work',
+            field=models.ForeignKey(related_name='tracks', to='music.Work', blank=True, null=True),
+        ),
+    ]

api/funkwhale_api/music/migrations/0010_auto_20160920_1742.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import versatileimagefield.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0009_auto_20160920_1614'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='lyrics',
+            name='url',
+            field=models.URLField(unique=True),
+        ),
+    ]

api/funkwhale_api/music/migrations/0011_rename_files.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+import os
+
+from django.db import migrations, models
+from funkwhale_api.common.utils import rename_file
+
+
+def rename_files(apps, schema_editor):
+    """
+    This migration script is utterly broken and made me redownload all my audio files.
+    So next time -> Write some actual tests before running a migration script
+    on thousand of tracks...
+    """
+    return
+    # TrackFile = apps.get_model("music", "TrackFile")
+    # qs = TrackFile.objects.select_related(
+    #     'track__album__artist', 'track__artist')
+    # total = len(qs)
+    #
+    #
+    # for i, tf in enumerate(qs):
+    #     try:
+    #         new_name = '{} - {} - {}'.format(
+    #             tf.track.artist.name,
+    #             tf.track.album.title,
+    #             tf.track.title,
+    #         )
+    #     except AttributeError:
+    #         new_name = '{} - {}'.format(
+    #             tf.track.artist.name,
+    #             tf.track.title,
+    #         )
+    #     rename_file(
+    #         instance=tf,
+    #         field_name='audio_file',
+    #         allow_missing_file=True,
+    #         new_name=new_name)
+    #     print('Renamed file {}/{} (new name: {})'.format(
+    #         i + 1, total, tf.audio_file.name
+    #     ))
+
+
+def rewind(apps, schema_editor):
+    pass
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0010_auto_20160920_1742'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='trackfile',
+            name='audio_file',
+            field=models.FileField(upload_to='tracks/%Y/%m/%d', max_length=255),
+        ),
+        migrations.RunPython(rename_files, rewind),
+    ]

api/funkwhale_api/music/migrations/0012_auto_20161122_1905.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import versatileimagefield.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0011_rename_files'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='album',
+            name='cover',
+            field=versatileimagefield.fields.VersatileImageField(null=True, blank=True, upload_to='albums/covers/%Y/%m/%d'),
+        ),
+    ]