Merge branch 'release/0.2'

.env.dev

-BACKEND_URL=http://localhost:12081
-YOUTUBE_API_KEY=
-API_AUTHENTICATION_REQUIRED=False
+BACKEND_URL=http://localhost:6001
+API_AUTHENTICATION_REQUIRED=True
+CACHALOT_ENABLED=False

.gitlab-ci.yml

@@ -70,6 +70,7 @@ docker_develop:
   stage: deploy
   before_script:
     - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
+    - cp -r front/dist api/frontend
     - cd api
   script:
     - docker build -t $IMAGE .
@@ -83,6 +84,7 @@ docker_release:
   stage: deploy
   before_script:
     - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
+    - cp -r front/dist api/frontend
     - cd api
   script:
     - docker build -t $IMAGE -t $IMAGE_LATEST .

api/compose/django/entrypoint.sh

@@ -9,10 +9,15 @@ export REDIS_URL=redis://redis:6379/0
 # the official postgres image uses 'postgres' as default user if not set explictly.
 if [ -z "$POSTGRES_ENV_POSTGRES_USER" ]; then
     export POSTGRES_ENV_POSTGRES_USER=postgres
-fi 
+fi
 
 export DATABASE_URL=postgres://$POSTGRES_ENV_POSTGRES_USER:$POSTGRES_ENV_POSTGRES_PASSWORD@postgres:5432/$POSTGRES_ENV_POSTGRES_USER
 
 export CELERY_BROKER_URL=$REDIS_URL
 
-exec "$@"
+# we copy the frontend files, if any so we can serve them from the outside
\ No newline at end of file
+if [ -d "frontend" ]; then
+  mkdir -p /frontend
+  cp -r frontend/* /frontend/
+fi
+exec "$@"

api/compose/nginx/Dockerfile

-FROM nginx:latest
-ADD nginx.conf /etc/nginx/nginx.conf
\ No newline at end of file

api/config/api_urls.py

@@ -4,26 +4,40 @@ from funkwhale_api.music import views
 from funkwhale_api.playlists import views as playlists_views
 from rest_framework_jwt import views as jwt_views
 
+from dynamic_preferences.api.viewsets import GlobalPreferencesViewSet
+from dynamic_preferences.users.viewsets import UserPreferencesViewSet
 
 router = routers.SimpleRouter()
+router.register(r'settings', GlobalPreferencesViewSet, base_name='settings')
 router.register(r'tags', views.TagViewSet, 'tags')
 router.register(r'tracks', views.TrackViewSet, 'tracks')
+router.register(r'trackfiles', views.TrackFileViewSet, 'trackfiles')
 router.register(r'artists', views.ArtistViewSet, 'artists')
 router.register(r'albums', views.AlbumViewSet, 'albums')
 router.register(r'import-batches', views.ImportBatchViewSet, 'import-batches')
 router.register(r'submit', views.SubmitViewSet, 'submit')
 router.register(r'playlists', playlists_views.PlaylistViewSet, 'playlists')
-router.register(r'playlist-tracks', playlists_views.PlaylistTrackViewSet, 'playlist-tracks')
+router.register(
+    r'playlist-tracks',
+    playlists_views.PlaylistTrackViewSet,
+    'playlist-tracks')
 v1_patterns = router.urls
 
 v1_patterns += [
-    url(r'^providers/', include('funkwhale_api.providers.urls', namespace='providers')),
-    url(r'^favorites/', include('funkwhale_api.favorites.urls', namespace='favorites')),
-    url(r'^search$', views.Search.as_view(), name='search'),
-    url(r'^radios/', include('funkwhale_api.radios.urls', namespace='radios')),
-    url(r'^history/', include('funkwhale_api.history.urls', namespace='history')),
-    url(r'^users/', include('funkwhale_api.users.api_urls', namespace='users')),
-    url(r'^token/', jwt_views.obtain_jwt_token),
+    url(r'^providers/',
+        include('funkwhale_api.providers.urls', namespace='providers')),
+    url(r'^favorites/',
+        include('funkwhale_api.favorites.urls', namespace='favorites')),
+    url(r'^search$',
+        views.Search.as_view(), name='search'),
+    url(r'^radios/',
+        include('funkwhale_api.radios.urls', namespace='radios')),
+    url(r'^history/',
+        include('funkwhale_api.history.urls', namespace='history')),
+    url(r'^users/',
+        include('funkwhale_api.users.api_urls', namespace='users')),
+    url(r'^token/',
+        jwt_views.obtain_jwt_token),
     url(r'^token/refresh/', jwt_views.refresh_jwt_token),
 ]
 

api/config/settings/common.py

@@ -53,6 +53,7 @@ THIRD_PARTY_APPS = (
     'rest_auth',
     'rest_auth.registration',
     'mptt',
+    'dynamic_preferences',
 )
 
 # Apps specific for this project go here.
@@ -65,6 +66,7 @@ LOCAL_APPS = (
     'funkwhale_api.history',
     'funkwhale_api.playlists',
     'funkwhale_api.providers.audiofile',
+    'funkwhale_api.providers.youtube',
 )
 
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
@@ -217,7 +219,6 @@ STATICFILES_FINDERS = (
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#media-root
 MEDIA_ROOT = str(APPS_DIR('media'))
 
-USE_SAMPLE_TRACK = env.bool("USE_SAMPLE_TRACK", False)
 
 
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#media-url
@@ -261,7 +262,6 @@ BROKER_URL = env("CELERY_BROKER_URL", default='django://')
 
 # Location of root django.contrib.admin URL, use {% url 'admin:index' %}
 ADMIN_URL = r'^admin/'
-SESSION_SAVE_EVERY_REQUEST = True
 # Your common stuff: Below this line define 3rd party library settings
 CELERY_DEFAULT_RATE_LIMIT = 1
 CELERYD_TASK_TIME_LIMIT = 300
@@ -290,6 +290,7 @@ REST_FRAMEWORK = {
     'PAGE_SIZE': 25,
 
     'DEFAULT_AUTHENTICATION_CLASSES': (
+        'funkwhale_api.common.authentication.JSONWebTokenAuthenticationQS',
         'rest_framework_jwt.authentication.JSONWebTokenAuthentication',
         'rest_framework.authentication.SessionAuthentication',
         'rest_framework.authentication.BasicAuthentication',
@@ -299,9 +300,24 @@ REST_FRAMEWORK = {
     )
 }
 
-FUNKWHALE_PROVIDERS = {
-    'youtube': {
-        'api_key': env('YOUTUBE_API_KEY', default='REPLACE_ME')
-    }
-}
 ATOMIC_REQUESTS = False
+
+# Wether we should check user permission before serving audio files (meaning
+# return an obfuscated url)
+# This require a special configuration on the reverse proxy side
+# See https://wellfire.co/learn/nginx-django-x-accel-redirects/ for example
+PROTECT_AUDIO_FILES = env.bool('PROTECT_AUDIO_FILES', default=True)
+
+# Which path will be used to process the internal redirection
+# **DO NOT** put a slash at the end
+PROTECT_FILES_PATH = env('PROTECT_FILES_PATH', default='/_protected')
+
+
+# use this setting to tweak for how long you want to cache
+# musicbrainz results. (value is in seconds)
+MUSICBRAINZ_CACHE_DURATION = env.int(
+    'MUSICBRAINZ_CACHE_DURATION',
+    default=300
+)
+
+CACHALOT_ENABLED = env.bool('CACHALOT_ENABLED', default=True)

api/funkwhale_api/__init__.py

 # -*- coding: utf-8 -*-
-__version__ = '0.1.0'
+__version__ = '0.2.0'
 __version_info__ = tuple([int(num) if num.isdigit() else num for num in __version__.replace('-', '.', 1).split('.')])

api/funkwhale_api/common/authentication.py

+from rest_framework import exceptions
+from rest_framework_jwt import authentication
+from rest_framework_jwt.settings import api_settings
+
+
+class JSONWebTokenAuthenticationQS(
+        authentication.BaseJSONWebTokenAuthentication):
+
+    www_authenticate_realm = 'api'
+
+    def get_jwt_value(self, request):
+        token = request.query_params.get('jwt')
+        if 'jwt' in request.query_params and not token:
+            msg = _('Invalid Authorization header. No credentials provided.')
+            raise exceptions.AuthenticationFailed(msg)
+        return token
+
+    def authenticate_header(self, request):
+        return '{0} realm="{1}"'.format(
+            api_settings.JWT_AUTH_HEADER_PREFIX, self.www_authenticate_realm)

api/funkwhale_api/common/tests/test_jwt_querystring.py

+from test_plus.test import TestCase
+from rest_framework_jwt.settings import api_settings
+
+from funkwhale_api.users.models import User
+
+
+jwt_payload_handler = api_settings.JWT_PAYLOAD_HANDLER
+jwt_encode_handler = api_settings.JWT_ENCODE_HANDLER
+
+
+class TestJWTQueryString(TestCase):
+    www_authenticate_realm = 'api'
+
+    def test_can_authenticate_using_token_param_in_url(self):
+        user = User.objects.create_superuser(
+            username='test', email='test@test.com', password='test')
+
+        url = self.reverse('api:v1:tracks-list')
+        with self.settings(API_AUTHENTICATION_REQUIRED=True):
+            response = self.client.get(url)
+
+        self.assertEqual(response.status_code, 401)
+
+        payload = jwt_payload_handler(user)
+        token = jwt_encode_handler(payload)
+        print(payload, token)
+        with self.settings(API_AUTHENTICATION_REQUIRED=True):
+            response = self.client.get(url, data={
+                'jwt': token
+            })
+
+            self.assertEqual(response.status_code, 200)

api/funkwhale_api/music/metadata.py

@@ -37,13 +37,26 @@ def get_mp3_recording_id(f, k):
     except IndexError:
         raise TagNotFound(k)
 
+
+def convert_track_number(v):
+    try:
+        return int(v)
+    except ValueError:
+        # maybe the position is of the form "1/4"
+        pass
+
+    try:
+        return int(v.split('/')[0])
+    except (ValueError, AttributeError, IndexError):
+        pass
+
 CONF = {
     'OggVorbis': {
         'getter': lambda f, k: f[k][0],
         'fields': {
             'track_number': {
                 'field': 'TRACKNUMBER',
-                'to_application': int
+                'to_application': convert_track_number
             },
             'title': {
                 'field': 'title'
@@ -74,7 +87,7 @@ CONF = {
         'fields': {
             'track_number': {
                 'field': 'TPOS',
-                'to_application': lambda v: int(v.split('/')[0])
+                'to_application': convert_track_number
             },
             'title': {
                 'field': 'TIT2'

api/funkwhale_api/music/models.py

@@ -8,7 +8,6 @@ import markdown
 
 from django.conf import settings
 from django.db import models
-from django.contrib.staticfiles.templatetags.staticfiles import static
 from django.core.files.base import ContentFile
 from django.core.files import File
 from django.core.urlresolvers import reverse
@@ -354,10 +353,12 @@ class TrackFile(models.Model):
 
     @property
     def path(self):
-        if settings.USE_SAMPLE_TRACK:
-            return static('music/sample1.ogg')
+        if settings.PROTECT_AUDIO_FILES:
+            return reverse(
+                'api:v1:trackfiles-serve', kwargs={'pk': self.pk})
         return self.audio_file.url
 
+
 class ImportBatch(models.Model):
     creation_date = models.DateTimeField(default=timezone.now)
     submitted_by = models.ForeignKey('users.User', related_name='imports')

api/funkwhale_api/music/tests/factories.py

+import factory
+
+
+class ArtistFactory(factory.django.DjangoModelFactory):
+    name = factory.Sequence(lambda n: 'artist-{0}'.format(n))
+    mbid = factory.Faker('uuid4')
+
+    class Meta:
+        model = 'music.Artist'
+
+
+class AlbumFactory(factory.django.DjangoModelFactory):
+    title = factory.Sequence(lambda n: 'album-{0}'.format(n))
+    mbid = factory.Faker('uuid4')
+    release_date = factory.Faker('date')
+    cover = factory.django.ImageField()
+    artist = factory.SubFactory(ArtistFactory)
+
+    class Meta:
+        model = 'music.Album'
+
+
+class TrackFactory(factory.django.DjangoModelFactory):
+    title = factory.Sequence(lambda n: 'track-{0}'.format(n))
+    mbid = factory.Faker('uuid4')
+    album = factory.SubFactory(AlbumFactory)
+    artist = factory.SelfAttribute('album.artist')
+    position = 1
+
+    class Meta:
+        model = 'music.Track'
+
+
+class TrackFileFactory(factory.django.DjangoModelFactory):
+    track = factory.SubFactory(TrackFactory)
+    audio_file = factory.django.FileField()
+
+    class Meta:
+        model = 'music.TrackFile'

api/funkwhale_api/music/tests/test_api.py

@@ -10,6 +10,8 @@ from funkwhale_api.music import serializers
 from funkwhale_api.users.models import User
 
 from . import data as api_data
+from . import factories
+
 
 class TestAPI(TMPDirTestCaseMixin, TestCase):
 
@@ -214,3 +216,26 @@ class TestAPI(TMPDirTestCaseMixin, TestCase):
             with self.settings(API_AUTHENTICATION_REQUIRED=False):
                 response = getattr(self.client, method)(url)
             self.assertEqual(response.status_code, 200)
+
+    def test_track_file_url_is_restricted_to_authenticated_users(self):
+        f = factories.TrackFileFactory()
+        self.assertNotEqual(f.audio_file, None)
+        url = f.path
+
+        with self.settings(API_AUTHENTICATION_REQUIRED=True):
+            response = self.client.get(url)
+
+        self.assertEqual(response.status_code, 401)
+
+        user = User.objects.create_superuser(
+            username='test', email='test@test.com', password='test')
+        self.client.login(username=user.username, password='test')
+        with self.settings(API_AUTHENTICATION_REQUIRED=True):
+            response = self.client.get(url)
+
+        self.assertEqual(response.status_code, 200)
+
+        self.assertEqual(
+            response['X-Accel-Redirect'],
+            '/_protected{}'.format(f.audio_file.url)
+        )

api/funkwhale_api/music/views.py

 import os
 import json
+import unicodedata
+import urllib
 from django.core.urlresolvers import reverse
 from django.db import models, transaction
 from django.db.models.functions import Length
+from django.conf import settings
 from rest_framework import viewsets, views
 from rest_framework.decorators import detail_route, list_route
 from rest_framework.response import Response
@@ -51,6 +54,7 @@ class ArtistViewSet(SearchMixin, viewsets.ReadOnlyModelViewSet):
     search_fields = ['name']
     ordering_fields = ('creation_date',)
 
+
 class AlbumViewSet(SearchMixin, viewsets.ReadOnlyModelViewSet):
     queryset = (
         models.Album.objects.all()
@@ -63,6 +67,7 @@ class AlbumViewSet(SearchMixin, viewsets.ReadOnlyModelViewSet):
     search_fields = ['title']
     ordering_fields = ('creation_date',)
 
+
 class ImportBatchViewSet(viewsets.ReadOnlyModelViewSet):
     queryset = models.ImportBatch.objects.all().order_by('-creation_date')
     serializer_class = serializers.ImportBatchSerializer
@@ -70,6 +75,7 @@ class ImportBatchViewSet(viewsets.ReadOnlyModelViewSet):
     def get_queryset(self):
         return super().get_queryset().filter(submitted_by=self.request.user)
 
+
 class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet):
     """
     A simple ViewSet for viewing and editing accounts.
@@ -120,6 +126,29 @@ class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet):
         return Response(serializer.data)
 
 
+class TrackFileViewSet(viewsets.ReadOnlyModelViewSet):
+    queryset = (models.TrackFile.objects.all().order_by('-id'))
+    serializer_class = serializers.TrackFileSerializer
+    permission_classes = [ConditionalAuthentication]
+
+    @detail_route(methods=['get'])
+    def serve(self, request, *args, **kwargs):
+        try:
+            f = models.TrackFile.objects.get(pk=kwargs['pk'])
+        except models.TrackFile.DoesNotExist:
+            return Response(status=404)
+
+        response = Response()
+        filename = "filename*=UTF-8''{}{}".format(
+            urllib.parse.quote(f.track.full_name),
+            os.path.splitext(f.audio_file.name)[-1])
+        response["Content-Disposition"] = "attachment; {}".format(filename)
+        response['X-Accel-Redirect'] = "{}{}".format(
+            settings.PROTECT_FILES_PATH,
+            f.audio_file.url)
+        return response
+
+
 class TagViewSet(viewsets.ReadOnlyModelViewSet):
     queryset = Tag.objects.all().order_by('name')
     serializer_class = serializers.TagSerializer

api/funkwhale_api/musicbrainz/client.py

 import musicbrainzngs
+import memoize.djangocache
 
 from django.conf import settings
 from funkwhale_api import __version__
+
 _api = musicbrainzngs
 _api.set_useragent('funkwhale', str(__version__), 'contact@eliotberriot.com')
 
 
+store = memoize.djangocache.Cache('default')
+memo = memoize.Memoizer(store, namespace='memoize:musicbrainz')
+
+
 def clean_artist_search(query, **kwargs):
     cleaned_kwargs = {}
     if kwargs.get('name'):
@@ -17,30 +23,55 @@ class API(object):
     _api = _api
 
     class artists(object):
-        search = clean_artist_search
-        get = _api.get_artist_by_id
+        search = memo(
+            clean_artist_search, max_age=settings.MUSICBRAINZ_CACHE_DURATION)
+        get = memo(
+            _api.get_artist_by_id,
+            max_age=settings.MUSICBRAINZ_CACHE_DURATION)
 
     class images(object):
-        get_front = _api.get_image_front
+        get_front = memo(
+            _api.get_image_front,
+            max_age=settings.MUSICBRAINZ_CACHE_DURATION)
 
     class recordings(object):
-        search = _api.search_recordings
-        get = _api.get_recording_by_id
+        search = memo(
+            _api.search_recordings,
+            max_age=settings.MUSICBRAINZ_CACHE_DURATION)
+        get = memo(
+            _api.get_recording_by_id,
+            max_age=settings.MUSICBRAINZ_CACHE_DURATION)
 
     class works(object):
-        search = _api.