Fixed #4: can now import artists and releases with a clean interface :party:

.env.dev

 BACKEND_URL=http://localhost:6001
-YOUTUBE_API_KEY=
 API_AUTHENTICATION_REQUIRED=True
+CACHALOT_ENABLED=False

api/config/api_urls.py

@@ -4,8 +4,11 @@ from funkwhale_api.music import views
 from funkwhale_api.playlists import views as playlists_views
 from rest_framework_jwt import views as jwt_views
 
+from dynamic_preferences.api.viewsets import GlobalPreferencesViewSet
+from dynamic_preferences.users.viewsets import UserPreferencesViewSet
 
 router = routers.SimpleRouter()
+router.register(r'settings', GlobalPreferencesViewSet, base_name='settings')
 router.register(r'tags', views.TagViewSet, 'tags')
 router.register(r'tracks', views.TrackViewSet, 'tracks')
 router.register(r'trackfiles', views.TrackFileViewSet, 'trackfiles')
@@ -14,17 +17,27 @@ router.register(r'albums', views.AlbumViewSet, 'albums')
 router.register(r'import-batches', views.ImportBatchViewSet, 'import-batches')
 router.register(r'submit', views.SubmitViewSet, 'submit')
 router.register(r'playlists', playlists_views.PlaylistViewSet, 'playlists')
-router.register(r'playlist-tracks', playlists_views.PlaylistTrackViewSet, 'playlist-tracks')
+router.register(
+    r'playlist-tracks',
+    playlists_views.PlaylistTrackViewSet,
+    'playlist-tracks')
 v1_patterns = router.urls
 
 v1_patterns += [
-    url(r'^providers/', include('funkwhale_api.providers.urls', namespace='providers')),
-    url(r'^favorites/', include('funkwhale_api.favorites.urls', namespace='favorites')),
-    url(r'^search$', views.Search.as_view(), name='search'),
-    url(r'^radios/', include('funkwhale_api.radios.urls', namespace='radios')),
-    url(r'^history/', include('funkwhale_api.history.urls', namespace='history')),
-    url(r'^users/', include('funkwhale_api.users.api_urls', namespace='users')),
-    url(r'^token/', jwt_views.obtain_jwt_token),
+    url(r'^providers/',
+        include('funkwhale_api.providers.urls', namespace='providers')),
+    url(r'^favorites/',
+        include('funkwhale_api.favorites.urls', namespace='favorites')),
+    url(r'^search$',
+        views.Search.as_view(), name='search'),
+    url(r'^radios/',
+        include('funkwhale_api.radios.urls', namespace='radios')),
+    url(r'^history/',
+        include('funkwhale_api.history.urls', namespace='history')),
+    url(r'^users/',
+        include('funkwhale_api.users.api_urls', namespace='users')),
+    url(r'^token/',
+        jwt_views.obtain_jwt_token),
     url(r'^token/refresh/', jwt_views.refresh_jwt_token),
 ]
 

api/config/settings/common.py

@@ -53,6 +53,7 @@ THIRD_PARTY_APPS = (
     'rest_auth',
     'rest_auth.registration',
     'mptt',
+    'dynamic_preferences',
 )
 
 # Apps specific for this project go here.
@@ -65,6 +66,7 @@ LOCAL_APPS = (
     'funkwhale_api.history',
     'funkwhale_api.playlists',
     'funkwhale_api.providers.audiofile',
+    'funkwhale_api.providers.youtube',
 )
 
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
@@ -298,11 +300,6 @@ REST_FRAMEWORK = {
     )
 }
 
-FUNKWHALE_PROVIDERS = {
-    'youtube': {
-        'api_key': env('YOUTUBE_API_KEY', default='REPLACE_ME')
-    }
-}
 ATOMIC_REQUESTS = False
 
 # Wether we should check user permission before serving audio files (meaning
@@ -314,3 +311,13 @@ PROTECT_AUDIO_FILES = env.bool('PROTECT_AUDIO_FILES', default=True)
 # Which path will be used to process the internal redirection
 # **DO NOT** put a slash at the end
 PROTECT_FILES_PATH = env('PROTECT_FILES_PATH', default='/_protected')
+
+
+# use this setting to tweak for how long you want to cache
+# musicbrainz results. (value is in seconds)
+MUSICBRAINZ_CACHE_DURATION = env.int(
+    'MUSICBRAINZ_CACHE_DURATION',
+    default=300
+)
+
+CACHALOT_ENABLED = env.bool('CACHALOT_ENABLED', default=True)

api/funkwhale_api/music/metadata.py

@@ -37,13 +37,26 @@ def get_mp3_recording_id(f, k):
     except IndexError:
         raise TagNotFound(k)
 
+
+def convert_track_number(v):
+    try:
+        return int(v)
+    except ValueError:
+        # maybe the position is of the form "1/4"
+        pass
+
+    try:
+        return int(v.split('/')[0])
+    except (ValueError, AttributeError, IndexError):
+        pass
+
 CONF = {
     'OggVorbis': {
         'getter': lambda f, k: f[k][0],
         'fields': {
             'track_number': {
                 'field': 'TRACKNUMBER',
-                'to_application': int
+                'to_application': convert_track_number
             },
             'title': {
                 'field': 'title'
@@ -74,7 +87,7 @@ CONF = {
         'fields': {
             'track_number': {
                 'field': 'TPOS',
-                'to_application': lambda v: int(v.split('/')[0])
+                'to_application': convert_track_number
             },
             'title': {
                 'field': 'TIT2'

api/funkwhale_api/musicbrainz/client.py

 import musicbrainzngs
+import memoize.djangocache
 
 from django.conf import settings
 from funkwhale_api import __version__
+
 _api = musicbrainzngs
 _api.set_useragent('funkwhale', str(__version__), 'contact@eliotberriot.com')
 
 
+store = memoize.djangocache.Cache('default')
+memo = memoize.Memoizer(store, namespace='memoize:musicbrainz')
+
+
 def clean_artist_search(query, **kwargs):
     cleaned_kwargs = {}
     if kwargs.get('name'):
@@ -17,30 +23,55 @@ class API(object):
     _api = _api
 
     class artists(object):
-        search = clean_artist_search
-        get = _api.get_artist_by_id
+        search = memo(
+            clean_artist_search, max_age=settings.MUSICBRAINZ_CACHE_DURATION)
+        get = memo(
+            _api.get_artist_by_id,
+            max_age=settings.MUSICBRAINZ_CACHE_DURATION)
 
     class images(object):
-        get_front = _api.get_image_front
+        get_front = memo(
+            _api.get_image_front,
+            max_age=settings.MUSICBRAINZ_CACHE_DURATION)
 
     class recordings(object):
-        search = _api.search_recordings
-        get = _api.get_recording_by_id
+        search = memo(
+            _api.search_recordings,
+            max_age=settings.MUSICBRAINZ_CACHE_DURATION)
+        get = memo(
+            _api.get_recording_by_id,
+            max_age=settings.MUSICBRAINZ_CACHE_DURATION)
 
     class works(object):
-        search = _api.search_works
-        get = _api.get_work_by_id
+        search = memo(
+            _api.search_works,
+            max_age=settings.MUSICBRAINZ_CACHE_DURATION)
+        get = memo(
+            _api.get_work_by_id,
+            max_age=settings.MUSICBRAINZ_CACHE_DURATION)
 
     class releases(object):
-        search = _api.search_releases
-        get = _api.get_release_by_id
-        browse = _api.browse_releases
+        search = memo(
+            _api.search_releases,
+            max_age=settings.MUSICBRAINZ_CACHE_DURATION)
+        get = memo(
+            _api.get_release_by_id,
+            max_age=settings.MUSICBRAINZ_CACHE_DURATION)
+        browse = memo(
+            _api.browse_releases,
+            max_age=settings.MUSICBRAINZ_CACHE_DURATION)
         # get_image_front = _api.get_image_front
 
     class release_groups(object):
-        search = _api.search_release_groups
-        get = _api.get_release_group_by_id
-        browse = _api.browse_release_groups
+        search = memo(
+            _api.search_release_groups,
+            max_age=settings.MUSICBRAINZ_CACHE_DURATION)
+        get = memo(
+            _api.get_release_group_by_id,
+            max_age=settings.MUSICBRAINZ_CACHE_DURATION)
+        browse = memo(
+            _api.browse_release_groups,
+            max_age=settings.MUSICBRAINZ_CACHE_DURATION)
         # get_image_front = _api.get_image_front
 
 api = API()

api/funkwhale_api/musicbrainz/tests/test_cache.py

+import unittest
+from test_plus.test import TestCase
+
+from funkwhale_api.musicbrainz import client
+
+
+class TestAPI(TestCase):
+    def test_can_search_recording_in_musicbrainz_api(self, *mocks):
+        r = {'hello': 'world'}
+        mocked = 'funkwhale_api.musicbrainz.client._api.search_artists'
+        with unittest.mock.patch(mocked, return_value=r) as m:
+            self.assertEqual(client.api.artists.search('test'), r)
+            # now call from cache
+            self.assertEqual(client.api.artists.search('test'), r)
+            self.assertEqual(client.api.artists.search('test'), r)
+
+            self.assertEqual(m.call_count, 1)

api/funkwhale_api/musicbrainz/views.py

@@ -2,6 +2,7 @@ from rest_framework import viewsets
 from rest_framework.views import APIView
 from rest_framework.response import Response
 from rest_framework.decorators import list_route
+import musicbrainzngs
 
 from funkwhale_api.common.permissions import ConditionalAuthentication
 
@@ -44,7 +45,7 @@ class ReleaseBrowse(APIView):
     def get(self, request, *args, **kwargs):
         result = api.releases.browse(
             release_group=kwargs['release_group_uuid'],
-            includes=['recordings'])
+            includes=['recordings', 'artist-credits'])
         return Response(result)
 
 
@@ -54,17 +55,18 @@ class SearchViewSet(viewsets.ViewSet):
     @list_route(methods=['get'])
     def recordings(self, request, *args, **kwargs):
         query = request.GET['query']
-        results = api.recordings.search(query, artist=query)
+        results = api.recordings.search(query)
         return Response(results)
 
     @list_route(methods=['get'])
     def releases(self, request, *args, **kwargs):
         query = request.GET['query']
-        results = api.releases.search(query, artist=query)
+        results = api.releases.search(query)
         return Response(results)
 
     @list_route(methods=['get'])
     def artists(self, request, *args, **kwargs):
         query = request.GET['query']
         results = api.artists.search(query)
+        # results = musicbrainzngs.search_artists(query)
         return Response(results)

api/funkwhale_api/providers/youtube/client.py

@@ -4,21 +4,20 @@ from apiclient.discovery import build
 from apiclient.errors import HttpError
 from oauth2client.tools import argparser
 
-from django.conf import settings
+from dynamic_preferences.registries import (
+    global_preferences_registry as registry)
 
-# Set DEVELOPER_KEY to the API key value from the APIs & auth > Registered apps
-# tab of
-#   https://cloud.google.com/console
-# Please ensure that you have enabled the YouTube Data API for your project.
-DEVELOPER_KEY = settings.FUNKWHALE_PROVIDERS['youtube']['api_key']
 YOUTUBE_API_SERVICE_NAME = "youtube"
 YOUTUBE_API_VERSION = "v3"
 VIDEO_BASE_URL = 'https://www.youtube.com/watch?v={0}'
 
 
 def _do_search(query):
-    youtube = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION,
-    developerKey=DEVELOPER_KEY)
+    manager = registry.manager()
+    youtube = build(
+        YOUTUBE_API_SERVICE_NAME,
+        YOUTUBE_API_VERSION,
+        developerKey=manager['providers_youtube__api_key'])
 
     return youtube.search().list(
         q=query,
@@ -55,4 +54,33 @@ class Client(object):
 
         return results
 
+    def to_funkwhale(self, result):
+        """
+        We convert youtube results to something more generic.
+
+        {
+            "id": "video id",
+            "type": "youtube#video",
+            "url": "https://www.youtube.com/watch?v=id",
+            "description": "description",
+            "channelId": "Channel id",
+            "title": "Title",
+            "channelTitle": "channel Title",
+            "publishedAt": "2012-08-22T18:41:03.000Z",
+            "cover": "http://coverurl"
+        }
+        """
+        return {
+            'id': result['id']['videoId'],
+            'url': 'https://www.youtube.com/watch?v={}'.format(
+                result['id']['videoId']),
+            'type': result['id']['kind'],
+            'title': result['snippet']['title'],
+            'description': result['snippet']['description'],
+            'channelId': result['snippet']['channelId'],
+            'channelTitle': result['snippet']['channelTitle'],
+            'publishedAt': result['snippet']['publishedAt'],
+            'cover': result['snippet']['thumbnails']['high']['url'],
+        }
+
 client = Client()

api/funkwhale_api/providers/youtube/dynamic_preferences_registry.py

+from dynamic_preferences.types import StringPreference, Section
+from dynamic_preferences.registries import global_preferences_registry
+
+youtube = Section('providers_youtube')
+
+
+@global_preferences_registry.register
+class APIKey(StringPreference):
+    section = youtube
+    name = 'api_key'
+    default = 'CHANGEME'
+    verbose_name = 'YouTube API key'
+    help_text = 'The API key used to query YouTube. Get one at https://console.developers.google.com/.'

api/funkwhale_api/providers/youtube/tests/test_youtube.py

@@ -8,7 +8,7 @@ from funkwhale_api.providers.youtube.client import client
 from . import data as api_data
 
 class TestAPI(TestCase):
-
+    maxDiff = None
     @unittest.mock.patch(
         'funkwhale_api.providers.youtube.client._do_search',
         return_value=api_data.search['8 bit adventure'])
@@ -25,11 +25,23 @@ class TestAPI(TestCase):
         return_value=api_data.search['8 bit adventure'])
     def test_can_get_search_results_from_funkwhale(self, *mocks):
         query = '8 bit adventure'
-        expected = json.dumps(client.search(query))
         url = self.reverse('api:v1:providers:youtube:search')
         response = self.client.get(url + '?query={0}'.format(query))
+        # we should cast the youtube result to something more generic
+        expected = {
+            "id": "0HxZn6CzOIo",
+            "url": "https://www.youtube.com/watch?v=0HxZn6CzOIo",
+            "type": "youtube#video",
+            "description": "Make sure to apply adhesive evenly before use. GET IT HERE: http://adhesivewombat.bandcamp.com/album/marsupial-madness Facebook: ...",
+            "channelId": "UCps63j3krzAG4OyXeEyuhFw",
+            "title": "AdhesiveWombat - 8 Bit Adventure",
+            "channelTitle": "AdhesiveWombat",
+            "publishedAt": "2012-08-22T18:41:03.000Z",
+            "cover": "https://i.ytimg.com/vi/0HxZn6CzOIo/hqdefault.jpg"
+        }
 
-        self.assertJSONEqual(expected, json.loads(response.content.decode('utf-8')))
+        self.assertEqual(
+            json.loads(response.content.decode('utf-8'))[0], expected)
 
     @unittest.mock.patch(
         'funkwhale_api.providers.youtube.client._do_search',
@@ -66,9 +78,22 @@ class TestAPI(TestCase):
             'q': '8 bit adventure',
         }
 
-        expected = json.dumps(client.search_multiple(queries))
+        expected = {
+            "id": "0HxZn6CzOIo",
+            "url": "https://www.youtube.com/watch?v=0HxZn6CzOIo",
+            "type": "youtube#video",
+            "description": "Make sure to apply adhesive evenly before use. GET IT HERE: http://adhesivewombat.bandcamp.com/album/marsupial-madness Facebook: ...",
+            "channelId": "UCps63j3krzAG4OyXeEyuhFw",
+            "title": "AdhesiveWombat - 8 Bit Adventure",
+            "channelTitle": "AdhesiveWombat",
+            "publishedAt": "2012-08-22T18:41:03.000Z",
+            "cover": "https://i.ytimg.com/vi/0HxZn6CzOIo/hqdefault.jpg"
+        }
+
         url = self.reverse('api:v1:providers:youtube:searchs')
         response = self.client.post(
             url, json.dumps(queries), content_type='application/json')
 
-        self.assertJSONEqual(expected, json.loads(response.content.decode('utf-8')))
+        self.assertEqual(
+            expected,
+            json.loads(response.content.decode('utf-8'))['1'][0])

api/funkwhale_api/providers/youtube/views.py

@@ -10,7 +10,10 @@ class APISearch(APIView):
 
     def get(self, request, *args, **kwargs):
         results = client.search(request.GET['query'])
-        return Response(results)
+        return Response([
+            client.to_funkwhale(result)
+            for result in results
+        ])
 
 
 class APISearchs(APIView):
@@ -18,4 +21,10 @@ class APISearchs(APIView):
 
     def post(self, request, *args, **kwargs):
         results = client.search_multiple(request.data)
-        return Response(results)
+        return Response({
+            key: [
+                client.to_funkwhale(result)
+                for result in group
+            ]
+            for key, group in results.items()
+        })

api/funkwhale_api/users/models.py

@@ -19,8 +19,11 @@ class User(AbstractUser):
     relevant_permissions = {
         # internal_codename : {external_codename}
         'music.add_importbatch': {
-            'external_codename': 'import.launch'
-        }
+            'external_codename': 'import.launch',
+        },
+        'dynamic_preferences.change_globalpreferencemodel': {
+            'external_codename': 'settings.change',
+        },
     }
 
     def __str__(self):

api/funkwhale_api/users/tests/test_views.py

@@ -47,7 +47,13 @@ class UserTestCase(TestCase):
         # login required
         self.assertEqual(response.status_code, 401)
 
-        user = UserFactory(is_staff=True, perms=['music.add_importbatch'])
+        user = UserFactory(
+            is_staff=True,
+            perms=[
+                'music.add_importbatch',
+                'dynamic_preferences.change_globalpreferencemodel',
+            ]
+        )
         self.assertTrue(user.has_perm('music.add_importbatch'))
         self.login(user)
 
@@ -63,3 +69,5 @@ class UserTestCase(TestCase):
         self.assertEqual(payload['name'], user.name)
         self.assertEqual(
             payload['permissions']['import.launch']['status'], True)
+        self.assertEqual(
+            payload['permissions']['settings.change']['status'], True)

api/requirements/base.txt

@@ -50,3 +50,9 @@ beautifulsoup4==4.6.0
 Markdown==2.6.8
 ipython==6.1.0
 mutagen==1.38
+
+
+# Until this is merged
+git+https://github.com/EliotBerriot/PyMemoize.git@django
+
+django-dynamic-preferences>=1.2,<1.3

dev.yml

@@ -27,7 +27,7 @@ services:
     env_file: .env.dev
     build:
       context: ./api
-      dockerfile: docker/Dockerfile.local
+      dockerfile: docker/Dockerfile.test
     links:
      - postgres
      - redis

front/src/audio/index.js

 import logger from '@/logging'
-
-const pad = (val) => {
-  val = Math.floor(val)
-  if (val < 10) {
-    return '0' + val
-  }
-  return val + ''
-}
+import time from '@/utils/time'
 
 const Cov = {
   on (el, type, func) {
@@ -108,7 +101,7 @@ class Audio {
     })
     this.state.duration = Math.round(this.$Audio.duration * 100) / 100
     this.state.loaded = Math.round(10000 * this.$Audio.buffered.end(0) / this.$Audio.duration) / 100
-    this.state.durationTimerFormat = this.timeParse(this.state.duration)
+    this.state.durationTimerFormat = time.parse(this.state.duration)
   }
 
   updatePlayState (e) {
@@ -116,9 +109,9 @@ class Audio {
     this.state.duration = Math.round(this.$Audio.duration * 100) / 100
     this.state.progress = Math.round(10000 * this.state.currentTime / this.state.duration) / 100
 
-    this.state.durationTimerFormat = this.timeParse(this.state.duration)
-    this.state.currentTimeFormat = this.timeParse(this.state.currentTime)
-    this.state.lastTimeFormat = this.timeParse(this.state.duration - this.state.currentTime)
+    this.state.durationTimerFormat = time.parse(this.state.duration)
+    this.state.currentTimeFormat = time.parse(this.state.currentTime)
+    this.state.lastTimeFormat = time.parse(this.state.duration - this.state.currentTime)
 
     this.hook.playState.forEach(func => {
       func(this.state)
@@ -181,14 +174,6 @@ class Audio {
     }
     this.$Audio.currentTime = time
   }
-
-  timeParse (sec) {
-    let min = 0
-    min = Math.floor(sec / 60)
-    sec = sec - min * 60
-    return pad(min) + ':' + pad(sec)
-  }
-
 }
 
 export default Audio

front/src/auth/index.js

@@ -10,14 +10,13 @@ const LOGIN_URL = config.API_URL + 'token/'
 const USER_PROFILE_URL = config.API_URL + 'users/users/me/'
 // const SIGNUP_URL = API_URL + 'users/'
 
-export default {
-
-  // User object will let us check authentication status
-  user: {
-    authenticated: false,
-    username: '',
-    profile: null
-  },
+let userData = {
+  authenticated: false,
+  username: '',
+  availablePermissions: {},
+  profile: {}
+}
+let auth = {
 
   // Send a request to the login URL and save the returned JWT
   login (context, creds, redirect, onError) {
@@ -87,7 +86,14 @@ export default {
     let self = this
     this.fetchProfile().then(data => {
       Vue.set(self.user, 'profile', data)
+      Object.keys(data.permissions).forEach(function (key) {
+        // this makes it easier to check for permissions in templates
+        Vue.set(self.user.availablePermissions, key, data.permissions[String(key)].status)
+      })
     })
     favoriteTracks.fetch()
   }
 }
+
+Vue.set(auth, 'user', userData)
+export default auth

front/src/components/Home.vue

@@ -6,7 +6,7 @@
             Welcome on funkwhale
         </h1>
         <p>We think listening music should be simple.</p>
-        <router-link class="ui icon teal button" to="/browse">
+        <router-link class="ui icon teal button" to="/library">
           Get me to the library
           <i class="right arrow icon"></i>
         </router-link>
@@ -90,9 +90,9 @@
         <p>Funkwhale is dead simple to use.</p>
         <div class="ui list">
           <div class="item">
-            <i class="browser icon"></i>
+            <i class="libraryr icon"></i>
             <div class="content">
-              No add-ons, no plugins : you only need a web browser
+              No add-ons, no plugins : you only need a web libraryr
             </div>
           </div>
           <div class="item">

front/src/components/Sidebar.vue

@@ -13,7 +13,7 @@