Merge branch 'release/0.12'

CHANGELOG

@@ -10,7 +10,155 @@ This changelog is viewable on the web at https://docs.funkwhale.audio/changelog.
 
 .. towncrier
 
-0.11 (unreleased)
+0.12 (2018-05-09)
+-----------------
+
+Upgrade instructions are available at
+  https://docs.funkwhale.audio/upgrading.html
+
+Features:
+
+- Subsonic API implementation to offer compatibility with existing clients such
+  as DSub (#75)
+- Use nodeinfo standard for publishing instance information (#192)
+
+
+Enhancements:
+
+- Play button now play tracks immediately instead of appending them to the
+  queue (#99, #156)
+
+
+Bugfixes:
+
+- Fix broken federated import (#193)
+
+
+Documentation:
+
+- Up-to-date documentation for upgrading front-end files on docker setup (#132)
+
+
+Subsonic API
+^^^^^^^^^^^^
+
+This release implements some core parts of the Subsonic API, which is widely
+deployed in various projects and supported by numerous clients.
+
+By offering this API in Funkwhale, we make it possible to access the instance
+library and listen to the music without from existing Subsonic clients, and
+without developping our own alternative clients for each and every platform.
+
+Most advanced Subsonic clients support offline caching of music files,
+playlist management and search, which makes them well-suited for nomadic use.
+
+Please head over :doc:`users/apps` for more informations about supported clients
+and user instructions.
+
+At the instance-level, the Subsonic API is enabled by default, but require
+and additional endpoint to be added in you reverse-proxy configuration.
+
+On nginx, add the following block::
+
+    location /rest/ {
+        include /etc/nginx/funkwhale_proxy.conf;
+        proxy_pass   http://funkwhale-api/api/subsonic/rest/;
+    }
+
+On Apache, add the following block::
+
+    <Location "/rest">
+        ProxyPass ${funkwhale-api}/api/subsonic/rest
+        ProxyPassReverse ${funkwhale-api}/api/subsonic/rest
+    </Location>
+
+The Subsonic can be disabled at the instance level from the django admin.
+
+.. note::
+
+    Because of Subsonic's API design which assumes cleartext storing of
+    user passwords, we chose to have a dedicated, separate password
+    for that purpose. Users can generate this password from their
+    settings page in the web client.
+
+
+Nodeinfo standard for instance information and stats
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. warning::
+
+    The ``/api/v1/instance/stats/`` endpoint which was used to display
+    instance data in the about page is removed in favor of the new
+    ``/api/v1/instance/nodeinfo/2.0/`` endpoint.
+
+In earlier version, we where using a custom endpoint and format for
+our instance information and statistics. While this was working,
+this was not compatible with anything else on the fediverse.
+
+We now offer a nodeinfo 2.0 endpoint which provides, in a single place,
+all the instance information such as library and user activity statistics,
+public instance settings (description, registration and federation status, etc.).
+
+We offer two settings to manage nodeinfo in your Funkwhale instance:
+
+1. One setting to completely disable nodeinfo, but this is not recommended
+   as the exposed data may be needed to make some parts of the front-end
+   work (especially the about page).
+2. One setting to disable only usage and library statistics in the nodeinfo
+   endpoint. This is useful if you want the nodeinfo endpoint to work,
+   but don't feel comfortable sharing aggregated statistics about your library
+   and user activity.
+
+To make your instance fully compatible with the nodeinfo protocol, you need to
+to edit your nginx configuration file:
+
+.. code-block::
+
+    # before
+    ...
+    location /.well-known/webfinger {
+        include /etc/nginx/funkwhale_proxy.conf;
+        proxy_pass   http://funkwhale-api/.well-known/webfinger;
+    }
+    ...
+
+    # after
+    ...
+    location /.well-known/ {
+        include /etc/nginx/funkwhale_proxy.conf;
+        proxy_pass   http://funkwhale-api/.well-known/;
+    }
+    ...
+
+You can do the same if you use apache:
+
+.. code-block::
+
+    # before
+    ...
+    <Location "/.well-known/webfinger">
+      ProxyPass ${funkwhale-api}/.well-known/webfinger
+      ProxyPassReverse ${funkwhale-api}/.well-known/webfinger
+    </Location>
+    ...
+
+    # after
+    ...
+    <Location "/.well-known/">
+      ProxyPass ${funkwhale-api}/.well-known/
+      ProxyPassReverse ${funkwhale-api}/.well-known/
+    </Location>
+    ...
+
+This will ensure all well-known endpoints are proxied to funkwhale, and
+not just webfinger one.
+
+Links:
+
+- About nodeinfo: https://github.com/jhass/nodeinfo
+
+
+0.11 (2018-05-06)
 -----------------
 
 Upgrade instructions are available at https://docs.funkwhale.audio/upgrading.html

api/config/api_urls.py

 from rest_framework import routers
+from rest_framework.urlpatterns import format_suffix_patterns
 from django.conf.urls import include, url
 from funkwhale_api.activity import views as activity_views
 from funkwhale_api.instance import views as instance_views
 from funkwhale_api.music import views
 from funkwhale_api.playlists import views as playlists_views
+from funkwhale_api.subsonic.views import SubsonicViewSet
 from rest_framework_jwt import views as jwt_views
 
 from dynamic_preferences.api.viewsets import GlobalPreferencesViewSet
@@ -27,6 +29,10 @@ router.register(
     'playlist-tracks')
 v1_patterns = router.urls
 
+subsonic_router = routers.SimpleRouter(trailing_slash=False)
+subsonic_router.register(r'subsonic/rest', SubsonicViewSet, base_name='subsonic')
+
+
 v1_patterns += [
     url(r'^instance/',
         include(
@@ -68,4 +74,4 @@ v1_patterns += [
 
 urlpatterns = [
     url(r'^v1/', include((v1_patterns, 'v1'), namespace='v1'))
-]
+] + format_suffix_patterns(subsonic_router.urls, allowed=['view'])

api/config/settings/common.py

@@ -133,6 +133,7 @@ LOCAL_APPS = (
     'funkwhale_api.providers.audiofile',
     'funkwhale_api.providers.youtube',
     'funkwhale_api.providers.acoustid',
+    'funkwhale_api.subsonic',
 )
 
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps

api/demo/demo-user.py

@@ -3,4 +3,5 @@ from funkwhale_api.users.models import User
 
 u = User.objects.create(email='demo@demo.com', username='demo', is_staff=True)
 u.set_password('demo')
+u.subsonic_api_token = 'demo'
 u.save()

api/funkwhale_api/__init__.py

 # -*- coding: utf-8 -*-
-__version__ = '0.11'
+__version__ = '0.12'
 __version_info__ = tuple([int(num) if num.isdigit() else num for num in __version__.replace('-', '.', 1).split('.')])

api/funkwhale_api/federation/views.py

@@ -85,13 +85,31 @@ class InstanceActorViewSet(FederationMixin, viewsets.GenericViewSet):
         return response.Response({}, status=200)
 
 
-class WellKnownViewSet(FederationMixin, viewsets.GenericViewSet):
+class WellKnownViewSet(viewsets.GenericViewSet):
     authentication_classes = []
     permission_classes = []
     renderer_classes = [renderers.WebfingerRenderer]
 
+    @list_route(methods=['get'])
+    def nodeinfo(self, request, *args, **kwargs):
+        if not preferences.get('instance__nodeinfo_enabled'):
+            return HttpResponse(status=404)
+        data = {
+            'links': [
+                {
+                    'rel': 'http://nodeinfo.diaspora.software/ns/schema/2.0',
+                    'href': utils.full_url(
+                        reverse('api:v1:instance:nodeinfo-2.0')
+                    )
+                }
+            ]
+        }
+        return response.Response(data)
+
     @list_route(methods=['get'])
     def webfinger(self, request, *args, **kwargs):
+        if not preferences.get('federation__enabled'):
+            return HttpResponse(status=405)
         try:
             resource_type, resource = webfinger.clean_resource(
                 request.GET['resource'])

api/funkwhale_api/instance/dynamic_preferences_registry.py

@@ -68,3 +68,31 @@ class RavenEnabled(types.BooleanPreference):
         'Wether error reporting to a Sentry instance using raven is enabled'
         ' for front-end errors'
     )
+
+
+@global_preferences_registry.register
+class InstanceNodeinfoEnabled(types.BooleanPreference):
+    show_in_api = False
+    section = instance
+    name = 'nodeinfo_enabled'
+    default = True
+    verbose_name = 'Enable nodeinfo endpoint'
+    help_text = (
+        'This endpoint is needed for your about page to work.'
+        'It\'s also helpful for the various monitoring '
+        'tools that map and analyzize the fediverse, '
+        'but you can disable it completely if needed.'
+    )
+
+
+@global_preferences_registry.register
+class InstanceNodeinfoStatsEnabled(types.BooleanPreference):
+    show_in_api = False
+    section = instance
+    name = 'nodeinfo_stats_enabled'
+    default = True
+    verbose_name = 'Enable usage and library stats in nodeinfo endpoint'
+    help_text = (
+        'Disable this f you don\'t want to share usage and library statistics'
+        'in the nodeinfo endpoint but don\'t want to disable it completely.'
+    )

api/funkwhale_api/instance/nodeinfo.py

+import memoize.djangocache
+
+import funkwhale_api
+from funkwhale_api.common import preferences
+
+from . import stats
+
+
+store = memoize.djangocache.Cache('default')
+memo = memoize.Memoizer(store, namespace='instance:stats')
+
+
+def get():
+    share_stats = preferences.get('instance__nodeinfo_stats_enabled')
+    data = {
+        'version': '2.0',
+        'software': {
+            'name': 'funkwhale',
+            'version': funkwhale_api.__version__
+        },
+        'protocols': ['activitypub'],
+        'services': {
+            'inbound': [],
+            'outbound': []
+        },
+        'openRegistrations': preferences.get('users__registration_enabled'),
+        'usage': {
+            'users': {
+                'total': 0,
+            }
+        },
+        'metadata': {
+            'shortDescription': preferences.get('instance__short_description'),
+            'longDescription': preferences.get('instance__long_description'),
+            'nodeName': preferences.get('instance__name'),
+            'library': {
+                'federationEnabled': preferences.get('federation__enabled'),
+                'federationNeedsApproval': preferences.get('federation__music_needs_approval'),
+                'anonymousCanListen': preferences.get('common__api_authentication_required'),
+            },
+        }
+    }
+    if share_stats:
+        getter = memo(
+            lambda: stats.get(),
+            max_age=600
+        )
+        statistics = getter()
+        data['usage']['users']['total'] = statistics['users']
+        data['metadata']['library']['tracks'] = {
+            'total': statistics['tracks'],
+        }
+        data['metadata']['library']['artists'] = {
+            'total': statistics['artists'],
+        }
+        data['metadata']['library']['albums'] = {
+            'total': statistics['albums'],
+        }
+        data['metadata']['library']['music'] = {
+            'hours': statistics['music_duration']
+        }
+
+        data['metadata']['usage'] = {
+            'favorites': {
+                'tracks': {
+                    'total': statistics['track_favorites'],
+                }
+            },
+            'listenings': {
+                'total': statistics['listenings']
+            }
+        }
+    return data

api/funkwhale_api/instance/urls.py

 from django.conf.urls import url
-from django.views.decorators.cache import cache_page
 
 from . import views
 
 
 urlpatterns = [
+    url(r'^nodeinfo/2.0/$', views.NodeInfo.as_view(), name='nodeinfo-2.0'),
     url(r'^settings/$', views.InstanceSettings.as_view(), name='settings'),
-    url(r'^stats/$',
-        cache_page(60 * 5)(views.InstanceStats.as_view()), name='stats'),
 ]

api/funkwhale_api/instance/views.py

@@ -4,9 +4,17 @@ from rest_framework.response import Response
 from dynamic_preferences.api import serializers
 from dynamic_preferences.registries import global_preferences_registry
 
+from funkwhale_api.common import preferences
+
+from . import nodeinfo
 from . import stats
 
 
+NODEINFO_2_CONTENT_TYPE = (
+    'application/json; profile=http://nodeinfo.diaspora.software/ns/schema/2.0#; charset=utf-8'  # noqa
+)
+
+
 class InstanceSettings(views.APIView):
     permission_classes = []
     authentication_classes = []
@@ -27,10 +35,13 @@ class InstanceSettings(views.APIView):
         return Response(data, status=200)
 
 
-class InstanceStats(views.APIView):
+class NodeInfo(views.APIView):
     permission_classes = []
     authentication_classes = []
 
     def get(self, request, *args, **kwargs):
-        data = stats.get()
-        return Response(data, status=200)
+        if not preferences.get('instance__nodeinfo_enabled'):
+            return Response(status=404)
+        data = nodeinfo.get()
+        return Response(
+            data, status=200, content_type=NODEINFO_2_CONTENT_TYPE)

api/funkwhale_api/music/factories.py

@@ -26,7 +26,7 @@ class ArtistFactory(factory.django.DjangoModelFactory):
 class AlbumFactory(factory.django.DjangoModelFactory):
     title = factory.Faker('sentence', nb_words=3)
     mbid = factory.Faker('uuid4')
-    release_date = factory.Faker('date')
+    release_date = factory.Faker('date_object')
     cover = factory.django.ImageField()
     artist = factory.SubFactory(ArtistFactory)
     release_group_id = factory.Faker('uuid4')

api/funkwhale_api/music/models.py

@@ -76,6 +76,11 @@ class APIModelMixin(models.Model):
                 self.musicbrainz_model, self.mbid)
 
 
+class ArtistQuerySet(models.QuerySet):
+    def with_albums_count(self):
+        return self.annotate(_albums_count=models.Count('albums'))
+
+
 class Artist(APIModelMixin):
     name = models.CharField(max_length=255)
 
@@ -89,6 +94,7 @@ class Artist(APIModelMixin):
         }
     }
     api = musicbrainz.api.artists
+    objects = ArtistQuerySet.as_manager()
 
     def __str__(self):
         return self.name
@@ -106,7 +112,7 @@ class Artist(APIModelMixin):
         kwargs.update({'name': name})
         return cls.objects.get_or_create(
             name__iexact=name,
-            defaults=kwargs)[0]
+            defaults=kwargs)
 
 
 def import_artist(v):
@@ -129,6 +135,11 @@ def import_tracks(instance, cleaned_data, raw_data):
         track = importers.load(Track, track_cleaned_data, track_data, Track.import_hooks)
 
 
+class AlbumQuerySet(models.QuerySet):
+    def with_tracks_count(self):
+        return self.annotate(_tracks_count=models.Count('tracks'))
+
+
 class Album(APIModelMixin):
     title = models.CharField(max_length=255)
     artist = models.ForeignKey(
@@ -173,6 +184,7 @@ class Album(APIModelMixin):
             'converter': import_artist,
         }
     }
+    objects = AlbumQuerySet.as_manager()
 
     def get_image(self):
         image_data =  musicbrainz.api.images.get_front(str(self.mbid))
@@ -196,7 +208,7 @@ class Album(APIModelMixin):
         kwargs.update({'title': title})
         return cls.objects.get_or_create(
             title__iexact=title,
-            defaults=kwargs)[0]
+            defaults=kwargs)
 
 
 def import_tags(instance, cleaned_data, raw_data):
@@ -403,7 +415,7 @@ class Track(APIModelMixin):
         kwargs.update({'title': title})
         return cls.objects.get_or_create(
             title__iexact=title,
-            defaults=kwargs)[0]
+            defaults=kwargs)
 
 
 class TrackFile(models.Model):
@@ -457,7 +469,13 @@ class TrackFile(models.Model):
     def filename(self):
         return '{}{}'.format(
             self.track.full_name,
-            os.path.splitext(self.audio_file.name)[-1])
+            self.extension)
+
+    @property
+    def extension(self):
+        if not self.audio_file:
+            return
+        return os.path.splitext(self.audio_file.name)[-1].replace('.', '', 1)
 
     def save(self, **kwargs):
         if not self.mimetype and self.audio_file:

api/funkwhale_api/music/tasks.py

@@ -39,7 +39,7 @@ def import_track_from_remote(library_track):
     except (KeyError, AssertionError):
         pass
     else:
-        return models.Track.get_or_create_from_api(mbid=track_mbid)
+        return models.Track.get_or_create_from_api(mbid=track_mbid)[0]
 
     try:
         album_mbid = metadata['release']['musicbrainz_id']
@@ -47,9 +47,9 @@ def import_track_from_remote(library_track):
     except (KeyError, AssertionError):
         pass
     else:
-        album = models.Album.get_or_create_from_api(mbid=album_mbid)
+        album, _ = models.Album.get_or_create_from_api(mbid=album_mbid)
         return models.Track.get_or_create_from_title(
-            library_track.title, artist=album.artist, album=album)
+            library_track.title, artist=album.artist, album=album)[0]
 
     try:
         artist_mbid = metadata['artist']['musicbrainz_id']
@@ -57,20 +57,20 @@ def import_track_from_remote(library_track):
     except (KeyError, AssertionError):
         pass
     else:
-        artist = models.Artist.get_or_create_from_api(mbid=artist_mbid)
-        album = models.Album.get_or_create_from_title(
+        artist, _ = models.Artist.get_or_create_from_api(mbid=artist_mbid)
+        album, _ = models.Album.get_or_create_from_title(
             library_track.album_title, artist=artist)
         return models.Track.get_or_create_from_title(
-            library_track.title, artist=artist, album=album)
+            library_track.title, artist=artist, album=album)[0]
 
     # worst case scenario, we have absolutely no way to link to a
     # musicbrainz resource, we rely on the name/titles
-    artist = models.Artist.get_or_create_from_name(
+    artist, _ = models.Artist.get_or_create_from_name(
         library_track.artist_name)
-    album = models.Album.get_or_create_from_title(
+    album, _ = models.Album.get_or_create_from_title(
         library_track.album_title, artist=artist)
     return models.Track.get_or_create_from_title(
-        library_track.title, artist=artist, album=album)
+        library_track.title, artist=artist, album=album)[0]
 
 
 def _do_import(import_job, replace=False, use_acoustid=True):

api/funkwhale_api/music/views.py

@@ -245,6 +245,53 @@ def get_file_path(audio_file):
         return path
 
 
+def handle_serve(track_file):
+    f = track_file
+    # we update the accessed_date
+    f.accessed_date = timezone.now()
+    f.save(update_fields=['accessed_date'])
+
+    mt = f.mimetype
+    audio_file = f.audio_file
+    try:
+        library_track = f.library_track
+    except ObjectDoesNotExist:
+        library_track = None
+    if library_track and not audio_file:
+        if not library_track.audio_file:
+            # we need to populate from cache
+            with transaction.atomic():
+                # why the transaction/select_for_update?
+                # this is because browsers may send multiple requests
+                # in a short time range, for partial content,
+                # thus resulting in multiple downloads from the remote
+                qs = LibraryTrack.objects.select_for_update()
+                library_track = qs.get(pk=library_track.pk)
+                library_track.download_audio()
+        audio_file = library_track.audio_file
+        file_path = get_file_path(audio_file)
+        mt = library_track.audio_mimetype
+    elif audio_file:
+        file_path = get_file_path(audio_file)
+    elif f.source and f.source.startswith('file://'):
+        file_path = get_file_path(f.source.replace('file://', '', 1))
+    response = Response()
+    filename = f.filename
+    mapping = {
+        'nginx': 'X-Accel-Redirect',
+        'apache2': 'X-Sendfile',
+    }
+    file_header = mapping[settings.REVERSE_PROXY_TYPE]
+    response[file_header] = file_path
+    filename = "filename*=UTF-8''{}".format(
+        urllib.parse.quote(filename))
+    response["Content-Disposition"] = "attachment; {}".format(filename)
+    if mt:
+        response["Content-Type"] = mt
+
+    return response
+
+
 class TrackFileViewSet(viewsets.ReadOnlyModelViewSet):
     queryset = (models.TrackFile.objects.all().order_by('-id'))
     serializer_class = serializers.TrackFileSerializer
@@ -261,54 +308,10 @@ class TrackFileViewSet(viewsets.ReadOnlyModelViewSet):
             'track__artist',
         )
         try:
-            f = queryset.get(pk=kwargs['pk'])
+            return handle_serve(queryset.get(pk=kwargs['pk']))
         except models.TrackFile.DoesNotExist:
             return Response(status=404)
 
-        # we update the accessed_date
-        f.accessed_date = timezone.now()
-        f.save(update_fields=['accessed_date'])
-
-        mt = f.mimetype
-        audio_file = f.audio_file
-        try:
-            library_track = f.library_track
-        except ObjectDoesNotExist:
-            library_track = None
-        if library_track and not audio_file:
-            if not library_track.audio_file:
-                # we need to populate from cache
-                with transaction.atomic():
-                    # why the transaction/select_for_update?
-                    # this is because browsers may send multiple requests
-                    # in a short time range, for partial content,
-                    # thus resulting in multiple downloads from the remote
-                    qs = LibraryTrack.objects.select_for_update()