Merge branch 'release/0.8'

.env.dev

 API_AUTHENTICATION_REQUIRED=True
 RAVEN_ENABLED=false
 RAVEN_DSN=https://44332e9fdd3d42879c7d35bf8562c6a4:0062dc16a22b41679cd5765e5342f716@sentry.eliotberriot.com/5
+DJANGO_ALLOWED_HOSTS=localhost,nginx
+DJANGO_SETTINGS_MODULE=config.settings.local
+DJANGO_SECRET_KEY=dev
+C_FORCE_ROOT=true
+FUNKWHALE_URL=http://localhost
+PYTHONDONTWRITEBYTECODE=true

.gitignore

@@ -86,3 +86,4 @@ front/selenium-debug.log
 docs/_build
 
 data/
+.env

.gitlab-ci.yml

@@ -13,6 +13,7 @@ stages:
 test_api:
   services:
     - postgres:9.4
+    - redis:3
   stage: test
   image: funkwhale/funkwhale:latest
   cache:
@@ -24,6 +25,7 @@ test_api:
     DATABASE_URL: "postgresql://postgres@postgres/postgres"
     FUNKWHALE_URL: "https://funkwhale.ci"
     CACHEOPS_ENABLED: "false"
+    DJANGO_SETTINGS_MODULE: config.settings.local
 
   before_script:
     - cd api
@@ -31,7 +33,7 @@ test_api:
     - pip install -r requirements/local.txt
     - pip install -r requirements/test.txt
   script:
-    - pytest
+    - pytest --cov=funkwhale_api tests/
   tags:
     - docker
 

CHANGELOG

@@ -3,6 +3,100 @@ Changelog
 
 .. towncrier
 
+0.8 (2018-04-02)
+----------------
+
+Features:
+
+- Add a detail page for radios (#64)
+- Implemented page title binding (#1)
+- Previous Track button restart playback after 3 seconds (#146)
+
+
+Enhancements:
+
+- Added credits to Francis Gading for the logotype (#101)
+- API endpoint for fetching instance activity and updated timeline to use this
+  new endpoint (#141)
+- Better error messages in case of missing environment variables (#140)
+- Implemented a @test@yourfunkwhaledomain bot to ensure federation works
+  properly. Send it "/ping" and it will answer back :)
+- Queue shuffle now apply only to tracks after the current one (#97)
+- Removed player from queue tab and consistently show current track in queue
+  (#131)
+- We now restrict some usernames from being used during signup (#139)
+
+
+Bugfixes:
+
+- Better error handling during file import (#120)
+- Better handling of utf-8 filenames during file import (#138)
+- Converted favicon from .ico to .png (#130)
+- Upgraded to Python 3.6 to fix weird but harmless weakref error on django task
+  (#121)
+
+
+Documentation:
+
+- Documented the upgrade process (#127)
+
+
+Preparing for federation
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+Federation of music libraries is one of the most asked feature.
+While there is still a lot of work to do, this version includes
+the foundation that will enable funkwhale servers to communicate
+between each others, and with other federated software, such as
+Mastodon.
+
+Funkwhale will use ActivityPub as it's federation protocol.
+
+In order to prepare for federation (see #136 and #137), new API endpoints
+have been added under /federation and /.well-known/webfinger.
+
+For these endpoints to work, you will need to update your nginx configuration,
+and add the following snippets::
+
+    location /federation/ {
+        include /etc/nginx/funkwhale_proxy.conf;
+        proxy_pass   http://funkwhale-api/federation/;
+    }
+
+    location /.well-known/webfinger {
+        include /etc/nginx/funkwhale_proxy.conf;
+        proxy_pass   http://funkwhale-api/.well-known/webfinger;
+    }
+
+This will ensure federation endpoints will be reachable in the future.
+You can of course skip this part if you know you will not federate your instance.
+
+A new ``FEDERATION_ENABLED`` env var have also been added to control wether
+federation is enabled or not on the application side. This settings defaults
+to True, which should have no consequencies at the moment, since actual
+federation is not implemented and the only available endpoints are for
+testing purposes.
+
+Add ``FEDERATION_ENABLED=false`` to your .env file to disable federation
+on the application side.
+
+The last step involves generating RSA private and public keys for signing
+your instance requests on the federation. This can be done via::
+
+    # on docker setups
+    docker-compose run --rm api python manage.py generate_keys --no-input
+
+    # on non-docker setups
+    source /srv/funkwhale/virtualenv/bin/activate
+    source /srv/funkwhale/load_env
+    python manage.py generate_keys --no-input
+
+To test and troobleshoot federation, we've added a bot account. This bot is available at @test@yourinstancedomain,
+and sending it "/ping", for example, via Mastodon, should trigger
+a response.
+
+
+
 0.7 (2018-03-21)
 ----------------
 

README.rst

@@ -73,6 +73,19 @@ via the following command::
     docker-compose -f dev.yml build
 
 
+Creating your env file
+^^^^^^^^^^^^^^^^^^^^^^
+
+We provide a working .env.dev configuration file that is suitable for
+development. However, to enable customization on your machine, you should
+also create a .env file that will hold your personal environment
+variables (those will not be commited to the project).
+
+Create it like this::
+
+    touch .env
+
+
 Database management
 ^^^^^^^^^^^^^^^^^^^
 

api/Dockerfile

-FROM python:3.5
+FROM python:3.6
 
 ENV PYTHONUNBUFFERED 1
 

api/compose/django/dev-entrypoint.sh

 #!/bin/bash
 set -e
-if [ $1 = "pytest" ]; then
-    # let pytest.ini handle it
-    unset DJANGO_SETTINGS_MODULE
-fi
 exec "$@"

api/config/api_urls.py

 from rest_framework import routers
 from django.conf.urls import include, url
+from funkwhale_api.activity import views as activity_views
 from funkwhale_api.instance import views as instance_views
 from funkwhale_api.music import views
 from funkwhale_api.playlists import views as playlists_views
@@ -10,6 +11,7 @@ from dynamic_preferences.users.viewsets import UserPreferencesViewSet
 
 router = routers.SimpleRouter()
 router.register(r'settings', GlobalPreferencesViewSet, base_name='settings')
+router.register(r'activity', activity_views.ActivityViewSet, 'activity')
 router.register(r'tags', views.TagViewSet, 'tags')
 router.register(r'tracks', views.TrackViewSet, 'tracks')
 router.register(r'trackfiles', views.TrackFileViewSet, 'trackfiles')

api/config/settings/common.py

@@ -10,6 +10,7 @@ https://docs.djangoproject.com/en/dev/ref/settings/
 """
 from __future__ import absolute_import, unicode_literals
 
+from urllib.parse import urlsplit
 import os
 import environ
 from funkwhale_api import __version__
@@ -24,8 +25,13 @@ try:
 except FileNotFoundError:
     pass
 
-ALLOWED_HOSTS = env.list('DJANGO_ALLOWED_HOSTS')
 FUNKWHALE_URL = env('FUNKWHALE_URL')
+FUNKWHALE_HOSTNAME = urlsplit(FUNKWHALE_URL).netloc
+
+FEDERATION_ENABLED = env.bool('FEDERATION_ENABLED', default=True)
+FEDERATION_HOSTNAME = env('FEDERATION_HOSTNAME', default=FUNKWHALE_HOSTNAME)
+
+ALLOWED_HOSTS = env.list('DJANGO_ALLOWED_HOSTS')
 
 # APP CONFIGURATION
 # ------------------------------------------------------------------------------
@@ -89,6 +95,7 @@ LOCAL_APPS = (
     'funkwhale_api.music',
     'funkwhale_api.requests',
     'funkwhale_api.favorites',
+    'funkwhale_api.federation',
     'funkwhale_api.radios',
     'funkwhale_api.history',
     'funkwhale_api.playlists',
@@ -231,6 +238,7 @@ STATIC_ROOT = env("STATIC_ROOT", default=str(ROOT_DIR('staticfiles')))
 
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#static-url
 STATIC_URL = env("STATIC_URL", default='/staticfiles/')
+DEFAULT_FILE_STORAGE = 'funkwhale_api.common.storage.ASCIIFileSystemStorage'
 
 # See: https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#std:setting-STATICFILES_DIRS
 STATICFILES_DIRS = (
@@ -336,7 +344,12 @@ REST_FRAMEWORK = {
     ),
     'DEFAULT_PAGINATION_CLASS': 'funkwhale_api.common.pagination.FunkwhalePagination',
     'PAGE_SIZE': 25,
-
+    'DEFAULT_PARSER_CLASSES': (
+        'rest_framework.parsers.JSONParser',
+        'rest_framework.parsers.FormParser',
+        'rest_framework.parsers.MultiPartParser',
+        'funkwhale_api.federation.parsers.ActivityParser',
+    ),
     'DEFAULT_AUTHENTICATION_CLASSES': (
         'funkwhale_api.common.authentication.JSONWebTokenAuthenticationQS',
         'rest_framework_jwt.authentication.JSONWebTokenAuthentication',
@@ -385,3 +398,16 @@ CSRF_USE_SESSIONS = True
 
 # Playlist settings
 PLAYLISTS_MAX_TRACKS = env.int('PLAYLISTS_MAX_TRACKS', default=250)
+
+ACCOUNT_USERNAME_BLACKLIST = [
+    'funkwhale',
+    'library',
+    'test',
+    'status',
+    'root',
+    'admin',
+    'owner',
+    'superuser',
+    'staff',
+    'service',
+] + env.list('ACCOUNT_USERNAME_BLACKLIST', default=[])

api/config/settings/local.py

@@ -72,6 +72,10 @@ LOGGING = {
             'handlers':['console'],
             'propagate': True,
             'level':'DEBUG',
-        }
+        },
+        '': {
+            'level': 'DEBUG',
+            'handlers': ['console'],
+        },
     },
 }

api/config/settings/test.py

-from .common import *  # noqa
-SECRET_KEY = env("DJANGO_SECRET_KEY", default='test')
-
-# Mail settings
-# ------------------------------------------------------------------------------
-EMAIL_HOST = 'localhost'
-EMAIL_PORT = 1025
-EMAIL_BACKEND = env('DJANGO_EMAIL_BACKEND',
-                    default='django.core.mail.backends.console.EmailBackend')
-
-# CACHING
-# ------------------------------------------------------------------------------
-CACHES = {
-    'default': {
-        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
-        'LOCATION': ''
-    }
-}
-
-CELERY_BROKER_URL = 'memory://'
-
-########## CELERY
-# In development, all tasks will be executed locally by blocking until the task returns
-CELERY_TASK_ALWAYS_EAGER = True
-########## END CELERY
-
-# Your local stuff: Below this line define 3rd party library settings
-API_AUTHENTICATION_REQUIRED = False
-CACHEOPS_ENABLED = False

api/config/urls.py

@@ -13,6 +13,9 @@ urlpatterns = [
     url(settings.ADMIN_URL, admin.site.urls),
 
     url(r'^api/', include(("config.api_urls", 'api'), namespace="api")),
+    url(r'^', include(
+        ('funkwhale_api.federation.urls', 'federation'),
+        namespace="federation")),
     url(r'^api/v1/auth/', include('rest_auth.urls')),
     url(r'^api/v1/auth/registration/', include('funkwhale_api.users.rest_auth_urls')),
     url(r'^accounts/', include('allauth.urls')),

api/docker/Dockerfile.test

-FROM python:3.5
+FROM python:3.6
 
 ENV PYTHONUNBUFFERED 1
 

api/funkwhale_api/activity/serializers.py

 from rest_framework import serializers
 
+from funkwhale_api.activity import record
+
 
 class ModelSerializer(serializers.ModelSerializer):
     id = serializers.CharField(source='get_activity_url')
@@ -8,3 +10,15 @@ class ModelSerializer(serializers.ModelSerializer):
 
     def get_url(self, obj):
         return self.get_id(obj)
+
+
+class AutoSerializer(serializers.Serializer):
+    """
+    A serializer that will automatically use registered activity serializers
+    to serialize an henerogeneous list of objects (favorites, listenings, etc.)
+    """
+    def to_representation(self, instance):
+        serializer = record.registry[instance._meta.label]['serializer'](
+            instance
+        )
+        return serializer.data

api/funkwhale_api/activity/utils.py

+from django.db import models
+
+from funkwhale_api.common import fields
+from funkwhale_api.favorites.models import TrackFavorite
+from funkwhale_api.history.models import Listening
+
+
+def combined_recent(limit, **kwargs):
+    datetime_field = kwargs.pop('datetime_field', 'creation_date')
+    source_querysets = {
+        qs.model._meta.label: qs for qs in kwargs.pop('querysets')
+    }
+    querysets = {
+        k: qs.annotate(
+            __type=models.Value(
+                qs.model._meta.label, output_field=models.CharField()
+            )
+        ).values('pk', datetime_field, '__type')
+        for k, qs in source_querysets.items()
+    }
+    _qs_list = list(querysets.values())
+    union_qs = _qs_list[0].union(*_qs_list[1:])
+    records = []
+    for row in union_qs.order_by('-{}'.format(datetime_field))[:limit]:
+        records.append({
+            'type': row['__type'],
+            'when': row[datetime_field],
+            'pk': row['pk']
+        })
+    # Now we bulk-load each object type in turn
+    to_load = {}
+    for record in records:
+        to_load.setdefault(record['type'], []).append(record['pk'])
+    fetched = {}
+
+    for key, pks in to_load.items():
+        for item in source_querysets[key].filter(pk__in=pks):
+            fetched[(key, item.pk)] = item
+
+    # Annotate 'records' with loaded objects
+    for record in records:
+        record['object'] = fetched[(record['type'], record['pk'])]
+    return records
+
+
+def get_activity(user, limit=20):
+    query = fields.privacy_level_query(
+        user, lookup_field='user__privacy_level')
+    querysets = [
+        Listening.objects.filter(query).select_related(
+            'track',
+            'user',
+            'track__artist',
+            'track__album__artist',
+        ),
+        TrackFavorite.objects.filter(query).select_related(
+            'track',
+            'user',
+            'track__artist',
+            'track__album__artist',
+        ),
+    ]
+    records = combined_recent(limit=limit, querysets=querysets)
+    return [r['object'] for r in records]

api/funkwhale_api/activity/views.py

+from rest_framework import viewsets
+from rest_framework.response import Response
+
+from funkwhale_api.common.permissions import ConditionalAuthentication
+from funkwhale_api.favorites.models import TrackFavorite
+
+from . import serializers
+from . import utils
+
+
+class ActivityViewSet(viewsets.GenericViewSet):
+
+    serializer_class = serializers.AutoSerializer
+    permission_classes = [ConditionalAuthentication]
+    queryset = TrackFavorite.objects.none()
+
+    def list(self, request, *args, **kwargs):
+        activity = utils.get_activity(user=request.user)
+        serializer = self.serializer_class(activity, many=True)
+        return Response({'results': serializer.data}, status=200)

api/funkwhale_api/common/fields.py

@@ -22,6 +22,6 @@ def privacy_level_query(user, lookup_field='privacy_level'):
 
     return models.Q(**{
         '{}__in'.format(lookup_field): [
-            'me', 'followers', 'instance', 'everyone'
+            'followers', 'instance', 'everyone'
         ]
     })

api/funkwhale_api/common/storage.py

+import unicodedata
+
+from django.core.files.storage import FileSystemStorage
+
+
+class ASCIIFileSystemStorage(FileSystemStorage):
+    """
+    Convert unicode characters in name to ASCII characters.
+    """
+    def get_valid_name(self, name):
+        name = unicodedata.normalize('NFKD', name).encode('ascii', 'ignore')
+        return super().get_valid_name(name)

api/funkwhale_api/federation/activity.py

+import logging
+import json
+import requests
+import requests_http_signature
+
+from . import signing
+
+logger = logging.getLogger(__name__)
+
+ACTIVITY_TYPES = [
+    'Accept',
+    'Add',
+    'Announce',
+    'Arrive',
+    'Block',
+    'Create',
+    'Delete',
+    'Dislike',
+    'Flag',
+    'Follow',
+    'Ignore',
+    'Invite',
+    'Join',
+    'Leave',
+    'Like',
+    'Listen',
+    'Move',
+    'Offer',
+    'Question',
+    'Reject',
+    'Read',
+    'Remove',
+    'TentativeReject',
+    'TentativeAccept',
+    'Travel',
+    'Undo',
+    'Update',
+    'View',
+]
+
+
+OBJECT_TYPES = [
+    'Article',
+    'Audio',
+    'Document',
+    'Event',
+    'Image',
+    'Note',
+    'Page',
+    'Place',
+    'Profile',
+    'Relationship',
+    'Tombstone',
+    'Video',
+]
+
+def deliver(activity, on_behalf_of, to=[]):
+    from . import actors
+    logger.info('Preparing activity delivery to %s', to)
+    auth = requests_http_signature.HTTPSignatureAuth(
+        use_auth_header=False,
+        headers=[
+            '(request-target)',
+            'user-agent',
+            'host',
+            'date',
+            'content-type',],
+        algorithm='rsa-sha256',
+        key=on_behalf_of.private_key.encode('utf-8'),
+        key_id=on_behalf_of.private_key_id,
+    )
+    for url in to:
+        recipient_actor = actors.get_actor(url)
+        logger.debug('delivering to %s', recipient_actor.inbox_url)
+        logger.debug('activity content: %s', json.dumps(activity))
+        response = requests.post(
+            auth=auth,
+            json=activity,
+            url=recipient_actor.inbox_url,
+            headers={
+                'Content-Type': 'application/activity+json'
+            }
+        )
+        response.raise_for_status()
+        logger.debug('Remote answered with %s', response.status_code)