Merge branch 'release/0.6'

.gitlab-ci.yml

@@ -22,7 +22,7 @@ test_api:
   variables:
     DJANGO_ALLOWED_HOSTS: "localhost"
     DATABASE_URL: "postgresql://postgres@postgres/postgres"
-
+    FUNKWHALE_URL: "https://funkwhale.ci"
   before_script:
     - cd api
     - pip install -r requirements/base.txt

CHANGELOG

@@ -3,6 +3,97 @@ Changelog
 
 .. towncrier
 
+0.6 (2018-03-04)
+----------------
+
+Features:
+
+- Basic activity stream for listening and favorites (#23)
+- Switched to django-channels and daphne for serving HTTP and websocket (#34)
+
+Upgrades notes
+**************
+
+This version contains breaking changes in the way funkwhale is deployed,
+please read the notes carefully.
+
+Instance timeline
+^^^^^^^^^^^^^^^^^
+
+A new "Activity" page is now available from the sidebar, where you can
+browse your instance activity. At the moment, this includes other users
+favorites and listening, but more activity types will be implemented in the
+future.
+
+Internally, we implemented those events by following the Activity Stream
+specification, which will help us to be compatible with other networks
+in the long-term.
+
+A new settings page has been added to control the visibility of your activity.
+By default, your activity will be browsable by anyone on your instance,
+but you can switch to a full private mode where nothing is shared.
+
+The setting form is available in your profile.
+
+Switch from gunicorn to daphne
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This release include an important change in the way we serve the HTTP API.
+To prepare for new realtime features and enable websocket support in Funkwhale,
+we are now using django-channels and daphne to serve HTTP and websocket traffic.
+
+This replaces gunicorn and the switch should be easy assuming you
+follow the upgrade process described bellow.
+
+If you are using docker, please remove the command instruction inside the
+api service, as the up-to-date command is now included directly in the image
+as the default entry point:
+
+.. code-block:: yaml
+
+    api:
+      restart: unless-stopped
+      image: funkwhale/funkwhale:${FUNKWHALE_VERSION:-latest}
+      command: ./compose/django/gunicorn.sh  # You can remove this line
+
+On non docker setups, you'll have to update the [Service] block of your
+funkwhale-server systemd unit file to launch the application server using daphne instead of gunicorn.
+
+The new configuration should be similar to this:
+
+.. code-block:: ini
+
+    [Service]
+    User=funkwhale
+    # adapt this depending on the path of your funkwhale installation
+    WorkingDirectory=/srv/funkwhale/api
+    EnvironmentFile=/srv/funkwhale/config/.env
+    ExecStart=/usr/local/bin/daphne -b ${FUNKWHALE_API_IP} -p ${FUNKWHALE_API_PORT} config.asgi:application
+
+Ensure you update funkwhale's dependencies as usual to install the required
+packages.
+
+On both docker and non-docker setup, you'll also have to update your nginx
+configuration for websocket support. Ensure you have the following blocks
+included in your virtualhost file:
+
+.. code-block:: txt
+
+    map $http_upgrade $connection_upgrade {
+        default upgrade;
+        ''      close;
+    }
+
+    server {
+        ...
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection $connection_upgrade;
+    }
+
+Remember to reload your nginx server after the edit.
+
+
 0.5.4 (2018-02-28)
 ------------------
 

api/Dockerfile

@@ -5,7 +5,11 @@ ENV PYTHONUNBUFFERED 1
 # Requirements have to be pulled and installed here, otherwise caching won't work
 RUN echo 'deb http://httpredir.debian.org/debian/ jessie-backports main' > /etc/apt/sources.list.d/ffmpeg.list
 COPY ./requirements.apt /requirements.apt
-RUN apt-get update -qq && grep "^[^#;]" requirements.apt | xargs apt-get install -y
+RUN apt-get update; \
+    grep "^[^#;]" requirements.apt | \
+    grep -Fv "python3-dev" | \
+    xargs apt-get install -y --no-install-recommends; \
+    rm -rf /usr/share/doc/* /usr/share/locale/*
 RUN curl -L https://github.com/acoustid/chromaprint/releases/download/v1.4.2/chromaprint-fpcalc-1.4.2-linux-x86_64.tar.gz | tar -xz -C /usr/local/bin --strip 1
 COPY ./requirements/base.txt /requirements/base.txt
 RUN pip install -r /requirements/base.txt
@@ -20,3 +24,4 @@ RUN pip install --upgrade youtube-dl
 WORKDIR /app
 
 ENTRYPOINT ["./compose/django/entrypoint.sh"]
+CMD ["./compose/django/daphne.sh"]

api/compose/django/gunicorn.sh → api/compose/django/daphne.sh

-#!/bin/sh
+#!/bin/bash -eux
 python /app/manage.py collectstatic --noinput
-/usr/local/bin/gunicorn config.wsgi -w 4 -b 0.0.0.0:5000 --chdir=/app
+/usr/local/bin/daphne --root-path=/app -b 0.0.0.0 -p 5000 config.asgi:application
\ No newline at end of file

api/config/asgi.py

+import django
+import os
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.production")
+
+django.setup()
+
+from .routing import application

api/config/routing.py

+from django.conf.urls import url
+
+from channels.auth import AuthMiddlewareStack
+from channels.routing import ProtocolTypeRouter, URLRouter
+
+from funkwhale_api.common.auth import TokenAuthMiddleware
+from funkwhale_api.instance import consumers
+
+
+application = ProtocolTypeRouter({
+    # Empty for now (http->django views is added by default)
+    "websocket": TokenAuthMiddleware(
+        URLRouter([
+            url("^api/v1/instance/activity$",
+                consumers.InstanceActivityConsumer),
+        ])
+    ),
+})

api/config/settings/common.py

@@ -25,11 +25,12 @@ except FileNotFoundError:
     pass
 
 ALLOWED_HOSTS = env.list('DJANGO_ALLOWED_HOSTS')
-
+FUNKWHALE_URL = env('FUNKWHALE_URL')
 
 # APP CONFIGURATION
 # ------------------------------------------------------------------------------
 DJANGO_APPS = (
+    'channels',
     # Default Django apps:
     'django.contrib.auth',
     'django.contrib.contenttypes',
@@ -82,6 +83,7 @@ if RAVEN_ENABLED:
 # Apps specific for this project go here.
 LOCAL_APPS = (
     'funkwhale_api.common',
+    'funkwhale_api.activity.apps.ActivityConfig',
     'funkwhale_api.users',  # custom users app
     # Your stuff: custom apps go here
     'funkwhale_api.instance',
@@ -253,9 +255,9 @@ MEDIA_URL = env("MEDIA_URL", default='/media/')
 # URL Configuration
 # ------------------------------------------------------------------------------
 ROOT_URLCONF = 'config.urls'
-
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#wsgi-application
 WSGI_APPLICATION = 'config.wsgi.application'
+ASGI_APPLICATION = "config.routing.application"
 
 # AUTHENTICATION CONFIGURATION
 # ------------------------------------------------------------------------------
@@ -284,6 +286,17 @@ CACHES = {
 }
 
 CACHES["default"]["BACKEND"] = "django_redis.cache.RedisCache"
+from urllib.parse import urlparse
+cache_url = urlparse(CACHES['default']['LOCATION'])
+CHANNEL_LAYERS = {
+    "default": {
+        "BACKEND": "channels_redis.core.RedisChannelLayer",
+        "CONFIG": {
+            "hosts": [(cache_url.hostname, cache_url.port)],
+        },
+    },
+}
+
 CACHES["default"]["OPTIONS"] = {
     "CLIENT_CLASS": "django_redis.client.DefaultClient",
     "IGNORE_EXCEPTIONS": True,  # mimics memcache behavior.

api/config/settings/production.py

@@ -58,8 +58,6 @@ CSRF_TRUSTED_ORIGINS = ALLOWED_HOSTS
 
 # END SITE CONFIGURATION
 
-INSTALLED_APPS += ("gunicorn", )
-
 # STORAGE CONFIGURATION
 # ------------------------------------------------------------------------------
 # Uploaded Media Files

api/docker/Dockerfile.test

 FROM python:3.5
 
 ENV PYTHONUNBUFFERED 1
-ENV PYTHONDONTWRITEBYTECODE 1
 
 # Requirements have to be pulled and installed here, otherwise caching won't work
 RUN echo 'deb http://httpredir.debian.org/debian/ jessie-backports main' > /etc/apt/sources.list.d/ffmpeg.list
 COPY ./requirements.apt /requirements.apt
-COPY ./install_os_dependencies.sh /install_os_dependencies.sh
-RUN bash install_os_dependencies.sh install
+RUN apt-get update; \
+    grep "^[^#;]" requirements.apt | \
+    grep -Fv "python3-dev" | \
+    xargs apt-get install -y --no-install-recommends; \
+    rm -rf /usr/share/doc/* /usr/share/locale/*
+
 RUN curl -L https://github.com/acoustid/chromaprint/releases/download/v1.4.2/chromaprint-fpcalc-1.4.2-linux-x86_64.tar.gz | tar -xz -C /usr/local/bin --strip 1
 
 RUN mkdir /requirements
@@ -18,4 +21,5 @@ RUN pip install -r /requirements/local.txt
 COPY ./requirements/test.txt /requirements/test.txt
 RUN pip install -r /requirements/test.txt
 
+COPY . /app
 WORKDIR /app

api/funkwhale_api/__init__.py

 # -*- coding: utf-8 -*-
-__version__ = '0.5.4'
+__version__ = '0.6'
 __version_info__ = tuple([int(num) if num.isdigit() else num for num in __version__.replace('-', '.', 1).split('.')])

api/funkwhale_api/activity/apps.py

+from django.apps import AppConfig, apps
+
+from . import record
+
+class ActivityConfig(AppConfig):
+    name = 'funkwhale_api.activity'
+
+    def ready(self):
+        super(ActivityConfig, self).ready()
+
+        app_names = [app.name for app in apps.app_configs.values()]
+        record.registry.autodiscover(app_names)

api/funkwhale_api/activity/record.py

+import persisting_theory
+
+
+class ActivityRegistry(persisting_theory.Registry):
+    look_into = 'activities'
+
+    def _register_for_model(self, model, attr, value):
+        key = model._meta.label
+        d = self.setdefault(key, {'consumers': []})
+        d[attr] = value
+
+    def register_serializer(self, serializer_class):
+        model = serializer_class.Meta.model
+        self._register_for_model(model, 'serializer', serializer_class)
+        return serializer_class
+
+    def register_consumer(self, label):
+        def decorator(func):
+            consumers = self[label]['consumers']
+            if func not in consumers:
+                consumers.append(func)
+            return func
+        return decorator
+
+
+registry = ActivityRegistry()
+
+
+
+
+def send(obj):
+    conf = registry[obj.__class__._meta.label]
+    consumers = conf['consumers']
+    if not consumers:
+        return
+    serializer = conf['serializer'](obj)
+    for consumer in consumers:
+        consumer(data=serializer.data, obj=obj)

api/funkwhale_api/activity/serializers.py

+from rest_framework import serializers
+
+
+class ModelSerializer(serializers.ModelSerializer):
+    id = serializers.CharField(source='get_activity_url')
+    local_id = serializers.IntegerField(source='id')
+    # url = serializers.SerializerMethodField()
+
+    def get_url(self, obj):
+        return self.get_id(obj)

api/funkwhale_api/common/auth.py

+from urllib.parse import parse_qs
+
+import jwt
+
+from django.contrib.auth.models import AnonymousUser
+from django.utils.encoding import smart_text
+
+from rest_framework import exceptions
+from rest_framework_jwt.settings import api_settings
+from rest_framework_jwt.authentication import BaseJSONWebTokenAuthentication
+
+
+
+class TokenHeaderAuth(BaseJSONWebTokenAuthentication):
+    def get_jwt_value(self, request):
+
+        try:
+            qs = request.get('query_string', b'').decode('utf-8')
+            parsed = parse_qs(qs)
+            token = parsed['token'][0]
+        except KeyError:
+            raise exceptions.AuthenticationFailed('No token')
+
+        if not token:
+            raise exceptions.AuthenticationFailed('Empty token')
+
+        return token
+
+
+class TokenAuthMiddleware:
+    """
+    Custom middleware (insecure) that takes user IDs from the query string.
+    """
+
+    def __init__(self, inner):
+        # Store the ASGI application we were passed
+        self.inner = inner
+
+    def __call__(self, scope):
+        auth = TokenHeaderAuth()
+        try:
+            user, token = auth.authenticate(scope)
+        except exceptions.AuthenticationFailed:
+            user = AnonymousUser()
+
+        scope['user'] = user
+        return self.inner(scope)

api/funkwhale_api/common/channels.py

+from asgiref.sync import async_to_sync
+from channels.layers import get_channel_layer
+
+channel_layer = get_channel_layer()
+group_send = async_to_sync(channel_layer.group_send)
+group_add = async_to_sync(channel_layer.group_add)

api/funkwhale_api/common/consumers.py

+from channels.generic.websocket import JsonWebsocketConsumer
+from funkwhale_api.common import channels
+
+
+class JsonAuthConsumer(JsonWebsocketConsumer):
+    def connect(self):
+        try:
+            assert self.scope['user'].pk is not None
+        except (AssertionError, AttributeError, KeyError):
+            return self.close()
+
+        return self.accept()
+
+    def accept(self):
+        super().accept()
+        for group in self.groups:
+            channels.group_add(group, self.channel_name)

api/funkwhale_api/favorites/activities.py

+from funkwhale_api.common import channels
+from funkwhale_api.activity import record
+
+from . import serializers
+
+record.registry.register_serializer(
+    serializers.TrackFavoriteActivitySerializer)
+
+
+@record.registry.register_consumer('favorites.TrackFavorite')
+def broadcast_track_favorite_to_instance_activity(data, obj):
+    if obj.user.privacy_level not in ['instance', 'everyone']:
+        return
+
+    channels.group_send('instance_activity', {
+        'type': 'event.send',
+        'text': '',
+        'data': data
+    })

api/funkwhale_api/favorites/models.py

+from django.conf import settings
 from django.db import models
 from django.utils import timezone
 
 from funkwhale_api.music.models import Track
 
+
 class TrackFavorite(models.Model):
     creation_date = models.DateTimeField(default=timezone.now)
     user = models.ForeignKey(
@@ -18,3 +20,7 @@ class TrackFavorite(models.Model):
     def add(cls, track, user):
         favorite, created = cls.objects.get_or_create(user=user, track=track)
         return favorite
+
+    def get_activity_url(self):
+        return '{}/favorites/tracks/{}'.format(
+            self.user.get_activity_url(), self.pk)