Merge branch 'channels' into 'develop'

Channels and instance timeline

See merge request funkwhale/funkwhale!74

.gitlab-ci.yml

@@ -22,7 +22,7 @@ test_api:
   variables:
     DJANGO_ALLOWED_HOSTS: "localhost"
     DATABASE_URL: "postgresql://postgres@postgres/postgres"
-
+    FUNKWHALE_URL: "https://funkwhale.ci"
   before_script:
     - cd api
     - pip install -r requirements/base.txt

api/Dockerfile

@@ -5,7 +5,11 @@ ENV PYTHONUNBUFFERED 1
 # Requirements have to be pulled and installed here, otherwise caching won't work
 RUN echo 'deb http://httpredir.debian.org/debian/ jessie-backports main' > /etc/apt/sources.list.d/ffmpeg.list
 COPY ./requirements.apt /requirements.apt
-RUN apt-get update -qq && grep "^[^#;]" requirements.apt | xargs apt-get install -y
+RUN apt-get update; \
+    grep "^[^#;]" requirements.apt | \
+    grep -Fv "python3-dev" | \
+    xargs apt-get install -y --no-install-recommends; \
+    rm -rf /usr/share/doc/* /usr/share/locale/*
 RUN curl -L https://github.com/acoustid/chromaprint/releases/download/v1.4.2/chromaprint-fpcalc-1.4.2-linux-x86_64.tar.gz | tar -xz -C /usr/local/bin --strip 1
 COPY ./requirements/base.txt /requirements/base.txt
 RUN pip install -r /requirements/base.txt
@@ -20,3 +24,4 @@ RUN pip install --upgrade youtube-dl
 WORKDIR /app
 
 ENTRYPOINT ["./compose/django/entrypoint.sh"]
+CMD ["./compose/django/daphne.sh"]

api/compose/django/gunicorn.sh → api/compose/django/daphne.sh

-#!/bin/sh
+#!/bin/bash -eux
 python /app/manage.py collectstatic --noinput
-/usr/local/bin/gunicorn config.wsgi -w 4 -b 0.0.0.0:5000 --chdir=/app
\ No newline at end of file
+/usr/local/bin/daphne --root-path=/app -b 0.0.0.0 -p 5000 config.asgi:application

api/config/asgi.py

+import django
+import os
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.production")
+
+django.setup()
+
+from .routing import application

api/config/routing.py

+from django.conf.urls import url
+
+from channels.auth import AuthMiddlewareStack
+from channels.routing import ProtocolTypeRouter, URLRouter
+
+from funkwhale_api.common.auth import TokenAuthMiddleware
+from funkwhale_api.instance import consumers
+
+
+application = ProtocolTypeRouter({
+    # Empty for now (http->django views is added by default)
+    "websocket": TokenAuthMiddleware(
+        URLRouter([
+            url("^api/v1/instance/activity$",
+                consumers.InstanceActivityConsumer),
+        ])
+    ),
+})

api/config/settings/common.py

@@ -25,11 +25,12 @@ except FileNotFoundError:
     pass
 
 ALLOWED_HOSTS = env.list('DJANGO_ALLOWED_HOSTS')
-
+FUNKWHALE_URL = env('FUNKWHALE_URL')
 
 # APP CONFIGURATION
 # ------------------------------------------------------------------------------
 DJANGO_APPS = (
+    'channels',
     # Default Django apps:
     'django.contrib.auth',
     'django.contrib.contenttypes',
@@ -82,6 +83,7 @@ if RAVEN_ENABLED:
 # Apps specific for this project go here.
 LOCAL_APPS = (
     'funkwhale_api.common',
+    'funkwhale_api.activity.apps.ActivityConfig',
     'funkwhale_api.users',  # custom users app
     # Your stuff: custom apps go here
     'funkwhale_api.instance',
@@ -253,9 +255,9 @@ MEDIA_URL = env("MEDIA_URL", default='/media/')
 # URL Configuration
 # ------------------------------------------------------------------------------
 ROOT_URLCONF = 'config.urls'
-
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#wsgi-application
 WSGI_APPLICATION = 'config.wsgi.application'
+ASGI_APPLICATION = "config.routing.application"
 
 # AUTHENTICATION CONFIGURATION
 # ------------------------------------------------------------------------------
@@ -284,6 +286,17 @@ CACHES = {
 }
 
 CACHES["default"]["BACKEND"] = "django_redis.cache.RedisCache"
+from urllib.parse import urlparse
+cache_url = urlparse(CACHES['default']['LOCATION'])
+CHANNEL_LAYERS = {
+    "default": {
+        "BACKEND": "channels_redis.core.RedisChannelLayer",
+        "CONFIG": {
+            "hosts": [(cache_url.hostname, cache_url.port)],
+        },
+    },
+}
+
 CACHES["default"]["OPTIONS"] = {
     "CLIENT_CLASS": "django_redis.client.DefaultClient",
     "IGNORE_EXCEPTIONS": True,  # mimics memcache behavior.

api/config/settings/production.py

@@ -58,8 +58,6 @@ CSRF_TRUSTED_ORIGINS = ALLOWED_HOSTS
 
 # END SITE CONFIGURATION
 
-INSTALLED_APPS += ("gunicorn", )
-
 # STORAGE CONFIGURATION
 # ------------------------------------------------------------------------------
 # Uploaded Media Files

api/docker/Dockerfile.test

 FROM python:3.5
 
 ENV PYTHONUNBUFFERED 1
-ENV PYTHONDONTWRITEBYTECODE 1
 
 # Requirements have to be pulled and installed here, otherwise caching won't work
 RUN echo 'deb http://httpredir.debian.org/debian/ jessie-backports main' > /etc/apt/sources.list.d/ffmpeg.list
 COPY ./requirements.apt /requirements.apt
-COPY ./install_os_dependencies.sh /install_os_dependencies.sh
-RUN bash install_os_dependencies.sh install
+RUN apt-get update; \
+    grep "^[^#;]" requirements.apt | \
+    grep -Fv "python3-dev" | \
+    xargs apt-get install -y --no-install-recommends; \
+    rm -rf /usr/share/doc/* /usr/share/locale/*
+
 RUN curl -L https://github.com/acoustid/chromaprint/releases/download/v1.4.2/chromaprint-fpcalc-1.4.2-linux-x86_64.tar.gz | tar -xz -C /usr/local/bin --strip 1
 
 RUN mkdir /requirements
@@ -18,4 +21,5 @@ RUN pip install -r /requirements/local.txt
 COPY ./requirements/test.txt /requirements/test.txt
 RUN pip install -r /requirements/test.txt
 
+COPY . /app
 WORKDIR /app

api/funkwhale_api/activity/apps.py

+from django.apps import AppConfig, apps
+
+from . import record
+
+class ActivityConfig(AppConfig):
+    name = 'funkwhale_api.activity'
+
+    def ready(self):
+        super(ActivityConfig, self).ready()
+
+        app_names = [app.name for app in apps.app_configs.values()]
+        record.registry.autodiscover(app_names)

api/funkwhale_api/activity/record.py

+import persisting_theory
+
+
+class ActivityRegistry(persisting_theory.Registry):
+    look_into = 'activities'
+
+    def _register_for_model(self, model, attr, value):
+        key = model._meta.label
+        d = self.setdefault(key, {'consumers': []})
+        d[attr] = value
+
+    def register_serializer(self, serializer_class):
+        model = serializer_class.Meta.model
+        self._register_for_model(model, 'serializer', serializer_class)
+        return serializer_class
+
+    def register_consumer(self, label):
+        def decorator(func):
+            consumers = self[label]['consumers']
+            if func not in consumers:
+                consumers.append(func)
+            return func
+        return decorator
+
+
+registry = ActivityRegistry()
+
+
+
+
+def send(obj):
+    conf = registry[obj.__class__._meta.label]
+    consumers = conf['consumers']
+    if not consumers:
+        return
+    serializer = conf['serializer'](obj)
+    for consumer in consumers:
+        consumer(data=serializer.data, obj=obj)

api/funkwhale_api/activity/serializers.py

+from rest_framework import serializers
+
+
+class ModelSerializer(serializers.ModelSerializer):
+    id = serializers.CharField(source='get_activity_url')
+    local_id = serializers.IntegerField(source='id')
+    # url = serializers.SerializerMethodField()
+
+    def get_url(self, obj):
+        return self.get_id(obj)

api/funkwhale_api/common/auth.py

+from urllib.parse import parse_qs
+
+import jwt
+
+from django.contrib.auth.models import AnonymousUser
+from django.utils.encoding import smart_text
+
+from rest_framework import exceptions
+from rest_framework_jwt.settings import api_settings
+from rest_framework_jwt.authentication import BaseJSONWebTokenAuthentication
+
+
+
+class TokenHeaderAuth(BaseJSONWebTokenAuthentication):
+    def get_jwt_value(self, request):
+
+        try:
+            qs = request.get('query_string', b'').decode('utf-8')
+            parsed = parse_qs(qs)
+            token = parsed['token'][0]
+        except KeyError:
+            raise exceptions.AuthenticationFailed('No token')
+
+        if not token:
+            raise exceptions.AuthenticationFailed('Empty token')
+
+        return token
+
+
+class TokenAuthMiddleware:
+    """
+    Custom middleware (insecure) that takes user IDs from the query string.
+    """
+
+    def __init__(self, inner):
+        # Store the ASGI application we were passed
+        self.inner = inner
+
+    def __call__(self, scope):
+        auth = TokenHeaderAuth()
+        try:
+            user, token = auth.authenticate(scope)
+        except exceptions.AuthenticationFailed:
+            user = AnonymousUser()
+
+        scope['user'] = user
+        return self.inner(scope)

api/funkwhale_api/common/channels.py

+from asgiref.sync import async_to_sync
+from channels.layers import get_channel_layer
+
+channel_layer = get_channel_layer()
+group_send = async_to_sync(channel_layer.group_send)
+group_add = async_to_sync(channel_layer.group_add)

api/funkwhale_api/common/consumers.py

+from channels.generic.websocket import JsonWebsocketConsumer
+from funkwhale_api.common import channels
+
+
+class JsonAuthConsumer(JsonWebsocketConsumer):
+    def connect(self):
+        try:
+            assert self.scope['user'].pk is not None
+        except (AssertionError, AttributeError, KeyError):
+            return self.close()
+
+        return self.accept()
+
+    def accept(self):
+        super().accept()
+        for group in self.groups:
+            channels.group_add(group, self.channel_name)

api/funkwhale_api/favorites/activities.py

+from funkwhale_api.common import channels
+from funkwhale_api.activity import record
+
+from . import serializers
+
+record.registry.register_serializer(
+    serializers.TrackFavoriteActivitySerializer)
+
+
+@record.registry.register_consumer('favorites.TrackFavorite')
+def broadcast_track_favorite_to_instance_activity(data, obj):
+    if obj.user.privacy_level not in ['instance', 'everyone']:
+        return
+
+    channels.group_send('instance_activity', {
+        'type': 'event.send',
+        'text': '',
+        'data': data
+    })

api/funkwhale_api/favorites/models.py

+from django.conf import settings
 from django.db import models
 from django.utils import timezone
 
 from funkwhale_api.music.models import Track
 
+
 class TrackFavorite(models.Model):
     creation_date = models.DateTimeField(default=timezone.now)
     user = models.ForeignKey(
@@ -18,3 +20,7 @@ class TrackFavorite(models.Model):
     def add(cls, track, user):
         favorite, created = cls.objects.get_or_create(user=user, track=track)
         return favorite
+
+    def get_activity_url(self):
+        return '{}/favorites/tracks/{}'.format(
+            self.user.get_activity_url(), self.pk)

api/funkwhale_api/favorites/serializers.py

+from django.conf import settings
+
 from rest_framework import serializers
 
+from funkwhale_api.activity import serializers as activity_serializers
 from funkwhale_api.music.serializers import TrackSerializerNested
+from funkwhale_api.music.serializers import TrackActivitySerializer
+from funkwhale_api.users.serializers import UserActivitySerializer
 
 from . import models
 
 
+class TrackFavoriteActivitySerializer(activity_serializers.ModelSerializer):
+    type = serializers.SerializerMethodField()
+    object = TrackActivitySerializer(source='track')
+    actor = UserActivitySerializer(source='user')
+    published = serializers.DateTimeField(source='creation_date')
+
+    class Meta:
+        model = models.TrackFavorite
+        fields = [
+            'id',
+            'local_id',
+            'object',
+            'type',
+            'actor',
+            'published'
+        ]
+
+    def get_actor(self, obj):
+        return UserActivitySerializer(obj.user).data
+
+    def get_type(self, obj):
+        return 'Like'
+
+
 class UserTrackFavoriteSerializer(serializers.ModelSerializer):
     # track = TrackSerializerNested(read_only=True)
     class Meta:

api/funkwhale_api/favorites/views.py

@@ -4,6 +4,7 @@ from rest_framework.response import Response
 from rest_framework import pagination
 from rest_framework.decorators import list_route
 
+from funkwhale_api.activity import record
 from funkwhale_api.music.models import Track
 from funkwhale_api.common.permissions import ConditionalAuthentication
 
@@ -33,6 +34,7 @@ class TrackFavoriteViewSet(mixins.CreateModelMixin,
         instance = self.perform_create(serializer)
         serializer = self.get_serializer(instance=instance)
         headers = self.get_success_headers(serializer.data)
+        record.send(instance)
         return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers)
 
     def get_queryset(self):