Merge branch 'federation-inbox' into 'develop'

Federation inbox

See merge request funkwhale/funkwhale!121

.env.dev

 API_AUTHENTICATION_REQUIRED=True
 RAVEN_ENABLED=false
 RAVEN_DSN=https://44332e9fdd3d42879c7d35bf8562c6a4:0062dc16a22b41679cd5765e5342f716@sentry.eliotberriot.com/5
+DJANGO_ALLOWED_HOSTS=localhost,nginx
+DJANGO_SETTINGS_MODULE=config.settings.local
+DJANGO_SECRET_KEY=dev
+C_FORCE_ROOT=true
+FUNKWHALE_URL=http://localhost
+PYTHONDONTWRITEBYTECODE=true

.gitlab-ci.yml

@@ -13,6 +13,7 @@ stages:
 test_api:
   services:
     - postgres:9.4
+    - redis:3
   stage: test
   image: funkwhale/funkwhale:latest
   cache:
@@ -24,6 +25,7 @@ test_api:
     DATABASE_URL: "postgresql://postgres@postgres/postgres"
     FUNKWHALE_URL: "https://funkwhale.ci"
     CACHEOPS_ENABLED: "false"
+    DJANGO_SETTINGS_MODULE: config.settings.local
 
   before_script:
     - cd api

api/compose/django/dev-entrypoint.sh

 #!/bin/bash
 set -e
-if [ $1 = "pytest" ]; then
-    # let pytest.ini handle it
-    unset DJANGO_SETTINGS_MODULE
-fi
 exec "$@"

api/config/settings/common.py

@@ -344,7 +344,12 @@ REST_FRAMEWORK = {
     ),
     'DEFAULT_PAGINATION_CLASS': 'funkwhale_api.common.pagination.FunkwhalePagination',
     'PAGE_SIZE': 25,
-
+    'DEFAULT_PARSER_CLASSES': (
+        'rest_framework.parsers.JSONParser',
+        'rest_framework.parsers.FormParser',
+        'rest_framework.parsers.MultiPartParser',
+        'funkwhale_api.federation.parsers.ActivityParser',
+    ),
     'DEFAULT_AUTHENTICATION_CLASSES': (
         'funkwhale_api.common.authentication.JSONWebTokenAuthenticationQS',
         'rest_framework_jwt.authentication.JSONWebTokenAuthentication',
@@ -396,6 +401,9 @@ PLAYLISTS_MAX_TRACKS = env.int('PLAYLISTS_MAX_TRACKS', default=250)
 
 ACCOUNT_USERNAME_BLACKLIST = [
     'funkwhale',
+    'library',
+    'test',
+    'status',
     'root',
     'admin',
     'owner',

api/config/settings/local.py

@@ -72,6 +72,10 @@ LOGGING = {
             'handlers':['console'],
             'propagate': True,
             'level':'DEBUG',
-        }
+        },
+        '': {
+            'level': 'DEBUG',
+            'handlers': ['console'],
+        },
     },
 }

api/config/settings/test.py

-from .common import *  # noqa
-SECRET_KEY = env("DJANGO_SECRET_KEY", default='test')
-
-# Mail settings
-# ------------------------------------------------------------------------------
-EMAIL_HOST = 'localhost'
-EMAIL_PORT = 1025
-EMAIL_BACKEND = env('DJANGO_EMAIL_BACKEND',
-                    default='django.core.mail.backends.console.EmailBackend')
-
-# CACHING
-# ------------------------------------------------------------------------------
-CACHES = {
-    'default': {
-        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
-        'LOCATION': ''
-    }
-}
-
-CELERY_BROKER_URL = 'memory://'
-
-########## CELERY
-# In development, all tasks will be executed locally by blocking until the task returns
-CELERY_TASK_ALWAYS_EAGER = True
-########## END CELERY
-
-# Your local stuff: Below this line define 3rd party library settings
-API_AUTHENTICATION_REQUIRED = False
-CACHEOPS_ENABLED = False

api/funkwhale_api/federation/activity.py

+import logging
+import json
+import requests
+import requests_http_signature
+
+from . import signing
+
+logger = logging.getLogger(__name__)
+
+ACTIVITY_TYPES = [
+    'Accept',
+    'Add',
+    'Announce',
+    'Arrive',
+    'Block',
+    'Create',
+    'Delete',
+    'Dislike',
+    'Flag',
+    'Follow',
+    'Ignore',
+    'Invite',
+    'Join',
+    'Leave',
+    'Like',
+    'Listen',
+    'Move',
+    'Offer',
+    'Question',
+    'Reject',
+    'Read',
+    'Remove',
+    'TentativeReject',
+    'TentativeAccept',
+    'Travel',
+    'Undo',
+    'Update',
+    'View',
+]
+
+
+OBJECT_TYPES = [
+    'Article',
+    'Audio',
+    'Document',
+    'Event',
+    'Image',
+    'Note',
+    'Page',
+    'Place',
+    'Profile',
+    'Relationship',
+    'Tombstone',
+    'Video',
+]
+
+def deliver(activity, on_behalf_of, to=[]):
+    from . import actors
+    logger.info('Preparing activity delivery to %s', to)
+    auth = requests_http_signature.HTTPSignatureAuth(
+        use_auth_header=False,
+        headers=[
+            '(request-target)',
+            'user-agent',
+            'host',
+            'date',
+            'content-type',],
+        algorithm='rsa-sha256',
+        key=on_behalf_of.private_key.encode('utf-8'),
+        key_id=on_behalf_of.private_key_id,
+    )
+    for url in to:
+        recipient_actor = actors.get_actor(url)
+        logger.debug('delivering to %s', recipient_actor.inbox_url)
+        logger.debug('activity content: %s', json.dumps(activity))
+        response = requests.post(
+            auth=auth,
+            json=activity,
+            url=recipient_actor.inbox_url,
+            headers={
+                'Content-Type': 'application/activity+json'
+            }
+        )
+        response.raise_for_status()
+        logger.debug('Remote answered with %s', response.status_code)

api/funkwhale_api/federation/actors.py

+import logging
+import requests
+import xml
+
+from django.conf import settings
+from django.urls import reverse
+from django.utils import timezone
+
+from rest_framework.exceptions import PermissionDenied
+
+from dynamic_preferences.registries import global_preferences_registry
+
+from . import activity
+from . import factories
+from . import models
+from . import serializers
+from . import utils
+
+logger = logging.getLogger(__name__)
+
+
+def remove_tags(text):
+    logger.debug('Removing tags from %s', text)
+    return ''.join(xml.etree.ElementTree.fromstring('<div>{}</div>'.format(text)).itertext())
+
+
+def get_actor_data(actor_url):
+    response = requests.get(
+        actor_url,
+        headers={
+            'Accept': 'application/activity+json',
+        }
+    )
+    response.raise_for_status()
+    try:
+        return response.json()
+    except:
+        raise ValueError(
+            'Invalid actor payload: {}'.format(response.text))
+
+def get_actor(actor_url):
+    data = get_actor_data(actor_url)
+    serializer = serializers.ActorSerializer(data=data)
+    serializer.is_valid(raise_exception=True)
+
+    return serializer.build()
+
+
+class SystemActor(object):
+    additional_attributes = {}
+
+    def get_actor_instance(self):
+        a = models.Actor(
+            **self.get_instance_argument(
+                self.id,
+                name=self.name,
+                summary=self.summary,
+                **self.additional_attributes
+            )
+        )
+        a.pk = self.id
+        return a
+
+    def get_instance_argument(self, id, name, summary, **kwargs):
+        preferences = global_preferences_registry.manager()
+        p = {
+            'preferred_username': id,
+            'domain': settings.FEDERATION_HOSTNAME,
+            'type': 'Person',
+            'name': name.format(host=settings.FEDERATION_HOSTNAME),
+            'manually_approves_followers': True,
+            'url': utils.full_url(
+                reverse(
+                    'federation:instance-actors-detail',
+                    kwargs={'actor': id})),
+            'shared_inbox_url': utils.full_url(
+                reverse(
+                    'federation:instance-actors-inbox',
+                    kwargs={'actor': id})),
+            'inbox_url': utils.full_url(
+                reverse(
+                    'federation:instance-actors-inbox',
+                    kwargs={'actor': id})),
+            'outbox_url': utils.full_url(
+                reverse(
+                    'federation:instance-actors-outbox',
+                    kwargs={'actor': id})),
+            'public_key': preferences['federation__public_key'],
+            'private_key': preferences['federation__private_key'],
+            'summary': summary.format(host=settings.FEDERATION_HOSTNAME)
+        }
+        p.update(kwargs)
+        return p
+
+    def get_inbox(self, data, actor=None):
+        raise NotImplementedError
+
+    def post_inbox(self, data, actor=None):
+        raise NotImplementedError
+
+    def get_outbox(self, data, actor=None):
+        raise NotImplementedError
+
+    def post_outbox(self, data, actor=None):
+        raise NotImplementedError
+
+
+class LibraryActor(SystemActor):
+    id = 'library'
+    name = '{host}\'s library'
+    summary = 'Bot account to federate with {host}\'s library'
+    additional_attributes = {
+        'manually_approves_followers': True
+    }
+
+
+class TestActor(SystemActor):
+    id = 'test'
+    name = '{host}\'s test account'
+    summary = (
+        'Bot account to test federation with {host}. '
+        'Send me /ping and I\'ll answer you.'
+    )
+    additional_attributes = {
+        'manually_approves_followers': False
+    }
+
+    def get_outbox(self, data, actor=None):
+        return {
+        	"@context": [
+        		"https://www.w3.org/ns/activitystreams",
+        		"https://w3id.org/security/v1",
+        		{}
+        	],
+        	"id": utils.full_url(
+                reverse(
+                    'federation:instance-actors-outbox',
+                    kwargs={'actor': self.id})),
+        	"type": "OrderedCollection",
+        	"totalItems": 0,
+        	"orderedItems": []
+        }
+
+    def post_inbox(self, data, actor=None):
+        if actor is None:
+            raise PermissionDenied('Actor not authenticated')
+
+        serializer = serializers.ActivitySerializer(
+            data=data, context={'actor': actor})
+        serializer.is_valid(raise_exception=True)
+
+        ac = serializer.validated_data
+        logger.info('Received activity on %s inbox', self.id)
+        if ac['type'] == 'Create' and ac['object']['type'] == 'Note':
+            # we received a toot \o/
+            command = self.parse_command(ac['object']['content'])
+            logger.debug('Parsed command: %s', command)
+            if command == 'ping':
+                self.handle_ping(ac, actor)
+
+    def parse_command(self, message):
+        """
+        Remove any links or fancy markup to extract /command from
+        a note message.
+        """
+        raw = remove_tags(message)
+        try:
+            return raw.split('/')[1]
+        except IndexError:
+            return
+
+    def handle_ping(self, ac, sender):
+        now = timezone.now()
+        test_actor = self.get_actor_instance()
+        reply_url = 'https://{}/activities/note/{}'.format(
+            settings.FEDERATION_HOSTNAME, now.timestamp()
+        )
+        reply_content = '{} Pong!'.format(
+            sender.mention_username
+        )
+        reply_activity = {
+            "@context": [
+        		"https://www.w3.org/ns/activitystreams",
+        		"https://w3id.org/security/v1",
+        		{}
+        	],
+            'type': 'Create',
+            'actor': test_actor.url,
+            'id': '{}/activity'.format(reply_url),
+            'published': now.isoformat(),
+            'to': ac['actor'],
+            'cc': [],
+            'object': factories.NoteFactory(
+                content='Pong!',
+                summary=None,
+                published=now.isoformat(),
+                id=reply_url,
+                inReplyTo=ac['object']['id'],
+                sensitive=False,
+                url=reply_url,
+                to=[ac['actor']],
+                attributedTo=test_actor.url,
+                cc=[],
+                attachment=[],
+                tag=[
+                    {
+                        "type": "Mention",
+                        "href": ac['actor'],
+                        "name": sender.mention_username
+                    }
+                ]
+            )
+        }
+        activity.deliver(
+            reply_activity,
+            to=[ac['actor']],
+            on_behalf_of=test_actor)
+
+SYSTEM_ACTORS = {
+    'library': LibraryActor(),
+    'test': TestActor(),
+}

api/funkwhale_api/federation/authentication.py

+import cryptography
+
+from django.contrib.auth.models import AnonymousUser
+
+from rest_framework import authentication
+from rest_framework import exceptions
+
+from . import actors
+from . import keys
+from . import serializers
+from . import signing
+from . import utils
+
+
+class SignatureAuthentication(authentication.BaseAuthentication):
+    def authenticate_actor(self, request):
+        headers = utils.clean_wsgi_headers(request.META)
+        try:
+            signature = headers['Signature']
+            key_id = keys.get_key_id_from_signature_header(signature)
+        except KeyError:
+            return
+        except ValueError as e:
+            raise exceptions.AuthenticationFailed(str(e))
+
+        try:
+            actor_data = actors.get_actor_data(key_id)
+        except Exception as e:
+            raise exceptions.AuthenticationFailed(str(e))
+
+        try:
+            public_key = actor_data['publicKey']['publicKeyPem']
+        except KeyError:
+            raise exceptions.AuthenticationFailed('No public key found')
+
+        serializer = serializers.ActorSerializer(data=actor_data)
+        if not serializer.is_valid():
+            raise exceptions.AuthenticationFailed('Invalid actor payload: {}'.format(serializer.errors))
+
+        try:
+            signing.verify_django(request, public_key.encode('utf-8'))
+        except cryptography.exceptions.InvalidSignature:
+            raise exceptions.AuthenticationFailed('Invalid signature')
+
+        return serializer.build()
+
+    def authenticate(self, request):
+        setattr(request, 'actor', None)
+        actor = self.authenticate_actor(request)
+        user = AnonymousUser()
+        setattr(request, 'actor', actor)
+        return (user, None)

api/funkwhale_api/federation/exceptions.py

@@ -2,3 +2,7 @@
 
 class MalformedPayload(ValueError):
     pass
+
+
+class MissingSignature(KeyError):
+    pass

api/funkwhale_api/federation/factories.py

@@ -2,9 +2,12 @@ import factory
 import requests
 import requests_http_signature
 
+from django.utils import timezone
+
 from funkwhale_api.factories import registry
 
 from . import keys
+from . import models
 
 
 registry.register(keys.get_key_pair, name='federation.KeyPair')
@@ -15,7 +18,13 @@ class SignatureAuthFactory(factory.Factory):
     algorithm = 'rsa-sha256'
     key = factory.LazyFunction(lambda: keys.get_key_pair()[0])
     key_id = factory.Faker('url')
-
+    use_auth_header = False
+    headers = [
+        '(request-target)',
+        'user-agent',
+        'host',
+        'date',
+        'content-type',]
     class Meta:
         model = requests_http_signature.HTTPSignatureAuth
 
@@ -28,3 +37,55 @@ class SignedRequestFactory(factory.Factory):
 
     class Meta:
         model = requests.Request
+
+    @factory.post_generation
+    def headers(self, create, extracted, **kwargs):
+        default_headers = {
+            'User-Agent': 'Test',
+            'Host': 'test.host',
+            'Date': 'Right now',
+            'Content-Type': 'application/activity+json'
+        }
+        if extracted:
+            default_headers.update(extracted)
+        self.headers.update(default_headers)
+
+
+@registry.register
+class ActorFactory(factory.DjangoModelFactory):
+
+    public_key = None
+    private_key = None
+    preferred_username = factory.Faker('user_name')
+    summary = factory.Faker('paragraph')
+    domain = factory.Faker('domain_name')
+    url = factory.LazyAttribute(lambda o: 'https://{}/users/{}'.format(o.domain, o.preferred_username))
+    inbox_url = factory.LazyAttribute(lambda o: 'https://{}/users/{}/inbox'.format(o.domain, o.preferred_username))
+    outbox_url = factory.LazyAttribute(lambda o: 'https://{}/users/{}/outbox'.format(o.domain, o.preferred_username))
+
+    class Meta:
+        model = models.Actor
+
+    @classmethod
+    def _generate(cls, create, attrs):
+        has_public = attrs.get('public_key') is not None
+        has_private = attrs.get('private_key') is not None
+        if not has_public and not has_private:
+            private, public = keys.get_key_pair()
+            attrs['private_key'] = private.decode('utf-8')
+            attrs['public_key'] = public.decode('utf-8')
+        return super()._generate(create, attrs)
+
+
+@registry.register(name='federation.Note')
+class NoteFactory(factory.Factory):
+    type = 'Note'
+    id = factory.Faker('url')
+    published = factory.LazyFunction(
+        lambda: timezone.now().isoformat()
+    )
+    inReplyTo = None
+    content = factory.Faker('sentence')
+
+    class Meta:
+        model = dict

api/funkwhale_api/federation/keys.py

@@ -2,10 +2,14 @@ from cryptography.hazmat.primitives import serialization as crypto_serialization
 from cryptography.hazmat.primitives.asymmetric import rsa
 from cryptography.hazmat.backends import default_backend as crypto_default_backend
 
+import re
 import requests
+import urllib.parse
 
 from . import exceptions
 
+KEY_ID_REGEX = re.compile(r'keyId=\"(?P<id>.*)\"')
+
 
 def get_key_pair(size=2048):
     key = rsa.generate_private_key(
@@ -25,19 +29,21 @@ def get_key_pair(size=2048):
     return private_key, public_key
 
 
-def get_public_key(actor_url):
-    """
-    Given an actor_url, request it and extract publicKey data from
-    the response payload.
-    """
-    response = requests.get(actor_url)
-    response.raise_for_status()
-    payload = response.json()
+def get_key_id_from_signature_header(header_string):
+    parts = header_string.split(',')
     try:
-        return {
-            'public_key_pem': payload['publicKey']['publicKeyPem'],
-            'id': payload['publicKey']['id'],
-            'owner': payload['publicKey']['owner'],
-        }
-    except KeyError:
-        raise exceptions.MalformedPayload(str(payload))
+        raw_key_id = [p for p in parts if p.startswith('keyId="')][0]
+    except IndexError:
+        raise ValueError('Missing key id')
+
+    match = KEY_ID_REGEX.match(raw_key_id)
+    if not match:
+        raise ValueError('Invalid key id')
+
+    key_id = match.groups()[0]
+    url = urllib.parse.urlparse(key_id)
+    if not url.scheme or not url.netloc:
+        raise ValueError('Invalid url')
+    if url.scheme not in ['http', 'https']:
+        raise ValueError('Invalid shceme')
+    return key_id