diff --git a/.env.dev b/.env.dev index 9923c31487bd305a1f9fa516ba93c4a4895ceb1e..2e883414365a1c83a982887c70bb272d04398760 100644 --- a/.env.dev +++ b/.env.dev @@ -1,3 +1,9 @@ API_AUTHENTICATION_REQUIRED=True RAVEN_ENABLED=false RAVEN_DSN=https://44332e9fdd3d42879c7d35bf8562c6a4:0062dc16a22b41679cd5765e5342f716@sentry.eliotberriot.com/5 +DJANGO_ALLOWED_HOSTS=localhost,nginx +DJANGO_SETTINGS_MODULE=config.settings.local +DJANGO_SECRET_KEY=dev +C_FORCE_ROOT=true +FUNKWHALE_URL=http://localhost +PYTHONDONTWRITEBYTECODE=true diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 01717831a97a104222ac69c26770bc38476d0c30..94b40bed3b7ca63cd52ae3d492b19cd211fef6d6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,6 +13,7 @@ stages: test_api: services: - postgres:9.4 + - redis:3 stage: test image: funkwhale/funkwhale:latest cache: @@ -24,6 +25,7 @@ test_api: DATABASE_URL: "postgresql://postgres@postgres/postgres" FUNKWHALE_URL: "https://funkwhale.ci" CACHEOPS_ENABLED: "false" + DJANGO_SETTINGS_MODULE: config.settings.local before_script: - cd api diff --git a/api/compose/django/dev-entrypoint.sh b/api/compose/django/dev-entrypoint.sh index 416207b43d66d6ec7d05b12dd508c33577b8afda..6deeebb0085ede8bd696d59fb78af1d6d778a41e 100755 --- a/api/compose/django/dev-entrypoint.sh +++ b/api/compose/django/dev-entrypoint.sh @@ -1,7 +1,3 @@ #!/bin/bash set -e -if [ $1 = "pytest" ]; then - # let pytest.ini handle it - unset DJANGO_SETTINGS_MODULE -fi exec "$@" diff --git a/api/config/settings/common.py b/api/config/settings/common.py index 32cdb5b7f53e54dedcab0a3665d647e349553170..fbe3b7045e24c67d87c2ee90441103ccc27ac57a 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -344,7 +344,12 @@ REST_FRAMEWORK = { ), 'DEFAULT_PAGINATION_CLASS': 'funkwhale_api.common.pagination.FunkwhalePagination', 'PAGE_SIZE': 25, - + 'DEFAULT_PARSER_CLASSES': ( + 'rest_framework.parsers.JSONParser', + 'rest_framework.parsers.FormParser', + 'rest_framework.parsers.MultiPartParser', + 'funkwhale_api.federation.parsers.ActivityParser', + ), 'DEFAULT_AUTHENTICATION_CLASSES': ( 'funkwhale_api.common.authentication.JSONWebTokenAuthenticationQS', 'rest_framework_jwt.authentication.JSONWebTokenAuthentication', @@ -396,6 +401,9 @@ PLAYLISTS_MAX_TRACKS = env.int('PLAYLISTS_MAX_TRACKS', default=250) ACCOUNT_USERNAME_BLACKLIST = [ 'funkwhale', + 'library', + 'test', + 'status', 'root', 'admin', 'owner', diff --git a/api/config/settings/local.py b/api/config/settings/local.py index 24ad871f75ae59f98f73a91a71fed87611afa942..dcbea66d26134664655d1ca0978e3121c5da5d96 100644 --- a/api/config/settings/local.py +++ b/api/config/settings/local.py @@ -72,6 +72,10 @@ LOGGING = { 'handlers':['console'], 'propagate': True, 'level':'DEBUG', - } + }, + '': { + 'level': 'DEBUG', + 'handlers': ['console'], + }, }, } diff --git a/api/config/settings/test.py b/api/config/settings/test.py deleted file mode 100644 index aff29c6571252d1f0d401c550cedf09998a5c512..0000000000000000000000000000000000000000 --- a/api/config/settings/test.py +++ /dev/null @@ -1,29 +0,0 @@ -from .common import * # noqa -SECRET_KEY = env("DJANGO_SECRET_KEY", default='test') - -# Mail settings -# ------------------------------------------------------------------------------ -EMAIL_HOST = 'localhost' -EMAIL_PORT = 1025 -EMAIL_BACKEND = env('DJANGO_EMAIL_BACKEND', - default='django.core.mail.backends.console.EmailBackend') - -# CACHING -# ------------------------------------------------------------------------------ -CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - 'LOCATION': '' - } -} - -CELERY_BROKER_URL = 'memory://' - -########## CELERY -# In development, all tasks will be executed locally by blocking until the task returns -CELERY_TASK_ALWAYS_EAGER = True -########## END CELERY - -# Your local stuff: Below this line define 3rd party library settings -API_AUTHENTICATION_REQUIRED = False -CACHEOPS_ENABLED = False diff --git a/api/funkwhale_api/federation/activity.py b/api/funkwhale_api/federation/activity.py new file mode 100644 index 0000000000000000000000000000000000000000..4eeb193b183007ad6c7c093cd20b881a3a06b89b --- /dev/null +++ b/api/funkwhale_api/federation/activity.py @@ -0,0 +1,85 @@ +import logging +import json +import requests +import requests_http_signature + +from . import signing + +logger = logging.getLogger(__name__) + +ACTIVITY_TYPES = [ + 'Accept', + 'Add', + 'Announce', + 'Arrive', + 'Block', + 'Create', + 'Delete', + 'Dislike', + 'Flag', + 'Follow', + 'Ignore', + 'Invite', + 'Join', + 'Leave', + 'Like', + 'Listen', + 'Move', + 'Offer', + 'Question', + 'Reject', + 'Read', + 'Remove', + 'TentativeReject', + 'TentativeAccept', + 'Travel', + 'Undo', + 'Update', + 'View', +] + + +OBJECT_TYPES = [ + 'Article', + 'Audio', + 'Document', + 'Event', + 'Image', + 'Note', + 'Page', + 'Place', + 'Profile', + 'Relationship', + 'Tombstone', + 'Video', +] + +def deliver(activity, on_behalf_of, to=[]): + from . import actors + logger.info('Preparing activity delivery to %s', to) + auth = requests_http_signature.HTTPSignatureAuth( + use_auth_header=False, + headers=[ + '(request-target)', + 'user-agent', + 'host', + 'date', + 'content-type',], + algorithm='rsa-sha256', + key=on_behalf_of.private_key.encode('utf-8'), + key_id=on_behalf_of.private_key_id, + ) + for url in to: + recipient_actor = actors.get_actor(url) + logger.debug('delivering to %s', recipient_actor.inbox_url) + logger.debug('activity content: %s', json.dumps(activity)) + response = requests.post( + auth=auth, + json=activity, + url=recipient_actor.inbox_url, + headers={ + 'Content-Type': 'application/activity+json' + } + ) + response.raise_for_status() + logger.debug('Remote answered with %s', response.status_code) diff --git a/api/funkwhale_api/federation/actors.py b/api/funkwhale_api/federation/actors.py new file mode 100644 index 0000000000000000000000000000000000000000..a6220ed16440784478e303e80f9badcce73e50f2 --- /dev/null +++ b/api/funkwhale_api/federation/actors.py @@ -0,0 +1,222 @@ +import logging +import requests +import xml + +from django.conf import settings +from django.urls import reverse +from django.utils import timezone + +from rest_framework.exceptions import PermissionDenied + +from dynamic_preferences.registries import global_preferences_registry + +from . import activity +from . import factories +from . import models +from . import serializers +from . import utils + +logger = logging.getLogger(__name__) + + +def remove_tags(text): + logger.debug('Removing tags from %s', text) + return ''.join(xml.etree.ElementTree.fromstring('<div>{}</div>'.format(text)).itertext()) + + +def get_actor_data(actor_url): + response = requests.get( + actor_url, + headers={ + 'Accept': 'application/activity+json', + } + ) + response.raise_for_status() + try: + return response.json() + except: + raise ValueError( + 'Invalid actor payload: {}'.format(response.text)) + +def get_actor(actor_url): + data = get_actor_data(actor_url) + serializer = serializers.ActorSerializer(data=data) + serializer.is_valid(raise_exception=True) + + return serializer.build() + + +class SystemActor(object): + additional_attributes = {} + + def get_actor_instance(self): + a = models.Actor( + **self.get_instance_argument( + self.id, + name=self.name, + summary=self.summary, + **self.additional_attributes + ) + ) + a.pk = self.id + return a + + def get_instance_argument(self, id, name, summary, **kwargs): + preferences = global_preferences_registry.manager() + p = { + 'preferred_username': id, + 'domain': settings.FEDERATION_HOSTNAME, + 'type': 'Person', + 'name': name.format(host=settings.FEDERATION_HOSTNAME), + 'manually_approves_followers': True, + 'url': utils.full_url( + reverse( + 'federation:instance-actors-detail', + kwargs={'actor': id})), + 'shared_inbox_url': utils.full_url( + reverse( + 'federation:instance-actors-inbox', + kwargs={'actor': id})), + 'inbox_url': utils.full_url( + reverse( + 'federation:instance-actors-inbox', + kwargs={'actor': id})), + 'outbox_url': utils.full_url( + reverse( + 'federation:instance-actors-outbox', + kwargs={'actor': id})), + 'public_key': preferences['federation__public_key'], + 'private_key': preferences['federation__private_key'], + 'summary': summary.format(host=settings.FEDERATION_HOSTNAME) + } + p.update(kwargs) + return p + + def get_inbox(self, data, actor=None): + raise NotImplementedError + + def post_inbox(self, data, actor=None): + raise NotImplementedError + + def get_outbox(self, data, actor=None): + raise NotImplementedError + + def post_outbox(self, data, actor=None): + raise NotImplementedError + + +class LibraryActor(SystemActor): + id = 'library' + name = '{host}\'s library' + summary = 'Bot account to federate with {host}\'s library' + additional_attributes = { + 'manually_approves_followers': True + } + + +class TestActor(SystemActor): + id = 'test' + name = '{host}\'s test account' + summary = ( + 'Bot account to test federation with {host}. ' + 'Send me /ping and I\'ll answer you.' + ) + additional_attributes = { + 'manually_approves_followers': False + } + + def get_outbox(self, data, actor=None): + return { + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + {} + ], + "id": utils.full_url( + reverse( + 'federation:instance-actors-outbox', + kwargs={'actor': self.id})), + "type": "OrderedCollection", + "totalItems": 0, + "orderedItems": [] + } + + def post_inbox(self, data, actor=None): + if actor is None: + raise PermissionDenied('Actor not authenticated') + + serializer = serializers.ActivitySerializer( + data=data, context={'actor': actor}) + serializer.is_valid(raise_exception=True) + + ac = serializer.validated_data + logger.info('Received activity on %s inbox', self.id) + if ac['type'] == 'Create' and ac['object']['type'] == 'Note': + # we received a toot \o/ + command = self.parse_command(ac['object']['content']) + logger.debug('Parsed command: %s', command) + if command == 'ping': + self.handle_ping(ac, actor) + + def parse_command(self, message): + """ + Remove any links or fancy markup to extract /command from + a note message. + """ + raw = remove_tags(message) + try: + return raw.split('/')[1] + except IndexError: + return + + def handle_ping(self, ac, sender): + now = timezone.now() + test_actor = self.get_actor_instance() + reply_url = 'https://{}/activities/note/{}'.format( + settings.FEDERATION_HOSTNAME, now.timestamp() + ) + reply_content = '{} Pong!'.format( + sender.mention_username + ) + reply_activity = { + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + {} + ], + 'type': 'Create', + 'actor': test_actor.url, + 'id': '{}/activity'.format(reply_url), + 'published': now.isoformat(), + 'to': ac['actor'], + 'cc': [], + 'object': factories.NoteFactory( + content='Pong!', + summary=None, + published=now.isoformat(), + id=reply_url, + inReplyTo=ac['object']['id'], + sensitive=False, + url=reply_url, + to=[ac['actor']], + attributedTo=test_actor.url, + cc=[], + attachment=[], + tag=[ + { + "type": "Mention", + "href": ac['actor'], + "name": sender.mention_username + } + ] + ) + } + activity.deliver( + reply_activity, + to=[ac['actor']], + on_behalf_of=test_actor) + +SYSTEM_ACTORS = { + 'library': LibraryActor(), + 'test': TestActor(), +} diff --git a/api/funkwhale_api/federation/authentication.py b/api/funkwhale_api/federation/authentication.py new file mode 100644 index 0000000000000000000000000000000000000000..e199ef134d03e0d7026ecffbbaaa1f38e8254e02 --- /dev/null +++ b/api/funkwhale_api/federation/authentication.py @@ -0,0 +1,52 @@ +import cryptography + +from django.contrib.auth.models import AnonymousUser + +from rest_framework import authentication +from rest_framework import exceptions + +from . import actors +from . import keys +from . import serializers +from . import signing +from . import utils + + +class SignatureAuthentication(authentication.BaseAuthentication): + def authenticate_actor(self, request): + headers = utils.clean_wsgi_headers(request.META) + try: + signature = headers['Signature'] + key_id = keys.get_key_id_from_signature_header(signature) + except KeyError: + return + except ValueError as e: + raise exceptions.AuthenticationFailed(str(e)) + + try: + actor_data = actors.get_actor_data(key_id) + except Exception as e: + raise exceptions.AuthenticationFailed(str(e)) + + try: + public_key = actor_data['publicKey']['publicKeyPem'] + except KeyError: + raise exceptions.AuthenticationFailed('No public key found') + + serializer = serializers.ActorSerializer(data=actor_data) + if not serializer.is_valid(): + raise exceptions.AuthenticationFailed('Invalid actor payload: {}'.format(serializer.errors)) + + try: + signing.verify_django(request, public_key.encode('utf-8')) + except cryptography.exceptions.InvalidSignature: + raise exceptions.AuthenticationFailed('Invalid signature') + + return serializer.build() + + def authenticate(self, request): + setattr(request, 'actor', None) + actor = self.authenticate_actor(request) + user = AnonymousUser() + setattr(request, 'actor', actor) + return (user, None) diff --git a/api/funkwhale_api/federation/exceptions.py b/api/funkwhale_api/federation/exceptions.py index 96fd24a7ed7eb2962b52aaccb578957911647249..31d864b36c065aacadd8b09ed02278e13bd46fcb 100644 --- a/api/funkwhale_api/federation/exceptions.py +++ b/api/funkwhale_api/federation/exceptions.py @@ -2,3 +2,7 @@ class MalformedPayload(ValueError): pass + + +class MissingSignature(KeyError): + pass diff --git a/api/funkwhale_api/federation/factories.py b/api/funkwhale_api/federation/factories.py index f5d612b0dad5133197fb4385b583833539920126..88c86f791937cd66fbcc72a42592b520078c8a7c 100644 --- a/api/funkwhale_api/federation/factories.py +++ b/api/funkwhale_api/federation/factories.py @@ -2,9 +2,12 @@ import factory import requests import requests_http_signature +from django.utils import timezone + from funkwhale_api.factories import registry from . import keys +from . import models registry.register(keys.get_key_pair, name='federation.KeyPair') @@ -15,7 +18,13 @@ class SignatureAuthFactory(factory.Factory): algorithm = 'rsa-sha256' key = factory.LazyFunction(lambda: keys.get_key_pair()[0]) key_id = factory.Faker('url') - + use_auth_header = False + headers = [ + '(request-target)', + 'user-agent', + 'host', + 'date', + 'content-type',] class Meta: model = requests_http_signature.HTTPSignatureAuth @@ -28,3 +37,55 @@ class SignedRequestFactory(factory.Factory): class Meta: model = requests.Request + + @factory.post_generation + def headers(self, create, extracted, **kwargs): + default_headers = { + 'User-Agent': 'Test', + 'Host': 'test.host', + 'Date': 'Right now', + 'Content-Type': 'application/activity+json' + } + if extracted: + default_headers.update(extracted) + self.headers.update(default_headers) + + +@registry.register +class ActorFactory(factory.DjangoModelFactory): + + public_key = None + private_key = None + preferred_username = factory.Faker('user_name') + summary = factory.Faker('paragraph') + domain = factory.Faker('domain_name') + url = factory.LazyAttribute(lambda o: 'https://{}/users/{}'.format(o.domain, o.preferred_username)) + inbox_url = factory.LazyAttribute(lambda o: 'https://{}/users/{}/inbox'.format(o.domain, o.preferred_username)) + outbox_url = factory.LazyAttribute(lambda o: 'https://{}/users/{}/outbox'.format(o.domain, o.preferred_username)) + + class Meta: + model = models.Actor + + @classmethod + def _generate(cls, create, attrs): + has_public = attrs.get('public_key') is not None + has_private = attrs.get('private_key') is not None + if not has_public and not has_private: + private, public = keys.get_key_pair() + attrs['private_key'] = private.decode('utf-8') + attrs['public_key'] = public.decode('utf-8') + return super()._generate(create, attrs) + + +@registry.register(name='federation.Note') +class NoteFactory(factory.Factory): + type = 'Note' + id = factory.Faker('url') + published = factory.LazyFunction( + lambda: timezone.now().isoformat() + ) + inReplyTo = None + content = factory.Faker('sentence') + + class Meta: + model = dict diff --git a/api/funkwhale_api/federation/keys.py b/api/funkwhale_api/federation/keys.py index 432560ef7446d5973d23850cff7a2cb253c9fba7..08d4034ea347a6a17bb5d5701217d54cc1c57fa0 100644 --- a/api/funkwhale_api/federation/keys.py +++ b/api/funkwhale_api/federation/keys.py @@ -2,10 +2,14 @@ from cryptography.hazmat.primitives import serialization as crypto_serialization from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.backends import default_backend as crypto_default_backend +import re import requests +import urllib.parse from . import exceptions +KEY_ID_REGEX = re.compile(r'keyId=\"(?P<id>.*)\"') + def get_key_pair(size=2048): key = rsa.generate_private_key( @@ -25,19 +29,21 @@ def get_key_pair(size=2048): return private_key, public_key -def get_public_key(actor_url): - """ - Given an actor_url, request it and extract publicKey data from - the response payload. - """ - response = requests.get(actor_url) - response.raise_for_status() - payload = response.json() +def get_key_id_from_signature_header(header_string): + parts = header_string.split(',') try: - return { - 'public_key_pem': payload['publicKey']['publicKeyPem'], - 'id': payload['publicKey']['id'], - 'owner': payload['publicKey']['owner'], - } - except KeyError: - raise exceptions.MalformedPayload(str(payload)) + raw_key_id = [p for p in parts if p.startswith('keyId="')][0] + except IndexError: + raise ValueError('Missing key id') + + match = KEY_ID_REGEX.match(raw_key_id) + if not match: + raise ValueError('Invalid key id') + + key_id = match.groups()[0] + url = urllib.parse.urlparse(key_id) + if not url.scheme or not url.netloc: + raise ValueError('Invalid url') + if url.scheme not in ['http', 'https']: + raise ValueError('Invalid shceme') + return key_id diff --git a/api/funkwhale_api/federation/migrations/0001_initial.py b/api/funkwhale_api/federation/migrations/0001_initial.py new file mode 100644 index 0000000000000000000000000000000000000000..a9157e57e3fa123c5e79b2d23176f7cd744ee926 --- /dev/null +++ b/api/funkwhale_api/federation/migrations/0001_initial.py @@ -0,0 +1,37 @@ +# Generated by Django 2.0.3 on 2018-03-31 13:43 + +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Actor', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('url', models.URLField(db_index=True, max_length=500, unique=True)), + ('outbox_url', models.URLField(max_length=500)), + ('inbox_url', models.URLField(max_length=500)), + ('following_url', models.URLField(blank=True, max_length=500, null=True)), + ('followers_url', models.URLField(blank=True, max_length=500, null=True)), + ('shared_inbox_url', models.URLField(blank=True, max_length=500, null=True)), + ('type', models.CharField(choices=[('Person', 'Person'), ('Application', 'Application'), ('Group', 'Group'), ('Organization', 'Organization'), ('Service', 'Service')], default='Person', max_length=25)), + ('name', models.CharField(blank=True, max_length=200, null=True)), + ('domain', models.CharField(max_length=1000)), + ('summary', models.CharField(blank=True, max_length=500, null=True)), + ('preferred_username', models.CharField(blank=True, max_length=200, null=True)), + ('public_key', models.CharField(blank=True, max_length=5000, null=True)), + ('private_key', models.CharField(blank=True, max_length=5000, null=True)), + ('creation_date', models.DateTimeField(default=django.utils.timezone.now)), + ('last_fetch_date', models.DateTimeField(default=django.utils.timezone.now)), + ('manually_approves_followers', models.NullBooleanField(default=None)), + ], + ), + ] diff --git a/api/funkwhale_api/federation/migrations/__init__.py b/api/funkwhale_api/federation/migrations/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/api/funkwhale_api/federation/models.py b/api/funkwhale_api/federation/models.py new file mode 100644 index 0000000000000000000000000000000000000000..d76ad173be80c0d37dc063a7b0d829952f9e16db --- /dev/null +++ b/api/funkwhale_api/federation/models.py @@ -0,0 +1,59 @@ +from django.conf import settings +from django.db import models +from django.utils import timezone + +TYPE_CHOICES = [ + ('Person', 'Person'), + ('Application', 'Application'), + ('Group', 'Group'), + ('Organization', 'Organization'), + ('Service', 'Service'), +] + + +class Actor(models.Model): + url = models.URLField(unique=True, max_length=500, db_index=True) + outbox_url = models.URLField(max_length=500) + inbox_url = models.URLField(max_length=500) + following_url = models.URLField(max_length=500, null=True, blank=True) + followers_url = models.URLField(max_length=500, null=True, blank=True) + shared_inbox_url = models.URLField(max_length=500, null=True, blank=True) + type = models.CharField( + choices=TYPE_CHOICES, default='Person', max_length=25) + name = models.CharField(max_length=200, null=True, blank=True) + domain = models.CharField(max_length=1000) + summary = models.CharField(max_length=500, null=True, blank=True) + preferred_username = models.CharField( + max_length=200, null=True, blank=True) + public_key = models.CharField(max_length=5000, null=True, blank=True) + private_key = models.CharField(max_length=5000, null=True, blank=True) + creation_date = models.DateTimeField(default=timezone.now) + last_fetch_date = models.DateTimeField( + default=timezone.now) + manually_approves_followers = models.NullBooleanField(default=None) + + @property + def webfinger_subject(self): + return '{}@{}'.format( + self.preferred_username, + settings.FEDERATION_HOSTNAME, + ) + + @property + def private_key_id(self): + return '{}#main-key'.format(self.url) + + @property + def mention_username(self): + return '@{}@{}'.format(self.preferred_username, self.domain) + + def save(self, **kwargs): + lowercase_fields = [ + 'domain', + ] + for field in lowercase_fields: + v = getattr(self, field, None) + if v: + setattr(self, field, v.lower()) + + super().save(**kwargs) diff --git a/api/funkwhale_api/federation/parsers.py b/api/funkwhale_api/federation/parsers.py new file mode 100644 index 0000000000000000000000000000000000000000..874d808f973dfcdf99372332b3991136c44d8605 --- /dev/null +++ b/api/funkwhale_api/federation/parsers.py @@ -0,0 +1,5 @@ +from rest_framework import parsers + + +class ActivityParser(parsers.JSONParser): + media_type = 'application/activity+json' diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py index d1533b62d626d5451cc775646d0bbc9f9f0c33a9..2137e8d910373e0c8d23b3e38c0d9952e4a93787 100644 --- a/api/funkwhale_api/federation/serializers.py +++ b/api/funkwhale_api/federation/serializers.py @@ -1,38 +1,175 @@ +import urllib.parse + from django.urls import reverse from django.conf import settings +from rest_framework import serializers from dynamic_preferences.registries import global_preferences_registry +from . import activity +from . import models from . import utils -def repr_instance_actor(): - """ - We do not use a serializer here, since it's pretty static - """ - actor_url = utils.full_url(reverse('federation:instance-actor')) - preferences = global_preferences_registry.manager() - public_key = preferences['federation__public_key'] +class ActorSerializer(serializers.ModelSerializer): + # left maps to activitypub fields, right to our internal models + id = serializers.URLField(source='url') + outbox = serializers.URLField(source='outbox_url') + inbox = serializers.URLField(source='inbox_url') + following = serializers.URLField(source='following_url', required=False) + followers = serializers.URLField(source='followers_url', required=False) + preferredUsername = serializers.CharField( + source='preferred_username', required=False) + publicKey = serializers.JSONField(source='public_key', required=False) + manuallyApprovesFollowers = serializers.NullBooleanField( + source='manually_approves_followers', required=False) + summary = serializers.CharField(max_length=None, required=False) + + class Meta: + model = models.Actor + fields = [ + 'id', + 'type', + 'name', + 'summary', + 'preferredUsername', + 'publicKey', + 'inbox', + 'outbox', + 'following', + 'followers', + 'manuallyApprovesFollowers', + ] - return { - '@context': [ + def to_representation(self, instance): + ret = super().to_representation(instance) + ret['@context'] = [ 'https://www.w3.org/ns/activitystreams', 'https://w3id.org/security/v1', {}, - ], - 'id': utils.full_url(reverse('federation:instance-actor')), - 'type': 'Person', - 'inbox': utils.full_url(reverse('federation:instance-inbox')), - 'outbox': utils.full_url(reverse('federation:instance-outbox')), - 'preferredUsername': 'service', - 'name': 'Service Bot - {}'.format(settings.FEDERATION_HOSTNAME), - 'summary': 'Bot account for federating with {}'.format( - settings.FEDERATION_HOSTNAME - ), - 'publicKey': { - 'id': '{}#main-key'.format(actor_url), - 'owner': actor_url, - 'publicKeyPem': public_key - }, - - } + ] + if instance.public_key: + ret['publicKey'] = { + 'owner': instance.url, + 'publicKeyPem': instance.public_key, + 'id': '{}#main-key'.format(instance.url) + } + ret['endpoints'] = {} + if instance.shared_inbox_url: + ret['endpoints']['sharedInbox'] = instance.shared_inbox_url + return ret + + def prepare_missing_fields(self): + kwargs = {} + domain = urllib.parse.urlparse(self.validated_data['url']).netloc + kwargs['domain'] = domain + for endpoint, url in self.initial_data.get('endpoints', {}).items(): + if endpoint == 'sharedInbox': + kwargs['shared_inbox_url'] = url + break + try: + kwargs['public_key'] = self.initial_data['publicKey']['publicKeyPem'] + except KeyError: + pass + return kwargs + + def build(self): + d = self.validated_data.copy() + d.update(self.prepare_missing_fields()) + return self.Meta.model(**d) + + def save(self, **kwargs): + kwargs.update(self.prepare_missing_fields()) + return super().save(**kwargs) + + def validate_summary(self, value): + if value: + return value[:500] + + +class ActorWebfingerSerializer(serializers.ModelSerializer): + class Meta: + model = models.Actor + fields = ['url'] + + def to_representation(self, instance): + data = {} + data['subject'] = 'acct:{}'.format(instance.webfinger_subject) + data['links'] = [ + { + 'rel': 'self', + 'href': instance.url, + 'type': 'application/activity+json' + } + ] + data['aliases'] = [ + instance.url + ] + return data + + +class ActivitySerializer(serializers.Serializer): + actor = serializers.URLField() + id = serializers.URLField() + type = serializers.ChoiceField( + choices=[(c, c) for c in activity.ACTIVITY_TYPES]) + object = serializers.JSONField() + + def validate_object(self, value): + try: + type = value['type'] + except KeyError: + raise serializers.ValidationError('Missing object type') + + try: + object_serializer = OBJECT_SERIALIZERS[type] + except KeyError: + raise serializers.ValidationError( + 'Unsupported type {}'.format(type)) + + serializer = object_serializer(data=value) + serializer.is_valid(raise_exception=True) + return serializer.data + + def validate_actor(self, value): + request_actor = self.context.get('actor') + if request_actor and request_actor.url != value: + raise serializers.ValidationError( + 'The actor making the request do not match' + ' the activity actor' + ) + return value + + +class ObjectSerializer(serializers.Serializer): + id = serializers.URLField() + url = serializers.URLField(required=False, allow_null=True) + type = serializers.ChoiceField( + choices=[(c, c) for c in activity.OBJECT_TYPES]) + content = serializers.CharField( + required=False, allow_null=True) + summary = serializers.CharField( + required=False, allow_null=True) + name = serializers.CharField( + required=False, allow_null=True) + published = serializers.DateTimeField( + required=False, allow_null=True) + updated = serializers.DateTimeField( + required=False, allow_null=True) + to = serializers.ListField( + child=serializers.URLField(), + required=False, allow_null=True) + cc = serializers.ListField( + child=serializers.URLField(), + required=False, allow_null=True) + bto = serializers.ListField( + child=serializers.URLField(), + required=False, allow_null=True) + bcc = serializers.ListField( + child=serializers.URLField(), + required=False, allow_null=True) + +OBJECT_SERIALIZERS = { + t: ObjectSerializer + for t in activity.OBJECT_TYPES +} diff --git a/api/funkwhale_api/federation/signing.py b/api/funkwhale_api/federation/signing.py index 87ac82bac0b9a5b599ce7d3721e8f4f876c02e8c..7e4d2aa5ae08748ea5b6975aa7345d33f993ab1c 100644 --- a/api/funkwhale_api/federation/signing.py +++ b/api/funkwhale_api/federation/signing.py @@ -1,11 +1,18 @@ +import logging import requests import requests_http_signature +from . import exceptions +from . import utils + +logger = logging.getLogger(__name__) + def verify(request, public_key): return requests_http_signature.HTTPSignatureAuth.verify( request, - key_resolver=lambda **kwargs: public_key + key_resolver=lambda **kwargs: public_key, + use_auth_header=False, ) @@ -14,21 +21,35 @@ def verify_django(django_request, public_key): Given a django WSGI request, create an underlying requests.PreparedRequest instance we can verify """ - headers = django_request.META.get('headers', {}).copy() + headers = utils.clean_wsgi_headers(django_request.META) for h, v in list(headers.items()): # we include lower-cased version of the headers for compatibility # with requests_http_signature headers[h.lower()] = v try: - signature = headers['authorization'] + signature = headers['Signature'] except KeyError: raise exceptions.MissingSignature - + url = 'http://noop{}'.format(django_request.path) + query = django_request.META['QUERY_STRING'] + if query: + url += '?{}'.format(query) + signature_headers = signature.split('headers="')[1].split('",')[0] + expected = signature_headers.split(' ') + logger.debug('Signature expected headers: %s', expected) + for header in expected: + try: + headers[header] + except KeyError: + logger.debug('Missing header: %s', header) request = requests.Request( method=django_request.method, - url='http://noop', + url=url, data=django_request.body, headers=headers) - + for h in request.headers.keys(): + v = request.headers[h] + if v: + request.headers[h] = str(v) prepared_request = request.prepare() return verify(request, public_key) diff --git a/api/funkwhale_api/federation/urls.py b/api/funkwhale_api/federation/urls.py index 5b7895451f815924589d1864b30e5ebd63166481..f2c6f4c78c61973436b3d92aacebdec3506156fd 100644 --- a/api/funkwhale_api/federation/urls.py +++ b/api/funkwhale_api/federation/urls.py @@ -4,9 +4,9 @@ from . import views router = routers.SimpleRouter(trailing_slash=False) router.register( - r'federation/instance', - views.InstanceViewSet, - 'instance') + r'federation/instance/actors', + views.InstanceActorViewSet, + 'instance-actors') router.register( r'.well-known', views.WellKnownViewSet, diff --git a/api/funkwhale_api/federation/utils.py b/api/funkwhale_api/federation/utils.py index e83f54b5d54c79d684d80c057e7e0fbb7cd5d1f1..df093add8f934bbaabae8f6e9e68712f3762c53d 100644 --- a/api/funkwhale_api/federation/utils.py +++ b/api/funkwhale_api/federation/utils.py @@ -12,3 +12,24 @@ def full_url(path): return root + '/' + path else: return root + path + + +def clean_wsgi_headers(raw_headers): + """ + Convert WSGI headers from CONTENT_TYPE to Content-Type notation + """ + cleaned = {} + non_prefixed = [ + 'content_type', + 'content_length', + ] + for raw_header, value in raw_headers.items(): + h = raw_header.lower() + if not h.startswith('http_') and h not in non_prefixed: + continue + + words = h.replace('http_', '', 1).split('_') + cleaned_header = '-'.join([w.capitalize() for w in words]) + cleaned[cleaned_header] = value + + return cleaned diff --git a/api/funkwhale_api/federation/views.py b/api/funkwhale_api/federation/views.py index 5f1ee36f76fada4bc3aa90e6941958fe1781f7a1..2e3feb8d082ebdca00917e59e13d5f3cc601eb33 100644 --- a/api/funkwhale_api/federation/views.py +++ b/api/funkwhale_api/federation/views.py @@ -5,8 +5,10 @@ from django.http import HttpResponse from rest_framework import viewsets from rest_framework import views from rest_framework import response -from rest_framework.decorators import list_route +from rest_framework.decorators import list_route, detail_route +from . import actors +from . import authentication from . import renderers from . import serializers from . import webfinger @@ -19,22 +21,50 @@ class FederationMixin(object): return super().dispatch(request, *args, **kwargs) -class InstanceViewSet(FederationMixin, viewsets.GenericViewSet): - authentication_classes = [] +class InstanceActorViewSet(FederationMixin, viewsets.GenericViewSet): + lookup_field = 'actor' + lookup_value_regex = '[a-z]*' + authentication_classes = [ + authentication.SignatureAuthentication] permission_classes = [] renderer_classes = [renderers.ActivityPubRenderer] - @list_route(methods=['get']) - def actor(self, request, *args, **kwargs): - return response.Response(serializers.repr_instance_actor()) + def get_object(self): + try: + return actors.SYSTEM_ACTORS[self.kwargs['actor']] + except KeyError: + raise Http404 - @list_route(methods=['get']) + def retrieve(self, request, *args, **kwargs): + system_actor = self.get_object() + actor = system_actor.get_actor_instance() + serializer = serializers.ActorSerializer(actor) + return response.Response(serializer.data, status=200) + + @detail_route(methods=['get', 'post']) def inbox(self, request, *args, **kwargs): - raise NotImplementedError() + system_actor = self.get_object() + handler = getattr(system_actor, '{}_inbox'.format( + request.method.lower() + )) - @list_route(methods=['get']) + try: + data = handler(request.data, actor=request.actor) + except NotImplementedError: + return response.Response(status=405) + return response.Response(data, status=200) + + @detail_route(methods=['get', 'post']) def outbox(self, request, *args, **kwargs): - raise NotImplementedError() + system_actor = self.get_object() + handler = getattr(system_actor, '{}_outbox'.format( + request.method.lower() + )) + try: + data = handler(request.data, actor=request.actor) + except NotImplementedError: + return response.Response(status=405) + return response.Response(data, status=200) class WellKnownViewSet(FederationMixin, viewsets.GenericViewSet): @@ -69,6 +99,5 @@ class WellKnownViewSet(FederationMixin, viewsets.GenericViewSet): def handler_acct(self, clean_result): username, hostname = clean_result - if username == 'service': - return webfinger.serialize_system_acct() - return {} + actor = actors.SYSTEM_ACTORS[username].get_actor_instance() + return serializers.ActorWebfingerSerializer(actor).data diff --git a/api/funkwhale_api/federation/webfinger.py b/api/funkwhale_api/federation/webfinger.py index a9281c2b596899b7094ca659994441f9212f24e8..4e97533852421210aead271aa510c40d29478e6e 100644 --- a/api/funkwhale_api/federation/webfinger.py +++ b/api/funkwhale_api/federation/webfinger.py @@ -2,7 +2,9 @@ from django import forms from django.conf import settings from django.urls import reverse +from . import actors from . import utils + VALID_RESOURCE_TYPES = ['acct'] @@ -27,26 +29,11 @@ def clean_acct(acct_string): except ValueError: raise forms.ValidationError('Invalid format') - if hostname != settings.FEDERATION_HOSTNAME: - raise forms.ValidationError('Invalid hostname') + if hostname.lower() != settings.FEDERATION_HOSTNAME: + raise forms.ValidationError( + 'Invalid hostname {}'.format(hostname)) - if username != 'service': + if username not in actors.SYSTEM_ACTORS: raise forms.ValidationError('Invalid username') return username, hostname - - -def serialize_system_acct(): - return { - 'subject': 'acct:service@{}'.format(settings.FEDERATION_HOSTNAME), - 'aliases': [ - utils.full_url(reverse('federation:instance-actor')) - ], - 'links': [ - { - 'rel': 'self', - 'type': 'application/activity+json', - 'href': utils.full_url(reverse('federation:instance-actor')), - } - ] - } diff --git a/api/requirements/base.txt b/api/requirements/base.txt index 02cf1c70224c8387fdffc052a937b8d8397f9c23..b66e297a9942524b02df71fdcc67c81225e62c1e 100644 --- a/api/requirements/base.txt +++ b/api/requirements/base.txt @@ -61,4 +61,6 @@ django-cacheops>=4,<4.1 daphne==2.0.4 cryptography>=2,<3 -requests-http-signature==0.0.3 +# requests-http-signature==0.0.3 +# clone until the branch is merged and released upstream +git+https://github.com/EliotBerriot/requests-http-signature.git@signature-header-support diff --git a/api/requirements/test.txt b/api/requirements/test.txt index e11f26ca75ad36ab5efe1f91d86cd51b04928f9b..20a14abea1685387764012ea5ac3e01d60a09a47 100644 --- a/api/requirements/test.txt +++ b/api/requirements/test.txt @@ -10,4 +10,5 @@ pytest-mock pytest-sugar pytest-xdist pytest-cov +pytest-env requests-mock diff --git a/api/setup.cfg b/api/setup.cfg index 34daa8c6834452229971467c7876400b842b64c1..a2b8b92c682696a0ad4569cb6c9f9e25c01f9b9f 100644 --- a/api/setup.cfg +++ b/api/setup.cfg @@ -7,6 +7,12 @@ max-line-length = 120 exclude=.tox,.git,*/migrations/*,*/static/CACHE/*,docs,node_modules [tool:pytest] -DJANGO_SETTINGS_MODULE=config.settings.test python_files = tests.py test_*.py *_tests.py testpaths = tests +env = + SECRET_KEY=test + DJANGO_EMAIL_BACKEND=django.core.mail.backends.console.EmailBackend + CELERY_BROKER_URL=memory:// + CELERY_TASK_ALWAYS_EAGER=True + CACHEOPS_ENABLED=False + FEDERATION_HOSTNAME=test.federation diff --git a/api/tests/conftest.py b/api/tests/conftest.py index 2b5a4f799d0fd23cba94372bdf7415a373eafd66..d5bb565651c4b1282920fa455356db4bf6704c35 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -1,11 +1,13 @@ import factory -import tempfile -import shutil import pytest import requests_mock +import shutil +import tempfile from django.contrib.auth.models import AnonymousUser from django.core.cache import cache as django_cache +from django.test import client + from dynamic_preferences.registries import global_preferences_registry from rest_framework.test import APIClient @@ -118,6 +120,11 @@ def api_request(): return APIRequestFactory() +@pytest.fixture +def fake_request(): + return client.RequestFactory() + + @pytest.fixture def activity_registry(): r = record.registry diff --git a/api/tests/federation/conftest.py b/api/tests/federation/conftest.py new file mode 100644 index 0000000000000000000000000000000000000000..c5831914bef6a59ddc80e88731c29127ec7b38b3 --- /dev/null +++ b/api/tests/federation/conftest.py @@ -0,0 +1,10 @@ +import pytest + + +@pytest.fixture +def authenticated_actor(nodb_factories, mocker): + actor = nodb_factories['federation.Actor']() + mocker.patch( + 'funkwhale_api.federation.authentication.SignatureAuthentication.authenticate_actor', + return_value=actor) + yield actor diff --git a/api/tests/federation/test_activity.py b/api/tests/federation/test_activity.py new file mode 100644 index 0000000000000000000000000000000000000000..a6e1d28aa23623251a1ab26661d2814c14704f00 --- /dev/null +++ b/api/tests/federation/test_activity.py @@ -0,0 +1,32 @@ +from funkwhale_api.federation import activity + +def test_deliver(nodb_factories, r_mock, mocker): + to = nodb_factories['federation.Actor']() + mocker.patch( + 'funkwhale_api.federation.actors.get_actor', + return_value=to) + sender = nodb_factories['federation.Actor']() + ac = { + 'id': 'http://test.federation/activity', + 'type': 'Create', + 'actor': sender.url, + 'object': { + 'id': 'http://test.federation/note', + 'type': 'Note', + 'content': 'Hello', + } + } + + r_mock.post(to.inbox_url) + + activity.deliver( + ac, + to=[to.url], + on_behalf_of=sender, + ) + request = r_mock.request_history[0] + + assert r_mock.called is True + assert r_mock.call_count == 1 + assert request.url == to.inbox_url + assert request.headers['content-type'] == 'application/activity+json' diff --git a/api/tests/federation/test_actors.py b/api/tests/federation/test_actors.py new file mode 100644 index 0000000000000000000000000000000000000000..b3b0f8df0bb7fc3b8295bb48b6b8bd6ad967c01f --- /dev/null +++ b/api/tests/federation/test_actors.py @@ -0,0 +1,190 @@ +import pytest + +from django.urls import reverse +from django.utils import timezone + +from rest_framework import exceptions + +from funkwhale_api.federation import actors +from funkwhale_api.federation import serializers +from funkwhale_api.federation import utils + + +def test_actor_fetching(r_mock): + payload = { + 'id': 'https://actor.mock/users/actor#main-key', + 'owner': 'test', + 'publicKeyPem': 'test_pem', + } + actor_url = 'https://actor.mock/' + r_mock.get(actor_url, json=payload) + r = actors.get_actor_data(actor_url) + + assert r == payload + + +def test_get_library(settings, preferences): + preferences['federation__public_key'] = 'public_key' + expected = { + 'preferred_username': 'library', + 'domain': settings.FEDERATION_HOSTNAME, + 'type': 'Person', + 'name': '{}\'s library'.format(settings.FEDERATION_HOSTNAME), + 'manually_approves_followers': True, + 'url': utils.full_url( + reverse( + 'federation:instance-actors-detail', + kwargs={'actor': 'library'})), + 'shared_inbox_url': utils.full_url( + reverse( + 'federation:instance-actors-inbox', + kwargs={'actor': 'library'})), + 'inbox_url': utils.full_url( + reverse( + 'federation:instance-actors-inbox', + kwargs={'actor': 'library'})), + 'outbox_url': utils.full_url( + reverse( + 'federation:instance-actors-outbox', + kwargs={'actor': 'library'})), + 'public_key': 'public_key', + 'summary': 'Bot account to federate with {}\'s library'.format( + settings.FEDERATION_HOSTNAME), + } + actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + for key, value in expected.items(): + assert getattr(actor, key) == value + + +def test_get_test(settings, preferences): + preferences['federation__public_key'] = 'public_key' + expected = { + 'preferred_username': 'test', + 'domain': settings.FEDERATION_HOSTNAME, + 'type': 'Person', + 'name': '{}\'s test account'.format(settings.FEDERATION_HOSTNAME), + 'manually_approves_followers': False, + 'url': utils.full_url( + reverse( + 'federation:instance-actors-detail', + kwargs={'actor': 'test'})), + 'shared_inbox_url': utils.full_url( + reverse( + 'federation:instance-actors-inbox', + kwargs={'actor': 'test'})), + 'inbox_url': utils.full_url( + reverse( + 'federation:instance-actors-inbox', + kwargs={'actor': 'test'})), + 'outbox_url': utils.full_url( + reverse( + 'federation:instance-actors-outbox', + kwargs={'actor': 'test'})), + 'public_key': 'public_key', + 'summary': 'Bot account to test federation with {}. Send me /ping and I\'ll answer you.'.format( + settings.FEDERATION_HOSTNAME), + } + actor = actors.SYSTEM_ACTORS['test'].get_actor_instance() + for key, value in expected.items(): + assert getattr(actor, key) == value + + +def test_test_get_outbox(): + expected = { + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + {} + ], + "id": utils.full_url( + reverse( + 'federation:instance-actors-outbox', + kwargs={'actor': 'test'})), + "type": "OrderedCollection", + "totalItems": 0, + "orderedItems": [] + } + + data = actors.SYSTEM_ACTORS['test'].get_outbox({}, actor=None) + + assert data == expected + + +def test_test_post_inbox_requires_authenticated_actor(): + with pytest.raises(exceptions.PermissionDenied): + actors.SYSTEM_ACTORS['test'].post_inbox({}, actor=None) + + +def test_test_post_outbox_validates_actor(nodb_factories): + actor = nodb_factories['federation.Actor']() + data = { + 'actor': 'noop' + } + with pytest.raises(exceptions.ValidationError) as exc_info: + actors.SYSTEM_ACTORS['test'].post_inbox(data, actor=actor) + msg = 'The actor making the request do not match' + assert msg in exc_info.value + + +def test_test_post_outbox_handles_create_note( + settings, mocker, factories): + deliver = mocker.patch( + 'funkwhale_api.federation.activity.deliver') + actor = factories['federation.Actor']() + now = timezone.now() + mocker.patch('django.utils.timezone.now', return_value=now) + data = { + 'actor': actor.url, + 'type': 'Create', + 'id': 'http://test.federation/activity', + 'object': { + 'type': 'Note', + 'id': 'http://test.federation/object', + 'content': '<p><a>@mention</a> /ping</p>' + } + } + test_actor = actors.SYSTEM_ACTORS['test'].get_actor_instance() + expected_note = factories['federation.Note']( + id='https://test.federation/activities/note/{}'.format( + now.timestamp() + ), + content='Pong!', + published=now.isoformat(), + inReplyTo=data['object']['id'], + cc=[], + summary=None, + sensitive=False, + attributedTo=test_actor.url, + attachment=[], + to=[actor.url], + url='https://{}/activities/note/{}'.format( + settings.FEDERATION_HOSTNAME, now.timestamp() + ), + tag=[{ + 'href': actor.url, + 'name': actor.mention_username, + 'type': 'Mention', + }] + ) + expected_activity = { + '@context': [ + 'https://www.w3.org/ns/activitystreams', + 'https://w3id.org/security/v1', + {} + ], + 'actor': test_actor.url, + 'id': 'https://{}/activities/note/{}/activity'.format( + settings.FEDERATION_HOSTNAME, now.timestamp() + ), + 'to': actor.url, + 'type': 'Create', + 'published': now.isoformat(), + 'object': expected_note, + 'cc': [], + } + actors.SYSTEM_ACTORS['test'].post_inbox(data, actor=actor) + deliver.assert_called_once_with( + expected_activity, + to=[actor.url], + on_behalf_of=actors.SYSTEM_ACTORS['test'].get_actor_instance() + ) diff --git a/api/tests/federation/test_authentication.py b/api/tests/federation/test_authentication.py new file mode 100644 index 0000000000000000000000000000000000000000..1837b3950f471ba549d8f45077d5b773cc010da3 --- /dev/null +++ b/api/tests/federation/test_authentication.py @@ -0,0 +1,42 @@ +from funkwhale_api.federation import authentication +from funkwhale_api.federation import keys +from funkwhale_api.federation import signing + + +def test_authenticate(nodb_factories, mocker, api_request): + private, public = keys.get_key_pair() + actor_url = 'https://test.federation/actor' + mocker.patch( + 'funkwhale_api.federation.actors.get_actor_data', + return_value={ + 'id': actor_url, + 'outbox': 'https://test.com', + 'inbox': 'https://test.com', + 'publicKey': { + 'publicKeyPem': public.decode('utf-8'), + 'owner': actor_url, + 'id': actor_url + '#main-key', + } + }) + signed_request = nodb_factories['federation.SignedRequest']( + auth__key=private, + auth__key_id=actor_url + '#main-key', + auth__headers=[ + 'date', + ] + ) + prepared = signed_request.prepare() + django_request = api_request.get( + '/', + **{ + 'HTTP_DATE': prepared.headers['date'], + 'HTTP_SIGNATURE': prepared.headers['signature'], + } + ) + authenticator = authentication.SignatureAuthentication() + user, _ = authenticator.authenticate(django_request) + actor = django_request.actor + + assert user.is_anonymous is True + assert actor.public_key == public.decode('utf-8') + assert actor.url == actor_url diff --git a/api/tests/federation/test_keys.py b/api/tests/federation/test_keys.py index 1c30c30b17ed6450e7ec133498f3788be16a9ee6..9dd71be092bc39dcfb09f9b0e931c51a9ea37f5c 100644 --- a/api/tests/federation/test_keys.py +++ b/api/tests/federation/test_keys.py @@ -1,16 +1,25 @@ +import pytest + from funkwhale_api.federation import keys -def test_public_key_fetching(r_mock): - payload = { - 'id': 'https://actor.mock/users/actor#main-key', - 'owner': 'test', - 'publicKeyPem': 'test_pem', - } - actor = 'https://actor.mock/' - r_mock.get(actor, json={'publicKey': payload}) - r = keys.get_public_key(actor) +@pytest.mark.parametrize('raw, expected', [ + ('algorithm="test",keyId="https://test.com"', 'https://test.com'), + ('keyId="https://test.com",algorithm="test"', 'https://test.com'), +]) +def test_get_key_from_header(raw, expected): + r = keys.get_key_id_from_signature_header(raw) + assert r == expected + - assert r['id'] == payload['id'] - assert r['owner'] == payload['owner'] - assert r['public_key_pem'] == payload['publicKeyPem'] +@pytest.mark.parametrize('raw', [ + 'algorithm="test",keyid="badCase"', + 'algorithm="test",wrong="wrong"', + 'keyId = "wrong"', + 'keyId=\'wrong\'', + 'keyId="notanurl"', + 'keyId="wrong://test.com"', +]) +def test_get_key_from_header_invalid(raw): + with pytest.raises(ValueError): + keys.get_key_id_from_signature_header(raw) diff --git a/api/tests/federation/test_serializers.py b/api/tests/federation/test_serializers.py index 18b8525f281c3cbea6387bdd4a7336fd2b0d0b6b..efa92b16a26dcdf72287282895872c10e5d10070 100644 --- a/api/tests/federation/test_serializers.py +++ b/api/tests/federation/test_serializers.py @@ -1,36 +1,146 @@ from django.urls import reverse from funkwhale_api.federation import keys +from funkwhale_api.federation import models from funkwhale_api.federation import serializers -def test_repr_instance_actor(db, preferences, settings): - _, public_key = keys.get_key_pair() - preferences['federation__public_key'] = public_key.decode('utf-8') - settings.FEDERATION_HOSTNAME = 'test.federation' - settings.FUNKWHALE_URL = 'https://test.federation' - actor_url = settings.FUNKWHALE_URL + reverse('federation:instance-actor') - inbox_url = settings.FUNKWHALE_URL + reverse('federation:instance-inbox') - outbox_url = settings.FUNKWHALE_URL + reverse('federation:instance-outbox') +def test_actor_serializer_from_ap(db): + payload = { + 'id': 'https://test.federation/user', + 'type': 'Person', + 'following': 'https://test.federation/user/following', + 'followers': 'https://test.federation/user/followers', + 'inbox': 'https://test.federation/user/inbox', + 'outbox': 'https://test.federation/user/outbox', + 'preferredUsername': 'user', + 'name': 'Real User', + 'summary': 'Hello world', + 'url': 'https://test.federation/@user', + 'manuallyApprovesFollowers': False, + 'publicKey': { + 'id': 'https://test.federation/user#main-key', + 'owner': 'https://test.federation/user', + 'publicKeyPem': 'yolo' + }, + 'endpoints': { + 'sharedInbox': 'https://test.federation/inbox' + }, + } + + serializer = serializers.ActorSerializer(data=payload) + assert serializer.is_valid() + + actor = serializer.build() + + assert actor.url == payload['id'] + assert actor.inbox_url == payload['inbox'] + assert actor.outbox_url == payload['outbox'] + assert actor.shared_inbox_url == payload['endpoints']['sharedInbox'] + assert actor.followers_url == payload['followers'] + assert actor.following_url == payload['following'] + assert actor.public_key == payload['publicKey']['publicKeyPem'] + assert actor.preferred_username == payload['preferredUsername'] + assert actor.name == payload['name'] + assert actor.domain == 'test.federation' + assert actor.summary == payload['summary'] + assert actor.type == 'Person' + assert actor.manually_approves_followers == payload['manuallyApprovesFollowers'] + + +def test_actor_serializer_only_mandatory_field_from_ap(db): + payload = { + 'id': 'https://test.federation/user', + 'type': 'Person', + 'following': 'https://test.federation/user/following', + 'followers': 'https://test.federation/user/followers', + 'inbox': 'https://test.federation/user/inbox', + 'outbox': 'https://test.federation/user/outbox', + 'preferredUsername': 'user', + } + + serializer = serializers.ActorSerializer(data=payload) + assert serializer.is_valid() + + actor = serializer.build() + assert actor.url == payload['id'] + assert actor.inbox_url == payload['inbox'] + assert actor.outbox_url == payload['outbox'] + assert actor.followers_url == payload['followers'] + assert actor.following_url == payload['following'] + assert actor.preferred_username == payload['preferredUsername'] + assert actor.domain == 'test.federation' + assert actor.type == 'Person' + assert actor.manually_approves_followers is None + + +def test_actor_serializer_to_ap(): expected = { '@context': [ 'https://www.w3.org/ns/activitystreams', 'https://w3id.org/security/v1', {}, ], - 'id': actor_url, - 'type': 'Person', - 'preferredUsername': 'service', - 'name': 'Service Bot - test.federation', - 'summary': 'Bot account for federating with test.federation', - 'inbox': inbox_url, - 'outbox': outbox_url, - 'publicKey': { - 'id': '{}#main-key'.format(actor_url), - 'owner': actor_url, - 'publicKeyPem': public_key.decode('utf-8') - }, + 'id': 'https://test.federation/user', + 'type': 'Person', + 'following': 'https://test.federation/user/following', + 'followers': 'https://test.federation/user/followers', + 'inbox': 'https://test.federation/user/inbox', + 'outbox': 'https://test.federation/user/outbox', + 'preferredUsername': 'user', + 'name': 'Real User', + 'summary': 'Hello world', + 'manuallyApprovesFollowers': False, + 'publicKey': { + 'id': 'https://test.federation/user#main-key', + 'owner': 'https://test.federation/user', + 'publicKeyPem': 'yolo' + }, + 'endpoints': { + 'sharedInbox': 'https://test.federation/inbox' + }, + } + ac = models.Actor( + url=expected['id'], + inbox_url=expected['inbox'], + outbox_url=expected['outbox'], + shared_inbox_url=expected['endpoints']['sharedInbox'], + followers_url=expected['followers'], + following_url=expected['following'], + public_key=expected['publicKey']['publicKeyPem'], + preferred_username=expected['preferredUsername'], + name=expected['name'], + domain='test.federation', + summary=expected['summary'], + type='Person', + manually_approves_followers=False, + + ) + serializer = serializers.ActorSerializer(ac) + + assert serializer.data == expected + + +def test_webfinger_serializer(): + expected = { + 'subject': 'acct:service@test.federation', + 'links': [ + { + 'rel': 'self', + 'href': 'https://test.federation/federation/instance/actor', + 'type': 'application/activity+json', + } + ], + 'aliases': [ + 'https://test.federation/federation/instance/actor', + ] } + actor = models.Actor( + url=expected['links'][0]['href'], + preferred_username='service', + domain='test.federation', + ) + serializer = serializers.ActorWebfingerSerializer(actor) - assert expected == serializers.repr_instance_actor() + assert serializer.data == expected diff --git a/api/tests/federation/test_signing.py b/api/tests/federation/test_signing.py index dc678f749bb0d7063ff58842c24de2391261b9ef..0c1ec2e0ba1dcd16722c2615eeeb48a5f01d7e87 100644 --- a/api/tests/federation/test_signing.py +++ b/api/tests/federation/test_signing.py @@ -7,23 +7,23 @@ from funkwhale_api.federation import signing from funkwhale_api.federation import keys -def test_can_sign_and_verify_request(factories): - private, public = factories['federation.KeyPair']() - auth = factories['federation.SignatureAuth'](key=private) - request = factories['federation.SignedRequest']( +def test_can_sign_and_verify_request(nodb_factories): + private, public = nodb_factories['federation.KeyPair']() + auth = nodb_factories['federation.SignatureAuth'](key=private) + request = nodb_factories['federation.SignedRequest']( auth=auth ) prepared_request = request.prepare() assert 'date' in prepared_request.headers - assert 'authorization' in prepared_request.headers - assert prepared_request.headers['authorization'].startswith('Signature') - assert signing.verify(prepared_request, public) is None + assert 'signature' in prepared_request.headers + assert signing.verify( + prepared_request, public) is None -def test_can_sign_and_verify_request_digest(factories): - private, public = factories['federation.KeyPair']() - auth = factories['federation.SignatureAuth'](key=private) - request = factories['federation.SignedRequest']( +def test_can_sign_and_verify_request_digest(nodb_factories): + private, public = nodb_factories['federation.KeyPair']() + auth = nodb_factories['federation.SignatureAuth'](key=private) + request = nodb_factories['federation.SignedRequest']( auth=auth, method='post', data=b'hello=world' @@ -31,70 +31,80 @@ def test_can_sign_and_verify_request_digest(factories): prepared_request = request.prepare() assert 'date' in prepared_request.headers assert 'digest' in prepared_request.headers - assert 'authorization' in prepared_request.headers - assert prepared_request.headers['authorization'].startswith('Signature') + assert 'signature' in prepared_request.headers assert signing.verify(prepared_request, public) is None -def test_verify_fails_with_wrong_key(factories): - wrong_private, wrong_public = factories['federation.KeyPair']() - request = factories['federation.SignedRequest']() +def test_verify_fails_with_wrong_key(nodb_factories): + wrong_private, wrong_public = nodb_factories['federation.KeyPair']() + request = nodb_factories['federation.SignedRequest']() prepared_request = request.prepare() with pytest.raises(cryptography.exceptions.InvalidSignature): signing.verify(prepared_request, wrong_public) -def test_can_verify_django_request(factories, api_request): +def test_can_verify_django_request(factories, fake_request): private_key, public_key = keys.get_key_pair() signed_request = factories['federation.SignedRequest']( - auth__key=private_key + auth__key=private_key, + auth__headers=[ + 'date', + ] ) prepared = signed_request.prepare() - django_request = api_request.get( + django_request = fake_request.get( '/', - headers={ - 'Date': prepared.headers['date'], - 'Authorization': prepared.headers['authorization'], + **{ + 'HTTP_DATE': prepared.headers['date'], + 'HTTP_SIGNATURE': prepared.headers['signature'], } ) assert signing.verify_django(django_request, public_key) is None -def test_can_verify_django_request_digest(factories, api_request): +def test_can_verify_django_request_digest(factories, fake_request): private_key, public_key = keys.get_key_pair() signed_request = factories['federation.SignedRequest']( auth__key=private_key, method='post', - data=b'hello=world' + data=b'hello=world', + auth__headers=[ + 'date', + 'digest', + ] ) prepared = signed_request.prepare() - django_request = api_request.post( + django_request = fake_request.post( '/', - headers={ - 'Date': prepared.headers['date'], - 'Digest': prepared.headers['digest'], - 'Authorization': prepared.headers['authorization'], + **{ + 'HTTP_DATE': prepared.headers['date'], + 'HTTP_DIGEST': prepared.headers['digest'], + 'HTTP_SIGNATURE': prepared.headers['signature'], } ) assert signing.verify_django(django_request, public_key) is None -def test_can_verify_django_request_digest_failure(factories, api_request): +def test_can_verify_django_request_digest_failure(factories, fake_request): private_key, public_key = keys.get_key_pair() signed_request = factories['federation.SignedRequest']( auth__key=private_key, method='post', - data=b'hello=world' + data=b'hello=world', + auth__headers=[ + 'date', + 'digest', + ] ) prepared = signed_request.prepare() - django_request = api_request.post( + django_request = fake_request.post( '/', - headers={ - 'Date': prepared.headers['date'], - 'Digest': prepared.headers['digest'] + 'noop', - 'Authorization': prepared.headers['authorization'], + **{ + 'HTTP_DATE': prepared.headers['date'], + 'HTTP_DIGEST': prepared.headers['digest'] + 'noop', + 'HTTP_SIGNATURE': prepared.headers['signature'], } ) @@ -102,17 +112,20 @@ def test_can_verify_django_request_digest_failure(factories, api_request): signing.verify_django(django_request, public_key) -def test_can_verify_django_request_failure(factories, api_request): +def test_can_verify_django_request_failure(factories, fake_request): private_key, public_key = keys.get_key_pair() signed_request = factories['federation.SignedRequest']( - auth__key=private_key + auth__key=private_key, + auth__headers=[ + 'date', + ] ) prepared = signed_request.prepare() - django_request = api_request.get( + django_request = fake_request.get( '/', - headers={ - 'Date': 'Wrong', - 'Authorization': prepared.headers['authorization'], + **{ + 'HTTP_DATE': 'Wrong', + 'HTTP_SIGNATURE': prepared.headers['signature'], } ) with pytest.raises(cryptography.exceptions.InvalidSignature): diff --git a/api/tests/federation/test_utils.py b/api/tests/federation/test_utils.py index 8bada65bb03c2b17bbc3428b3846a8c1a3f416b7..dc371ad9ed976a5d1179a14d9dc58e831e2873e8 100644 --- a/api/tests/federation/test_utils.py +++ b/api/tests/federation/test_utils.py @@ -12,3 +12,37 @@ from funkwhale_api.federation import utils def test_full_url(settings, url, path, expected): settings.FUNKWHALE_URL = url assert utils.full_url(path) == expected + + +def test_extract_headers_from_meta(): + wsgi_headers = { + 'HTTP_HOST': 'nginx', + 'HTTP_X_REAL_IP': '172.20.0.4', + 'HTTP_X_FORWARDED_FOR': '188.165.228.227, 172.20.0.4', + 'HTTP_X_FORWARDED_PROTO': 'http', + 'HTTP_X_FORWARDED_HOST': 'localhost:80', + 'HTTP_X_FORWARDED_PORT': '80', + 'HTTP_CONNECTION': 'close', + 'CONTENT_LENGTH': '1155', + 'CONTENT_TYPE': 'txt/application', + 'HTTP_SIGNATURE': 'Hello', + 'HTTP_DATE': 'Sat, 31 Mar 2018 13:53:55 GMT', + 'HTTP_USER_AGENT': 'http.rb/3.0.0 (Mastodon/2.2.0; +https://mastodon.eliotberriot.com/)'} + + cleaned_headers = utils.clean_wsgi_headers(wsgi_headers) + + expected = { + 'Host': 'nginx', + 'X-Real-Ip': '172.20.0.4', + 'X-Forwarded-For': '188.165.228.227, 172.20.0.4', + 'X-Forwarded-Proto': 'http', + 'X-Forwarded-Host': 'localhost:80', + 'X-Forwarded-Port': '80', + 'Connection': 'close', + 'Content-Length': '1155', + 'Content-Type': 'txt/application', + 'Signature': 'Hello', + 'Date': 'Sat, 31 Mar 2018 13:53:55 GMT', + 'User-Agent': 'http.rb/3.0.0 (Mastodon/2.2.0; +https://mastodon.eliotberriot.com/)' + } + assert cleaned_headers == expected diff --git a/api/tests/federation/test_views.py b/api/tests/federation/test_views.py index 6a8de8c14a8af9944e23860feb2951f07c0a4fc1..0d2ac882fb25ecac154a8426ffc2949ca7f81435 100644 --- a/api/tests/federation/test_views.py +++ b/api/tests/federation/test_views.py @@ -2,29 +2,35 @@ from django.urls import reverse import pytest +from funkwhale_api.federation import actors from funkwhale_api.federation import serializers from funkwhale_api.federation import webfinger -def test_instance_actor(db, settings, api_client): - settings.FUNKWHALE_URL = 'http://test.com' - url = reverse('federation:instance-actor') + +@pytest.mark.parametrize('system_actor', actors.SYSTEM_ACTORS.keys()) +def test_instance_actors(system_actor, db, settings, api_client): + actor = actors.SYSTEM_ACTORS[system_actor].get_actor_instance() + url = reverse( + 'federation:instance-actors-detail', + kwargs={'actor': system_actor}) response = api_client.get(url) + serializer = serializers.ActorSerializer(actor) assert response.status_code == 200 - assert response.data == serializers.repr_instance_actor() + assert response.data == serializer.data -@pytest.mark.parametrize('route', [ - 'instance-outbox', - 'instance-inbox', - 'instance-actor', - 'well-known-webfinger', +@pytest.mark.parametrize('route,kwargs', [ + ('instance-actors-outbox', {'actor': 'library'}), + ('instance-actors-inbox', {'actor': 'library'}), + ('instance-actors-detail', {'actor': 'library'}), + ('well-known-webfinger', {}), ]) -def test_instance_inbox_405_if_federation_disabled( - db, settings, api_client, route): +def test_instance_endpoints_405_if_federation_disabled( + authenticated_actor, db, settings, api_client, route, kwargs): settings.FEDERATION_ENABLED = False - url = reverse('federation:{}'.format(route)) + url = reverse('federation:{}'.format(route), kwargs=kwargs) response = api_client.get(url) assert response.status_code == 405 @@ -33,7 +39,6 @@ def test_instance_inbox_405_if_federation_disabled( def test_wellknown_webfinger_validates_resource( db, api_client, settings, mocker): clean = mocker.spy(webfinger, 'clean_resource') - settings.FEDERATION_ENABLED = True url = reverse('federation:well-known-webfinger') response = api_client.get(url, data={'resource': 'something'}) @@ -45,14 +50,15 @@ def test_wellknown_webfinger_validates_resource( ) +@pytest.mark.parametrize('system_actor', actors.SYSTEM_ACTORS.keys()) def test_wellknown_webfinger_system( - db, api_client, settings, mocker): - settings.FEDERATION_ENABLED = True - settings.FEDERATION_HOSTNAME = 'test.federation' + system_actor, db, api_client, settings, mocker): + actor = actors.SYSTEM_ACTORS[system_actor].get_actor_instance() url = reverse('federation:well-known-webfinger') response = api_client.get( - url, data={'resource': 'acct:service@test.federation'}) + url, data={'resource': 'acct:{}'.format(actor.webfinger_subject)}) + serializer = serializers.ActorWebfingerSerializer(actor) assert response.status_code == 200 assert response['Content-Type'] == 'application/jrd+json' - assert response.data == webfinger.serialize_system_acct() + assert response.data == serializer.data diff --git a/api/tests/federation/test_webfinger.py b/api/tests/federation/test_webfinger.py index d2b00f8f1c61afee1133c31749f25af1ce6522a4..96258455ae6fe1f60e330d7768a58d6e94fc91fa 100644 --- a/api/tests/federation/test_webfinger.py +++ b/api/tests/federation/test_webfinger.py @@ -25,42 +25,18 @@ def test_webfinger_clean_resource_errors(resource, message): def test_webfinger_clean_acct(settings): - settings.FEDERATION_HOSTNAME = 'test.federation' - username, hostname = webfinger.clean_acct('service@test.federation') - assert username == 'service' + username, hostname = webfinger.clean_acct('library@test.federation') + assert username == 'library' assert hostname == 'test.federation' @pytest.mark.parametrize('resource,message', [ ('service', 'Invalid format'), - ('service@test.com', 'Invalid hostname'), + ('service@test.com', 'Invalid hostname test.com'), ('noop@test.federation', 'Invalid account'), ]) def test_webfinger_clean_acct_errors(resource, message, settings): - settings.FEDERATION_HOSTNAME = 'test.federation' - with pytest.raises(forms.ValidationError) as excinfo: webfinger.clean_resource(resource) assert message == str(excinfo) - - -def test_service_serializer(settings): - settings.FEDERATION_HOSTNAME = 'test.federation' - settings.FUNKWHALE_URL = 'https://test.federation' - - expected = { - 'subject': 'acct:service@test.federation', - 'links': [ - { - 'rel': 'self', - 'href': 'https://test.federation/federation/instance/actor', - 'type': 'application/activity+json', - } - ], - 'aliases': [ - 'https://test.federation/federation/instance/actor', - ] - } - - assert expected == webfinger.serialize_system_acct() diff --git a/changes/changelog.d/test-bot.enhancement b/changes/changelog.d/test-bot.enhancement new file mode 100644 index 0000000000000000000000000000000000000000..03100d7c8963acc9d9acf8d3035b64bda1bf1296 --- /dev/null +++ b/changes/changelog.d/test-bot.enhancement @@ -0,0 +1 @@ +Implemented a @test@yourfunkwhaledomain bot to ensure federation works properly. Send it "/ping" and it will answer back :) diff --git a/dev.yml b/dev.yml index 126efa683e7fe3650b82c03f02862acbc18cf4be..c0470a2ab6f6172127e90fa9d3911b0cae9b33db 100644 --- a/dev.yml +++ b/dev.yml @@ -38,13 +38,8 @@ services: - redis command: celery -A funkwhale_api.taskapp worker -l debug environment: - - "DJANGO_ALLOWED_HOSTS=localhost" - - "DJANGO_SETTINGS_MODULE=config.settings.local" - - "DJANGO_SECRET_KEY=dev" - - C_FORCE_ROOT=true - "DATABASE_URL=postgresql://postgres@postgres/postgres" - "CACHE_URL=redis://redis:6379/0" - - "FUNKWHALE_URL=http://localhost" volumes: - ./api:/app - ./data/music:/music @@ -60,13 +55,8 @@ services: - ./api:/app - ./data/music:/music environment: - - "PYTHONDONTWRITEBYTECODE=true" - - "DJANGO_ALLOWED_HOSTS=localhost,nginx" - - "DJANGO_SETTINGS_MODULE=config.settings.local" - - "DJANGO_SECRET_KEY=dev" - "DATABASE_URL=postgresql://postgres@postgres/postgres" - "CACHE_URL=redis://redis:6379/0" - - "FUNKWHALE_URL=http://localhost" links: - postgres - redis diff --git a/docker/nginx/entrypoint.sh b/docker/nginx/entrypoint.sh index 93b4a0533fe4edd3a7ba815cf9a7b1d0ac118824..1819acf1cf579cb3e0d70b754d06f2b6fc4959c6 100755 --- a/docker/nginx/entrypoint.sh +++ b/docker/nginx/entrypoint.sh @@ -1,8 +1,9 @@ #!/bin/bash -eux - +FIRST_HOST=$(echo ${DJANGO_ALLOWED_HOSTS} | cut -d, -f1) echo "Copying template file..." cp /etc/nginx/funkwhale_proxy.conf{.template,} -sed -i "s/X-Forwarded-Host \$host:\$server_port/X-Forwarded-Host localhost:${WEBPACK_DEVSERVER_PORT}/" /etc/nginx/funkwhale_proxy.conf +sed -i "s/X-Forwarded-Host \$host:\$server_port/X-Forwarded-Host ${FIRST_HOST}:${WEBPACK_DEVSERVER_PORT}/" /etc/nginx/funkwhale_proxy.conf +sed -i "s/proxy_set_header Host \$host/proxy_set_header Host ${FIRST_HOST}/" /etc/nginx/funkwhale_proxy.conf sed -i "s/proxy_set_header X-Forwarded-Port \$server_port/proxy_set_header X-Forwarded-Port ${WEBPACK_DEVSERVER_PORT}/" /etc/nginx/funkwhale_proxy.conf cat /etc/nginx/funkwhale_proxy.conf