Commit 18e8e4fa authored by Agate's avatar Agate 💬

Merge branch 'remote-scan' into 'develop'

Remote scan/import

Closes #137 and #136

See merge request funkwhale/funkwhale!126
parents 98381a00 8eba42d4
API_AUTHENTICATION_REQUIRED=True
RAVEN_ENABLED=false
RAVEN_DSN=https://44332e9fdd3d42879c7d35bf8562c6a4:0062dc16a22b41679cd5765e5342f716@sentry.eliotberriot.com/5
DJANGO_ALLOWED_HOSTS=localhost,nginx
DJANGO_ALLOWED_HOSTS=.funkwhale.test,localhost,nginx,0.0.0.0,127.0.0.1
DJANGO_SETTINGS_MODULE=config.settings.local
DJANGO_SECRET_KEY=dev
C_FORCE_ROOT=true
FUNKWHALE_URL=http://localhost
FUNKWHALE_HOSTNAME=localhost
FUNKWHALE_PROTOCOL=http
PYTHONDONTWRITEBYTECODE=true
WEBPACK_DEVSERVER_PORT=8080
......@@ -206,3 +206,80 @@ Typical workflow for a merge request
6. Push your branch
7. Create your merge request
8. Take a step back and enjoy, we're really grateful you did all of this and took the time to contribute!
Working with federation locally
-------------------------------
To achieve that, you'll need:
1. to update your dns resolver to resolve all your .dev hostnames locally
2. a reverse proxy (such as traefik) to catch those .dev requests and
and with https certificate
3. two instances (or more) running locally, following the regular dev setup
Resolve .dev names locally
^^^^^^^^^^^^^^^^^^^^^^^^^^
If you use dnsmasq, this is as simple as doing::
echo "address=/test/172.17.0.1" | sudo tee /etc/dnsmasq.d/test.conf
sudo systemctl restart dnsmasq
If you use NetworkManager with dnsmasq integration, use this instead::
echo "address=/test/172.17.0.1" | sudo tee /etc/NetworkManager/dnsmasq.d/test.conf
sudo systemctl restart NetworkManager
Add wildcard certificate to the trusted certificates
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Simply copy bundled certificates::
sudo cp docker/ssl/test.crt /usr/local/share/ca-certificates/
sudo update-ca-certificates
This certificate is a wildcard for ``*.funkwhale.test``
Run a reverse proxy for your instances
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Crete docker network
^^^^^^^^^^^^^^^^^^^^
Create the federation network::
docker network create federation
Launch everything
^^^^^^^^^^^^^^^^^
Launch the traefik proxy::
docker-compose -f docker/traefik.yml up -d
Then, in separate terminals, you can setup as many different instances as you
need::
export COMPOSE_PROJECT_NAME=node2
docker-compose -f dev.yml run --rm api python manage.py migrate
docker-compose -f dev.yml run --rm api python manage.py createsuperuser
docker-compose -f dev.yml up nginx api front
Note that by default, if you don't export the COMPOSE_PROJECT_NAME,
we will default to node1 as the name of your instance.
Assuming your project name is ``node1``, your server will be reachable
at ``https://node1.funkwhale.test/``. Not that you'll have to trust
the SSL Certificate as it's self signed.
When working on federation with traefik, ensure you have this in your ``env``::
# This will ensure we don't bind any port on the host, and thus enable
# multiple instances of funkwhale to be spawned concurrently.
WEBPACK_DEVSERVER_PORT_BINDING=
# This disable certificate verification
EXTERNAL_REQUESTS_VERIFY_SSL=false
# this ensure you don't have incorrect urls pointing to http resources
FUNKWHALE_PROTOCOL=https
......@@ -32,6 +32,10 @@ v1_patterns += [
include(
('funkwhale_api.instance.urls', 'instance'),
namespace='instance')),
url(r'^federation/',
include(
('funkwhale_api.federation.api_urls', 'federation'),
namespace='federation')),
url(r'^providers/',
include(
('funkwhale_api.providers.urls', 'providers'),
......
......@@ -25,8 +25,26 @@ try:
except FileNotFoundError:
pass
FUNKWHALE_URL = env('FUNKWHALE_URL')
FUNKWHALE_HOSTNAME = urlsplit(FUNKWHALE_URL).netloc
FUNKWHALE_HOSTNAME = None
FUNKWHALE_HOSTNAME_SUFFIX = env('FUNKWHALE_HOSTNAME_SUFFIX', default=None)
FUNKWHALE_HOSTNAME_PREFIX = env('FUNKWHALE_HOSTNAME_PREFIX', default=None)
if FUNKWHALE_HOSTNAME_PREFIX and FUNKWHALE_HOSTNAME_SUFFIX:
# We're in traefik case, in development
FUNKWHALE_HOSTNAME = '{}.{}'.format(
FUNKWHALE_HOSTNAME_PREFIX, FUNKWHALE_HOSTNAME_SUFFIX)
FUNKWHALE_PROTOCOL = env('FUNKWHALE_PROTOCOL', default='https')
else:
try:
FUNKWHALE_HOSTNAME = env('FUNKWHALE_HOSTNAME')
FUNKWHALE_PROTOCOL = env('FUNKWHALE_PROTOCOL', default='https')
except Exception:
FUNKWHALE_URL = env('FUNKWHALE_URL')
_parsed = urlsplit(FUNKWHALE_URL)
FUNKWHALE_HOSTNAME = _parsed.netloc
FUNKWHALE_PROTOCOL = _parsed.scheme
FUNKWHALE_URL = '{}://{}'.format(FUNKWHALE_PROTOCOL, FUNKWHALE_HOSTNAME)
FEDERATION_ENABLED = env.bool('FEDERATION_ENABLED', default=True)
FEDERATION_HOSTNAME = env('FEDERATION_HOSTNAME', default=FUNKWHALE_HOSTNAME)
......@@ -406,3 +424,8 @@ ACCOUNT_USERNAME_BLACKLIST = [
'staff',
'service',
] + env.list('ACCOUNT_USERNAME_BLACKLIST', default=[])
EXTERNAL_REQUESTS_VERIFY_SSL = env.bool(
'EXTERNAL_REQUESTS_VERIFY_SSL',
default=True
)
import django_filters
from django.db import models
from funkwhale_api.music import utils
PRIVACY_LEVEL_CHOICES = [
('me', 'Only me'),
......@@ -25,3 +29,15 @@ def privacy_level_query(user, lookup_field='privacy_level'):
'followers', 'instance', 'everyone'
]
})
class SearchFilter(django_filters.CharFilter):
def __init__(self, *args, **kwargs):
self.search_fields = kwargs.pop('search_fields')
super().__init__(*args, **kwargs)
def filter(self, qs, value):
if not value:
return qs
query = utils.get_query(value, self.search_fields)
return qs.filter(query)
import logging
import json
import requests_http_signature
import uuid
from funkwhale_api.common import session
from . import models
from . import signing
logger = logging.getLogger(__name__)
from . import serializers
from . import tasks
ACTIVITY_TYPES = [
'Accept',
......@@ -61,86 +52,16 @@ OBJECT_TYPES = [
def deliver(activity, on_behalf_of, to=[]):
from . import actors
logger.info('Preparing activity delivery to %s', to)
auth = signing.get_auth(
on_behalf_of.private_key, on_behalf_of.private_key_id)
for url in to:
recipient_actor = actors.get_actor(url)
logger.debug('delivering to %s', recipient_actor.inbox_url)
logger.debug('activity content: %s', json.dumps(activity))
response = session.get_session().post(
auth=auth,
json=activity,
url=recipient_actor.inbox_url,
headers={
'Content-Type': 'application/activity+json'
}
)
response.raise_for_status()
logger.debug('Remote answered with %s', response.status_code)
def get_follow(follow_id, follower, followed):
return {
'@context': [
'https://www.w3.org/ns/activitystreams',
'https://w3id.org/security/v1',
{}
],
'actor': follower.url,
'id': follower.url + '#follows/{}'.format(follow_id),
'object': followed.url,
'type': 'Follow'
}
def get_undo(id, actor, object):
return {
'@context': [
'https://www.w3.org/ns/activitystreams',
'https://w3id.org/security/v1',
{}
],
'type': 'Undo',
'id': id + '/undo',
'actor': actor.url,
'object': object,
}
def get_accept_follow(accept_id, accept_actor, follow, follow_actor):
return {
"@context": [
"https://www.w3.org/ns/activitystreams",
"https://w3id.org/security/v1",
{}
],
"id": accept_actor.url + '#accepts/follows/{}'.format(
accept_id),
"type": "Accept",
"actor": accept_actor.url,
"object": {
"id": follow['id'],
"type": "Follow",
"actor": follow_actor.url,
"object": accept_actor.url
},
}
return tasks.send.delay(
activity=activity,
actor_id=on_behalf_of.pk,
to=to
)
def accept_follow(target, follow, actor):
accept_uuid = uuid.uuid4()
accept = get_accept_follow(
accept_id=accept_uuid,
accept_actor=target,
follow=follow,
follow_actor=actor)
deliver(
accept,
to=[actor.url],
on_behalf_of=target)
return models.Follow.objects.get_or_create(
actor=actor,
target=target,
)
def accept_follow(follow):
serializer = serializers.AcceptFollowSerializer(follow)
return deliver(
serializer.data,
to=[follow.actor.url],
on_behalf_of=follow.target)
......@@ -31,6 +31,8 @@ def remove_tags(text):
def get_actor_data(actor_url):
response = session.get_session().get(
actor_url,
timeout=5,
verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
headers={
'Accept': 'application/activity+json',
}
......@@ -42,6 +44,7 @@ def get_actor_data(actor_url):
raise ValueError(
'Invalid actor payload: {}'.format(response.text))
def get_actor(actor_url):
data = get_actor_data(actor_url)
serializer = serializers.ActorSerializer(data=data)
......@@ -150,24 +153,32 @@ class SystemActor(object):
def handle_follow(self, ac, sender):
system_actor = self.get_actor_instance()
if self.manually_approves_followers:
fr, created = models.FollowRequest.objects.get_or_create(
actor=sender,
target=system_actor,
approved=None,
)
return fr
serializer = serializers.FollowSerializer(
data=ac, context={'follow_actor': sender})
if not serializer.is_valid():
return logger.info('Invalid follow payload')
approved = True if not self.manually_approves_followers else None
follow = serializer.save(approved=approved)
if follow.approved:
return activity.accept_follow(follow)
def handle_accept(self, ac, sender):
system_actor = self.get_actor_instance()
serializer = serializers.AcceptFollowSerializer(
data=ac,
context={'follow_target': sender, 'follow_actor': system_actor})
if not serializer.is_valid(raise_exception=True):
return logger.info('Received invalid payload')
return activity.accept_follow(
system_actor, ac, sender
)
return serializer.save()
def handle_undo_follow(self, ac, sender):
actor = self.get_actor_instance()
models.Follow.objects.filter(
actor=sender,
target=actor,
).delete()
system_actor = self.get_actor_instance()
serializer = serializers.UndoFollowSerializer(
data=ac, context={'actor': sender, 'target': system_actor})
if not serializer.is_valid():
return logger.info('Received invalid payload')
serializer.save()
def handle_undo(self, ac, sender):
if ac['object']['type'] != 'Follow':
......@@ -343,15 +354,15 @@ class TestActor(SystemActor):
super().handle_follow(ac, sender)
# also, we follow back
test_actor = self.get_actor_instance()
follow_uuid = uuid.uuid4()
follow = activity.get_follow(
follow_id=follow_uuid,
follower=test_actor,
followed=sender)
follow_back = models.Follow.objects.get_or_create(
actor=test_actor,
target=sender,
approved=None,
)[0]
activity.deliver(
follow,
to=[ac['actor']],
on_behalf_of=test_actor)
serializers.FollowSerializer(follow_back).data,
to=[follow_back.target.url],
on_behalf_of=follow_back.actor)
def handle_undo_follow(self, ac, sender):
super().handle_undo_follow(ac, sender)
......@@ -364,11 +375,7 @@ class TestActor(SystemActor):
)
except models.Follow.DoesNotExist:
return
undo = activity.get_undo(
id=follow.get_federation_url(),
actor=actor,
object=serializers.FollowSerializer(follow).data,
)
undo = serializers.UndoFollowSerializer(follow).data
follow.delete()
activity.deliver(
undo,
......
from django.contrib import admin
from . import models
@admin.register(models.Actor)
class ActorAdmin(admin.ModelAdmin):
list_display = [
'url',
'domain',
'preferred_username',
'type',
'creation_date',
'last_fetch_date']
search_fields = ['url', 'domain', 'preferred_username']
list_filter = [
'type'
]
@admin.register(models.Follow)
class FollowAdmin(admin.ModelAdmin):
list_display = [
'actor',
'target',
'approved',
'creation_date'
]
list_filter = [
'approved'
]
search_fields = ['actor__url', 'target__url']
list_select_related = True
@admin.register(models.Library)
class LibraryAdmin(admin.ModelAdmin):
list_display = [
'actor',
'url',
'creation_date',
'fetched_date',
'tracks_count']
search_fields = ['actor__url', 'url']
list_filter = [
'federation_enabled',
'download_files',
'autoimport',
]
list_select_related = True
@admin.register(models.LibraryTrack)
class LibraryTrackAdmin(admin.ModelAdmin):
list_display = [
'title',
'artist_name',
'album_title',
'url',
'library',
'creation_date',
'published_date',
]
search_fields = [
'library__url', 'url', 'artist_name', 'title', 'album_title']
list_select_related = True
from rest_framework import routers
from . import views
router = routers.SimpleRouter()
router.register(
r'libraries',
views.LibraryViewSet,
'libraries')
router.register(
r'library-tracks',
views.LibraryTrackViewSet,
'library-tracks')
urlpatterns = router.urls
......@@ -4,3 +4,17 @@ from dynamic_preferences import types
from dynamic_preferences.registries import global_preferences_registry
federation = types.Section('federation')
@global_preferences_registry.register
class MusicCacheDuration(types.IntPreference):
show_in_api = True
section = federation
name = 'music_cache_duration'
default = 60 * 24 * 2
verbose_name = 'Music cache duration'
help_text = (
'How much minutes do you want to keep a copy of federated tracks'
'locally? Federated files that were not listened in this interval '
'will be erased and refetched from the remote on the next listening.'
)
......@@ -113,15 +113,6 @@ class FollowFactory(factory.DjangoModelFactory):
)
@registry.register
class FollowRequestFactory(factory.DjangoModelFactory):
target = factory.SubFactory(ActorFactory)
actor = factory.SubFactory(ActorFactory)
class Meta:
model = models.FollowRequest
@registry.register
class LibraryFactory(factory.DjangoModelFactory):
actor = factory.SubFactory(ActorFactory)
......@@ -194,6 +185,11 @@ class LibraryTrackFactory(factory.DjangoModelFactory):
class Meta:
model = models.LibraryTrack
class Params:
with_audio_file = factory.Trait(
audio_file=factory.django.FileField()
)
@registry.register(name='federation.Note')
class NoteFactory(factory.Factory):
......
import django_filters
from funkwhale_api.common import fields
from . import models
class LibraryFilter(django_filters.FilterSet):
approved = django_filters.BooleanFilter('following__approved')
q = fields.SearchFilter(search_fields=[
'actor__domain',
])
class Meta:
model = models.Library
fields = {
'approved': ['exact'],
'federation_enabled': ['exact'],
'download_files': ['exact'],
'autoimport': ['exact'],
'tracks_count': ['exact'],
}
class LibraryTrackFilter(django_filters.FilterSet):
library = django_filters.CharFilter('library__uuid')
q = fields.SearchFilter(search_fields=[
'artist_name',
'title',
'album_title',
'library__actor__domain',
])
class Meta:
model = models.LibraryTrack
fields = {
'library': ['exact'],
'artist_name': ['exact', 'icontains'],
'title': ['exact', 'icontains'],
'album_title': ['exact', 'icontains'],
'audio_mimetype': ['exact', 'icontains'],
}
class FollowFilter(django_filters.FilterSet):
pending = django_filters.CharFilter(method='filter_pending')
ordering = django_filters.OrderingFilter(
# tuple-mapping retains order
fields=(
('creation_date', 'creation_date'),
('modification_date', 'modification_date'),
),
)
q = fields.SearchFilter(search_fields=[
'actor__domain',
'actor__preferred_username',
])
class Meta:
model = models.Follow
fields = ['approved', 'pending', 'q']
def filter_pending(self, queryset, field_name, value):
if value.lower() in ['true', '1', 'yes']:
queryset = queryset.filter(approved__isnull=True)
return queryset
import json
import requests
from django.conf import settings
from funkwhale_api.common import session
from . import actors
from . import models
from . import serializers
from . import signing
from . import webfinger
def scan_from_account_name(account_name):
"""
Given an account name such as library@test.library, will:
1. Perform the webfinger lookup
2. Perform the actor lookup
3. Perform the library's collection lookup
and return corresponding data in a dictionary.
"""
data = {}
try:
username, domain = webfinger.clean_acct(
account_name, ensure_local=False)
except serializers.ValidationError:
return {
'webfinger': {
'errors': ['Invalid account string']
}
}