Commit 253f026d authored by Eliot Berriot's avatar Eliot Berriot
Browse files

System actor

parent 8963218b
...@@ -94,6 +94,9 @@ FEDERATION_MUSIC_NEEDS_APPROVAL = env.bool( ...@@ -94,6 +94,9 @@ FEDERATION_MUSIC_NEEDS_APPROVAL = env.bool(
) )
# XXX: deprecated, see #186 # XXX: deprecated, see #186
FEDERATION_ACTOR_FETCH_DELAY = env.int("FEDERATION_ACTOR_FETCH_DELAY", default=60 * 12) FEDERATION_ACTOR_FETCH_DELAY = env.int("FEDERATION_ACTOR_FETCH_DELAY", default=60 * 12)
FEDERATION_SERVICE_ACTOR_USERNAME = env(
"FEDERATION_SERVICE_ACTOR_USERNAME", default="service"
)
ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=[]) + [FUNKWHALE_HOSTNAME] ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=[]) + [FUNKWHALE_HOSTNAME]
# APP CONFIGURATION # APP CONFIGURATION
......
...@@ -147,3 +147,24 @@ def order_for_search(qs, field): ...@@ -147,3 +147,24 @@ def order_for_search(qs, field):
this function will order the given qs based on the length of the given field this function will order the given qs based on the length of the given field
""" """
return qs.annotate(__size=models.functions.Length(field)).order_by("__size") return qs.annotate(__size=models.functions.Length(field)).order_by("__size")
def recursive_getattr(obj, key, permissive=False):
"""
Given a dictionary such as {'user': {'name': 'Bob'}} and
a dotted string such as user.name, returns 'Bob'.
If the value is not present, returns None
"""
v = obj
for k in key.split("."):
try:
v = v.get(k)
except (TypeError, AttributeError):
if not permissive:
raise
return
if v is None:
return
return v
...@@ -9,6 +9,8 @@ from django.db.models import Q ...@@ -9,6 +9,8 @@ from django.db.models import Q
from funkwhale_api.common import channels from funkwhale_api.common import channels
from funkwhale_api.common import utils as funkwhale_utils from funkwhale_api.common import utils as funkwhale_utils
recursive_getattr = funkwhale_utils.recursive_getattr
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
PUBLIC_ADDRESS = "https://www.w3.org/ns/activitystreams#Public" PUBLIC_ADDRESS = "https://www.w3.org/ns/activitystreams#Public"
...@@ -89,9 +91,9 @@ def should_reject(id, actor_id=None, payload={}): ...@@ -89,9 +91,9 @@ def should_reject(id, actor_id=None, payload={}):
media_types = ["Audio", "Artist", "Album", "Track", "Library", "Image"] media_types = ["Audio", "Artist", "Album", "Track", "Library", "Image"]
relevant_values = [ relevant_values = [
recursive_gettattr(payload, "type", permissive=True), recursive_getattr(payload, "type", permissive=True),
recursive_gettattr(payload, "object.type", permissive=True), recursive_getattr(payload, "object.type", permissive=True),
recursive_gettattr(payload, "target.type", permissive=True), recursive_getattr(payload, "target.type", permissive=True),
] ]
# if one of the payload types match our internal media types, then # if one of the payload types match our internal media types, then
# we apply policies that reject media # we apply policies that reject media
...@@ -343,7 +345,7 @@ class OutboxRouter(Router): ...@@ -343,7 +345,7 @@ class OutboxRouter(Router):
return activities return activities
def recursive_gettattr(obj, key, permissive=False): def recursive_getattr(obj, key, permissive=False):
""" """
Given a dictionary such as {'user': {'name': 'Bob'}} and Given a dictionary such as {'user': {'name': 'Bob'}} and
a dotted string such as user.name, returns 'Bob'. a dotted string such as user.name, returns 'Bob'.
...@@ -366,7 +368,7 @@ def recursive_gettattr(obj, key, permissive=False): ...@@ -366,7 +368,7 @@ def recursive_gettattr(obj, key, permissive=False):
def match_route(route, payload): def match_route(route, payload):
for key, value in route.items(): for key, value in route.items():
payload_value = recursive_gettattr(payload, key) payload_value = recursive_getattr(payload, key)
if payload_value != value: if payload_value != value:
return False return False
......
...@@ -5,8 +5,9 @@ from django.conf import settings ...@@ -5,8 +5,9 @@ from django.conf import settings
from django.utils import timezone from django.utils import timezone
from funkwhale_api.common import preferences, session from funkwhale_api.common import preferences, session
from funkwhale_api.users import models as users_models
from . import models, serializers from . import keys, models, serializers
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
...@@ -28,7 +29,7 @@ def get_actor_data(actor_url): ...@@ -28,7 +29,7 @@ def get_actor_data(actor_url):
def get_actor(fid, skip_cache=False): def get_actor(fid, skip_cache=False):
if not skip_cache: if not skip_cache:
try: try:
actor = models.Actor.objects.get(fid=fid) actor = models.Actor.objects.select_related().get(fid=fid)
except models.Actor.DoesNotExist: except models.Actor.DoesNotExist:
actor = None actor = None
fetch_delta = datetime.timedelta( fetch_delta = datetime.timedelta(
...@@ -42,3 +43,23 @@ def get_actor(fid, skip_cache=False): ...@@ -42,3 +43,23 @@ def get_actor(fid, skip_cache=False):
serializer.is_valid(raise_exception=True) serializer.is_valid(raise_exception=True)
return serializer.save(last_fetch_date=timezone.now()) return serializer.save(last_fetch_date=timezone.now())
def get_service_actor():
name, domain = (
settings.FEDERATION_SERVICE_ACTOR_USERNAME,
settings.FEDERATION_HOSTNAME,
)
try:
return models.Actor.objects.select_related().get(
preferred_username=name, domain__name=domain
)
except models.Actor.DoesNotExist:
pass
args = users_models.get_actor_data(name)
private, public = keys.get_key_pair()
args["private_key"] = private.decode("utf-8")
args["public_key"] = public.decode("utf-8")
args["type"] = "Service"
return models.Actor.objects.create(**args)
import cryptography import cryptography
import logging import logging
import datetime
from django.contrib.auth.models import AnonymousUser from django.contrib.auth.models import AnonymousUser
from rest_framework import authentication, exceptions as rest_exceptions from django.utils import timezone
from rest_framework import authentication, exceptions as rest_exceptions
from funkwhale_api.moderation import models as moderation_models from funkwhale_api.moderation import models as moderation_models
from . import actors, exceptions, keys, signing, utils from . import actors, exceptions, keys, signing, tasks, utils
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
...@@ -57,6 +59,15 @@ class SignatureAuthentication(authentication.BaseAuthentication): ...@@ -57,6 +59,15 @@ class SignatureAuthentication(authentication.BaseAuthentication):
actor = actors.get_actor(actor_url, skip_cache=True) actor = actors.get_actor(actor_url, skip_cache=True)
signing.verify_django(request, actor.public_key.encode("utf-8")) signing.verify_django(request, actor.public_key.encode("utf-8"))
# we trigger a nodeinfo update on the actor's domain, if needed
fetch_delay = 24 * 3600
now = timezone.now()
last_fetch = actor.domain.nodeinfo_fetch_date
if not last_fetch or (
last_fetch < (now - datetime.timedelta(seconds=fetch_delay))
):
tasks.update_domain_nodeinfo(domain_name=actor.domain.name)
actor.domain.refresh_from_db()
return actor return actor
def authenticate(self, request): def authenticate(self, request):
......
...@@ -69,6 +69,7 @@ def create_user(actor): ...@@ -69,6 +69,7 @@ def create_user(actor):
@registry.register @registry.register
class DomainFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory): class DomainFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
name = factory.Faker("domain_name") name = factory.Faker("domain_name")
nodeinfo_fetch_date = factory.LazyFunction(lambda: timezone.now())
class Meta: class Meta:
model = "federation.Domain" model = "federation.Domain"
......
# Generated by Django 2.1.5 on 2019-01-30 09:26
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
import django.db.models.deletion
import funkwhale_api.common.validators
import funkwhale_api.federation.models
class Migration(migrations.Migration):
dependencies = [
('federation', '0016_auto_20181227_1605'),
]
operations = [
migrations.RemoveField(
model_name='actor',
name='old_domain',
),
migrations.AddField(
model_name='domain',
name='service_actor',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='managed_domains', to='federation.Actor'),
),
migrations.AlterField(
model_name='domain',
name='name',
field=models.CharField(max_length=255, primary_key=True, serialize=False, validators=[funkwhale_api.common.validators.DomainValidator()]),
),
migrations.AlterField(
model_name='domain',
name='nodeinfo',
field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=funkwhale_api.federation.models.empty_dict, max_length=50000),
),
]
...@@ -46,7 +46,9 @@ class FederationMixin(models.Model): ...@@ -46,7 +46,9 @@ class FederationMixin(models.Model):
class ActorQuerySet(models.QuerySet): class ActorQuerySet(models.QuerySet):
def local(self, include=True): def local(self, include=True):
return self.exclude(user__isnull=include) if include:
return self.filter(domain__name=settings.FEDERATION_HOSTNAME)
return self.exclude(domain__name=settings.FEDERATION_HOSTNAME)
def with_current_usage(self): def with_current_usage(self):
qs = self qs = self
...@@ -92,7 +94,13 @@ class Domain(models.Model): ...@@ -92,7 +94,13 @@ class Domain(models.Model):
creation_date = models.DateTimeField(default=timezone.now) creation_date = models.DateTimeField(default=timezone.now)
nodeinfo_fetch_date = models.DateTimeField(default=None, null=True, blank=True) nodeinfo_fetch_date = models.DateTimeField(default=None, null=True, blank=True)
nodeinfo = JSONField(default=empty_dict, max_length=50000, blank=True) nodeinfo = JSONField(default=empty_dict, max_length=50000, blank=True)
service_actor = models.ForeignKey(
"Actor",
related_name="managed_domains",
on_delete=models.SET_NULL,
null=True,
blank=True,
)
objects = DomainQuerySet.as_manager() objects = DomainQuerySet.as_manager()
def __str__(self): def __str__(self):
......
...@@ -11,6 +11,7 @@ from requests.exceptions import RequestException ...@@ -11,6 +11,7 @@ from requests.exceptions import RequestException
from funkwhale_api.common import preferences from funkwhale_api.common import preferences
from funkwhale_api.common import session from funkwhale_api.common import session
from funkwhale_api.common import utils as common_utils
from funkwhale_api.music import models as music_models from funkwhale_api.music import models as music_models
from funkwhale_api.taskapp import celery from funkwhale_api.taskapp import celery
...@@ -18,6 +19,7 @@ from . import keys ...@@ -18,6 +19,7 @@ from . import keys
from . import models, signing from . import models, signing
from . import serializers from . import serializers
from . import routes from . import routes
from . import utils
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
...@@ -184,9 +186,27 @@ def update_domain_nodeinfo(domain): ...@@ -184,9 +186,27 @@ def update_domain_nodeinfo(domain):
nodeinfo = {"status": "ok", "payload": fetch_nodeinfo(domain.name)} nodeinfo = {"status": "ok", "payload": fetch_nodeinfo(domain.name)}
except (requests.RequestException, serializers.serializers.ValidationError) as e: except (requests.RequestException, serializers.serializers.ValidationError) as e:
nodeinfo = {"status": "error", "error": str(e)} nodeinfo = {"status": "error", "error": str(e)}
service_actor_id = common_utils.recursive_getattr(
nodeinfo, "payload.metadata.actorId", permissive=True
)
try:
domain.service_actor = (
utils.retrieve_ap_object(
service_actor_id,
queryset=models.Actor,
serializer_class=serializers.ActorSerializer,
)
if service_actor_id
else None
)
except (serializers.serializers.ValidationError, RequestException) as e:
logger.warning(
"Cannot fetch system actor for domain %s: %s", domain.name, str(e)
)
domain.nodeinfo_fetch_date = now domain.nodeinfo_fetch_date = now
domain.nodeinfo = nodeinfo domain.nodeinfo = nodeinfo
domain.save(update_fields=["nodeinfo", "nodeinfo_fetch_date"]) domain.save(update_fields=["nodeinfo", "nodeinfo_fetch_date", "service_actor"])
def delete_qs(qs): def delete_qs(qs):
......
...@@ -2,6 +2,7 @@ import memoize.djangocache ...@@ -2,6 +2,7 @@ import memoize.djangocache
import funkwhale_api import funkwhale_api
from funkwhale_api.common import preferences from funkwhale_api.common import preferences
from funkwhale_api.federation import actors
from . import stats from . import stats
...@@ -19,6 +20,7 @@ def get(): ...@@ -19,6 +20,7 @@ def get():
"openRegistrations": preferences.get("users__registration_enabled"), "openRegistrations": preferences.get("users__registration_enabled"),
"usage": {"users": {"total": 0, "activeHalfyear": 0, "activeMonth": 0}}, "usage": {"users": {"total": 0, "activeHalfyear": 0, "activeMonth": 0}},
"metadata": { "metadata": {
"actorId": actors.get_service_actor().fid,
"private": preferences.get("instance__nodeinfo_private"), "private": preferences.get("instance__nodeinfo_private"),
"shortDescription": preferences.get("instance__short_description"), "shortDescription": preferences.get("instance__short_description"),
"longDescription": preferences.get("instance__long_description"), "longDescription": preferences.get("instance__long_description"),
......
...@@ -245,41 +245,52 @@ class Invitation(models.Model): ...@@ -245,41 +245,52 @@ class Invitation(models.Model):
return super().save(**kwargs) return super().save(**kwargs)
def get_actor_data(user): def get_actor_data(username):
username = federation_utils.slugify_username(user.username) slugified_username = federation_utils.slugify_username(username)
return { return {
"preferred_username": username, "preferred_username": slugified_username,
"domain": federation_models.Domain.objects.get_or_create( "domain": federation_models.Domain.objects.get_or_create(
name=settings.FEDERATION_HOSTNAME name=settings.FEDERATION_HOSTNAME
)[0], )[0],
"type": "Person", "type": "Person",
"name": user.username, "name": username,
"manually_approves_followers": False, "manually_approves_followers": False,
"fid": federation_utils.full_url( "fid": federation_utils.full_url(
reverse("federation:actors-detail", kwargs={"preferred_username": username}) reverse(
"federation:actors-detail",
kwargs={"preferred_username": slugified_username},
)
), ),
"shared_inbox_url": federation_models.get_shared_inbox_url(), "shared_inbox_url": federation_models.get_shared_inbox_url(),
"inbox_url": federation_utils.full_url( "inbox_url": federation_utils.full_url(
reverse("federation:actors-inbox", kwargs={"preferred_username": username}) reverse(
"federation:actors-inbox",
kwargs={"preferred_username": slugified_username},
)
), ),
"outbox_url": federation_utils.full_url( "outbox_url": federation_utils.full_url(
reverse("federation:actors-outbox", kwargs={"preferred_username": username}) reverse(
"federation:actors-outbox",
kwargs={"preferred_username": slugified_username},
)
), ),
"followers_url": federation_utils.full_url( "followers_url": federation_utils.full_url(
reverse( reverse(
"federation:actors-followers", kwargs={"preferred_username": username} "federation:actors-followers",
kwargs={"preferred_username": slugified_username},
) )
), ),
"following_url": federation_utils.full_url( "following_url": federation_utils.full_url(
reverse( reverse(
"federation:actors-following", kwargs={"preferred_username": username} "federation:actors-following",
kwargs={"preferred_username": slugified_username},
) )
), ),
} }
def create_actor(user): def create_actor(user):
args = get_actor_data(user) args = get_actor_data(user.username)
private, public = keys.get_key_pair() private, public = keys.get_key_pair()
args["private_key"] = private.decode("utf-8") args["private_key"] = private.decode("utf-8")
args["public_key"] = public.decode("utf-8") args["public_key"] = public.decode("utf-8")
......
...@@ -46,3 +46,15 @@ def test_get_actor_refresh(factories, preferences, mocker): ...@@ -46,3 +46,15 @@ def test_get_actor_refresh(factories, preferences, mocker):
assert new_actor == actor assert new_actor == actor
assert new_actor.last_fetch_date > actor.last_fetch_date assert new_actor.last_fetch_date > actor.last_fetch_date
assert new_actor.preferred_username == "New me" assert new_actor.preferred_username == "New me"
def test_get_service_actor(db, settings):
settings.FEDERATION_HOSTNAME = "test.hello"
settings.FEDERATION_SERVICE_ACTOR_USERNAME = "bob"
actor = actors.get_service_actor()
assert actor.preferred_username == "bob"
assert actor.domain.name == "test.hello"
assert actor.private_key is not None
assert actor.type == "Service"
assert actor.public_key is not None
...@@ -5,6 +5,7 @@ from funkwhale_api.federation import authentication, exceptions, keys ...@@ -5,6 +5,7 @@ from funkwhale_api.federation import authentication, exceptions, keys
def test_authenticate(factories, mocker, api_request): def test_authenticate(factories, mocker, api_request):
private, public = keys.get_key_pair() private, public = keys.get_key_pair()
factories["federation.Domain"](name="test.federation", nodeinfo_fetch_date=None)
actor_url = "https://test.federation/actor" actor_url = "https://test.federation/actor"
mocker.patch( mocker.patch(
"funkwhale_api.federation.actors.get_actor_data", "funkwhale_api.federation.actors.get_actor_data",
...@@ -22,6 +23,10 @@ def test_authenticate(factories, mocker, api_request): ...@@ -22,6 +23,10 @@ def test_authenticate(factories, mocker, api_request):
}, },
}, },
) )
update_domain_nodeinfo = mocker.patch(
"funkwhale_api.federation.tasks.update_domain_nodeinfo"
)
signed_request = factories["federation.SignedRequest"]( signed_request = factories["federation.SignedRequest"](
auth__key=private, auth__key_id=actor_url + "#main-key", auth__headers=["date"] auth__key=private, auth__key_id=actor_url + "#main-key", auth__headers=["date"]
) )
...@@ -40,6 +45,7 @@ def test_authenticate(factories, mocker, api_request): ...@@ -40,6 +45,7 @@ def test_authenticate(factories, mocker, api_request):
assert user.is_anonymous is True assert user.is_anonymous is True
assert actor.public_key == public.decode("utf-8") assert actor.public_key == public.decode("utf-8")
assert actor.fid == actor_url assert actor.fid == actor_url
update_domain_nodeinfo.assert_called_once_with(domain_name="test.federation")
def test_authenticate_skips_blocked_domain(factories, api_request): def test_authenticate_skips_blocked_domain(factories, api_request):
......
...@@ -161,22 +161,32 @@ def test_fetch_nodeinfo(factories, r_mock, now): ...@@ -161,22 +161,32 @@ def test_fetch_nodeinfo(factories, r_mock, now):
def test_update_domain_nodeinfo(factories, mocker, now): def test_update_domain_nodeinfo(factories, mocker, now):
domain = factories["federation.Domain"]() domain = factories["federation.Domain"](nodeinfo_fetch_date=None)
mocker.patch.object(tasks, "fetch_nodeinfo", return_value={"hello": "world"}) actor = factories["federation.Actor"](fid="https://actor.id")
mocker.patch.object(
tasks,
"fetch_nodeinfo",
return_value={"hello": "world", "metadata": {"actorId": "https://actor.id"}},
)
assert domain.nodeinfo == {} assert domain.nodeinfo == {}
assert domain.nodeinfo_fetch_date is None assert domain.nodeinfo_fetch_date is None
assert domain.service_actor is None
tasks.update_domain_nodeinfo(domain_name=domain.name) tasks.update_domain_nodeinfo(domain_name=domain.name)
domain.refresh_from_db() domain.refresh_from_db()
assert domain.nodeinfo_fetch_date == now assert domain.nodeinfo_fetch_date == now
assert domain.nodeinfo == {"status": "ok", "payload": {"hello": "world"}} assert domain.nodeinfo == {
"status": "ok",
"payload": {"hello": "world", "metadata": {"actorId": "https://actor.id"}},
}
assert domain.service_actor == actor
def test_update_domain_nodeinfo_error(factories, r_mock, now): def test_update_domain_nodeinfo_error(factories, r_mock, now):
domain = factories["federation.Domain"]() domain = factories["federation.Domain"](nodeinfo_fetch_date=None)
wellknown_url = "https://{}/.well-known/nodeinfo".format(domain.name) wellknown_url = "https://{}/.well-known/nodeinfo".format(domain.name)
r_mock.get(wellknown_url, status_code=500) r_mock.get(wellknown_url, status_code=500)
......
...@@ -2,7 +2,7 @@ import pytest ...@@ -2,7 +2,7 @@ import pytest
from django.core.paginator import Paginator from django.core.paginator import Paginator
from django.urls import reverse from django.urls import reverse
from funkwhale_api.federation import serializers, webfinger from funkwhale_api.federation import actors, serializers, webfinger
def test_wellknown_webfinger_validates_resource(db, api_client, settings, mocker): def test_wellknown_webfinger_validates_resource(db, api_client, settings, mocker):
...@@ -54,6 +54,19 @@ def test_local_actor_detail(factories, api_client): ...@@ -54,6 +54,19 @@ def test_local_actor_detail(factories, api_client):
assert response.data == serializer.data assert response.data == serializer.data