Commit cdc617be authored by Eliot Berriot's avatar Eliot Berriot
Browse files

Logic to refetch remote entities

parent 63b10075
......@@ -30,6 +30,14 @@ class DomainAdmin(admin.ModelAdmin):
search_fields = ["name"]
@admin.register(models.Fetch)
class FetchAdmin(admin.ModelAdmin):
list_display = ["url", "actor", "status", "creation_date", "fetch_date", "detail"]
search_fields = ["url", "actor__username"]
list_filter = ["status"]
list_select_related = True
@admin.register(models.Activity)
class ActivityAdmin(admin.ModelAdmin):
list_display = ["type", "fid", "url", "actor", "creation_date"]
......
......@@ -144,3 +144,19 @@ class InboxItemActionSerializer(common_serializers.ActionSerializer):
def handle_read(self, objects):
return objects.update(is_read=True)
class FetchSerializer(serializers.ModelSerializer):
actor = federation_serializers.APIActorSerializer()
class Meta:
model = models.Fetch
fields = [
"id",
"url",
"actor",
"status",
"detail",
"creation_date",
"fetch_date",
]
......@@ -3,6 +3,7 @@ from rest_framework import routers
from . import api_views
router = routers.SimpleRouter()
router.register(r"fetches", api_views.FetchViewSet, "fetches")
router.register(r"follows/library", api_views.LibraryFollowViewSet, "library-follows")
router.register(r"inbox", api_views.InboxItemViewSet, "inbox")
router.register(r"libraries", api_views.LibraryViewSet, "libraries")
......
......@@ -5,6 +5,7 @@ from django.db.models import Count
from rest_framework import decorators
from rest_framework import mixins
from rest_framework import permissions
from rest_framework import response
from rest_framework import viewsets
......@@ -189,3 +190,10 @@ class InboxItemViewSet(
serializer.is_valid(raise_exception=True)
result = serializer.save()
return response.Response(result, status=200)
class FetchViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet):
queryset = models.Fetch.objects.select_related("actor")
serializer_class = api_serializers.FetchSerializer
permission_classes = [permissions.IsAuthenticated]
from django.db import transaction
from rest_framework import decorators
from rest_framework import permissions
from rest_framework import response
from rest_framework import status
from funkwhale_api.common import utils as common_utils
from . import api_serializers
from . import filters
from . import models
from . import tasks
from . import utils
def fetches_route():
@transaction.atomic
def fetches(self, request, *args, **kwargs):
obj = self.get_object()
if request.method == "GET":
queryset = models.Fetch.objects.get_for_object(obj).select_related("actor")
queryset = queryset.order_by("-creation_date")
filterset = filters.FetchFilter(request.GET, queryset=queryset)
page = self.paginate_queryset(filterset.qs)
if page is not None:
serializer = api_serializers.FetchSerializer(page, many=True)
return self.get_paginated_response(serializer.data)
serializer = api_serializers.FetchSerializer(queryset, many=True)
return response.Response(serializer.data)
if request.method == "POST":
if utils.is_local(obj.fid):
return response.Response(
{"detail": "Cannot fetch a local object"}, status=400
)
fetch = models.Fetch.objects.create(
url=obj.fid, actor=request.user.actor, object=obj
)
common_utils.on_commit(tasks.fetch.delay, fetch_id=fetch.pk)
serializer = api_serializers.FetchSerializer(fetch)
return response.Response(serializer.data, status=status.HTTP_201_CREATED)
return decorators.action(
methods=["get", "post"],
detail=True,
permission_classes=[permissions.IsAuthenticated],
)(fetches)
......@@ -166,7 +166,7 @@ class MusicLibraryFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
@registry.register
class LibraryScan(NoUpdateOnCreate, factory.django.DjangoModelFactory):
class LibraryScanFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
library = factory.SubFactory(MusicLibraryFactory)
actor = factory.SubFactory(ActorFactory)
total_files = factory.LazyAttribute(lambda o: o.library.uploads_count)
......@@ -175,6 +175,14 @@ class LibraryScan(NoUpdateOnCreate, factory.django.DjangoModelFactory):
model = "music.LibraryScan"
@registry.register
class FetchFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
actor = factory.SubFactory(ActorFactory)
class Meta:
model = "federation.Fetch"
@registry.register
class ActivityFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
actor = factory.SubFactory(ActorFactory)
......
......@@ -46,3 +46,14 @@ class InboxItemFilter(django_filters.FilterSet):
def filter_before(self, queryset, field_name, value):
return queryset.filter(pk__lte=value)
class FetchFilter(django_filters.FilterSet):
ordering = django_filters.OrderingFilter(
# tuple-mapping retains order
fields=(("creation_date", "creation_date"), ("fetch_date", "fetch_date"))
)
class Meta:
model = models.Fetch
fields = ["status", "object_id", "url"]
# Generated by Django 2.1.7 on 2019-04-17 14:57
import django.contrib.postgres.fields.jsonb
import django.core.serializers.json
from django.db import migrations, models
import django.db.models.deletion
import django.utils.timezone
import funkwhale_api.federation.models
class Migration(migrations.Migration):
dependencies = [
('contenttypes', '0002_remove_content_type_name'),
('federation', '0017_auto_20190130_0926'),
]
operations = [
migrations.CreateModel(
name='Fetch',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('url', models.URLField(db_index=True, max_length=500)),
('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
('fetch_date', models.DateTimeField(blank=True, null=True)),
('object_id', models.IntegerField(null=True)),
('status', models.CharField(choices=[('pending', 'Pending'), ('errored', 'Errored'), ('finished', 'Finished'), ('skipped', 'Skipped')], default='pending', max_length=20)),
('detail', django.contrib.postgres.fields.jsonb.JSONField(default=funkwhale_api.federation.models.empty_dict, encoder=django.core.serializers.json.DjangoJSONEncoder, max_length=50000)),
('actor', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='fetches', to='federation.Actor')),
('object_content_type', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')),
],
),
]
......@@ -288,6 +288,55 @@ class Actor(models.Model):
)
FETCH_STATUSES = [
("pending", "Pending"),
("errored", "Errored"),
("finished", "Finished"),
("skipped", "Skipped"),
]
class FetchQuerySet(models.QuerySet):
def get_for_object(self, object):
content_type = ContentType.objects.get_for_model(object)
return self.filter(object_content_type=content_type, object_id=object.pk)
class Fetch(models.Model):
url = models.URLField(max_length=500, db_index=True)
creation_date = models.DateTimeField(default=timezone.now)
fetch_date = models.DateTimeField(null=True, blank=True)
object_id = models.IntegerField(null=True)
object_content_type = models.ForeignKey(
ContentType, null=True, on_delete=models.CASCADE
)
object = GenericForeignKey("object_content_type", "object_id")
status = models.CharField(default="pending", choices=FETCH_STATUSES, max_length=20)
detail = JSONField(default=empty_dict, max_length=50000, encoder=DjangoJSONEncoder)
actor = models.ForeignKey(Actor, related_name="fetches", on_delete=models.CASCADE)
objects = FetchQuerySet.as_manager()
def save(self, **kwargs):
if not self.url and self.object:
self.url = self.object.fid
super().save(**kwargs)
@property
def serializers(self):
from . import contexts
from . import serializers
return {
contexts.FW.Artist: serializers.ArtistSerializer,
contexts.FW.Album: serializers.AlbumSerializer,
contexts.FW.Track: serializers.TrackSerializer,
contexts.AS.Audio: serializers.UploadSerializer,
contexts.FW.Library: serializers.LibrarySerializer,
}
class InboxItem(models.Model):
"""
Store activities binding to local actors, with read/unread status.
......
......@@ -771,6 +771,7 @@ class ArtistSerializer(MusicEntitySerializer):
]
class Meta:
model = music_models.Artist
jsonld_mapping = MUSIC_ENTITY_JSONLD_MAPPING
def to_representation(self, instance):
......@@ -804,6 +805,7 @@ class AlbumSerializer(MusicEntitySerializer):
]
class Meta:
model = music_models.Album
jsonld_mapping = funkwhale_utils.concat_dicts(
MUSIC_ENTITY_JSONLD_MAPPING,
{
......@@ -863,6 +865,7 @@ class TrackSerializer(MusicEntitySerializer):
]
class Meta:
model = music_models.Track
jsonld_mapping = funkwhale_utils.concat_dicts(
MUSIC_ENTITY_JSONLD_MAPPING,
{
......@@ -970,6 +973,7 @@ class UploadSerializer(jsonld.JsonLdSerializer):
track = TrackSerializer(required=True)
class Meta:
model = music_models.Upload
jsonld_mapping = {
"track": jsonld.first_obj(contexts.FW.track),
"library": jsonld.first_id(contexts.FW.library),
......
import datetime
import json
import logging
import os
import requests
from django.conf import settings
from django.db import transaction
from django.db.models import Q, F
from django.utils import timezone
from dynamic_preferences.registries import global_preferences_registry
......@@ -16,6 +18,7 @@ from funkwhale_api.music import models as music_models
from funkwhale_api.taskapp import celery
from . import actors
from . import jsonld
from . import keys
from . import models, signing
from . import serializers
......@@ -278,3 +281,83 @@ def rotate_actor_key(actor):
actor.private_key = pair[0].decode()
actor.public_key = pair[1].decode()
actor.save(update_fields=["private_key", "public_key"])
@celery.app.task(name="federation.fetch")
@transaction.atomic
@celery.require_instance(
models.Fetch.objects.filter(status="pending").select_related("actor"), "fetch"
)
def fetch(fetch):
actor = fetch.actor
auth = signing.get_auth(actor.private_key, actor.private_key_id)
def error(code, **kwargs):
fetch.status = "errored"
fetch.fetch_date = timezone.now()
fetch.detail = {"error_code": code}
fetch.detail.update(kwargs)
fetch.save(update_fields=["fetch_date", "status", "detail"])
try:
response = session.get_session().get(
auth=auth,
url=fetch.url,
timeout=5,
verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
headers={"Content-Type": "application/activity+json"},
)
logger.debug("Remote answered with %s", response.status_code)
response.raise_for_status()
except requests.exceptions.HTTPError as e:
return error("http", status_code=e.response.status_code if e.response else None)
except requests.exceptions.Timeout:
return error("timeout")
except requests.exceptions.ConnectionError as e:
return error("connection", message=str(e))
except requests.RequestException as e:
return error("request", message=str(e))
except Exception as e:
return error("unhandled", message=str(e))
try:
payload = response.json()
except json.decoder.JSONDecodeError:
return error("invalid_json")
try:
doc = jsonld.expand(payload)
except ValueError:
return error("invalid_jsonld")
try:
type = doc.get("@type", [])[0]
except IndexError:
return error("missing_jsonld_type")
try:
serializer_class = fetch.serializers[type]
model = serializer_class.Meta.model
except (KeyError, AttributeError):
fetch.status = "skipped"
fetch.fetch_date = timezone.now()
fetch.detail = {"reason": "unhandled_type", "type": type}
return fetch.save(update_fields=["fetch_date", "status", "detail"])
try:
id = doc.get("@id")
except IndexError:
existing = None
else:
existing = model.objects.filter(fid=id).first()
serializer = serializer_class(existing, data=payload)
if not serializer.is_valid():
return error("validation", validation_errors=serializer.errors)
try:
serializer.save()
except Exception as e:
error("save", message=str(e))
raise
fetch.status = "finished"
fetch.fetch_date = timezone.now()
return fetch.save(update_fields=["fetch_date", "status"])
......@@ -121,3 +121,13 @@ def get_domain_query_from_url(domain, url_field="fid"):
**{"{}__startswith".format(url_field): "https://{}/".format(domain)}
)
return query
def is_local(url):
if not url:
return True
d = settings.FEDERATION_HOSTNAME
return url.startswith("http://{}/".format(d)) or url.startswith(
"https://{}/".format(d)
)
......@@ -117,13 +117,7 @@ class APIModelMixin(models.Model):
@property
def is_local(self):
if not self.fid:
return True
d = settings.FEDERATION_HOSTNAME
return self.fid.startswith("http://{}/".format(d)) or self.fid.startswith(
"https://{}/".format(d)
)
return federation_utils.is_local(self.fid)
@property
def domain_name(self):
......
......@@ -22,6 +22,7 @@ from funkwhale_api.common import views as common_views
from funkwhale_api.federation.authentication import SignatureAuthentication
from funkwhale_api.federation import actors
from funkwhale_api.federation import api_serializers as federation_api_serializers
from funkwhale_api.federation import decorators as federation_decorators
from funkwhale_api.federation import routes
from funkwhale_api.users.oauth import permissions as oauth_permissions
......@@ -70,6 +71,7 @@ class ArtistViewSet(common_views.SkipFilterForGetObject, viewsets.ReadOnlyModelV
filterset_class = filters.ArtistFilter
ordering_fields = ("id", "name", "creation_date")
fetches = federation_decorators.fetches_route()
mutations = common_decorators.mutations_route(types=["update"])
def get_queryset(self):
......@@ -100,6 +102,7 @@ class AlbumViewSet(common_views.SkipFilterForGetObject, viewsets.ReadOnlyModelVi
ordering_fields = ("creation_date", "release_date", "title")
filterset_class = filters.AlbumFilter
fetches = federation_decorators.fetches_route()
mutations = common_decorators.mutations_route(types=["update"])
def get_queryset(self):
......@@ -201,7 +204,7 @@ class TrackViewSet(
"disc_number",
"artist__name",
)
fetches = federation_decorators.fetches_route()
mutations = common_decorators.mutations_route(types=["update"])
def get_queryset(self):
......@@ -437,6 +440,8 @@ class UploadViewSet(
"artist__name",
)
fetches = federation_decorators.fetches_route()
def get_queryset(self):
qs = super().get_queryset()
return qs.filter(library__actor=self.request.user.actor)
......
......@@ -167,3 +167,15 @@ def test_user_can_update_read_status_of_inbox_item(factories, logged_in_api_clie
ii.refresh_from_db()
assert ii.is_read is True
def test_can_detail_fetch(logged_in_api_client, factories):
fetch = factories["federation.Fetch"](url="http://test.object")
url = reverse("api:v1:federation:fetches-detail", kwargs={"pk": fetch.pk})
response = logged_in_api_client.get(url)
expected = api_serializers.FetchSerializer(fetch).data
assert response.status_code == 200
assert response.data == expected
from rest_framework import viewsets
from funkwhale_api.music import models as music_models
from funkwhale_api.federation import api_serializers
from funkwhale_api.federation import decorators
from funkwhale_api.federation import models
from funkwhale_api.federation import tasks
class V(viewsets.ModelViewSet):
queryset = music_models.Track.objects.all()
fetches = decorators.fetches_route()
permission_classes = []
def test_fetches_route_create(factories, api_request, mocker):
on_commit = mocker.patch("funkwhale_api.common.utils.on_commit")
user = factories["users.User"]()
actor = user.create_actor()
track = factories["music.Track"]()
view = V.as_view({"post": "fetches"})
request = api_request.post("/", format="json")
setattr(request, "user", user)
setattr(request, "session", {})
response = view(request, pk=track.pk)
assert response.status_code == 201
fetch = models.Fetch.objects.get_for_object(track).latest("id")
on_commit.assert_called_once_with(tasks.fetch.delay, fetch_id=fetch.pk)
assert fetch.url == track.fid
assert fetch.object == track
assert fetch.status == "pending"
assert fetch.actor == actor
expected = api_serializers.FetchSerializer(fetch).data
assert response.data == expected
def test_fetches_route_create_local(factories, api_request, mocker, settings):
user = factories["users.User"]()
user.create_actor()
track = factories["music.Track"](
fid="https://{}/test".format(settings.FEDERATION_HOSTNAME)
)
view = V.as_view({"post": "fetches"})
request = api_request.post("/", format="json")
setattr(request, "user", user)
setattr(request, "session", {})
response = view(request, pk=track.pk)
assert response.status_code == 400
def test_fetches_route_list(factories, api_request, mocker):
user = factories["users.User"]()
user.create_actor()
track = factories["music.Track"]()
fetches = [
factories["federation.Fetch"](object=track),
factories["federation.Fetch"](object=track),
]
view = V.as_view({"get": "fetches"})
request = api_request.get("/", format="json")
setattr(request, "user", user)
setattr(request, "session", {})
expected = {
"next": None,
"previous": None,
"count": 2,
"results": api_serializers.FetchSerializer(reversed(fetches), many=True).data,
}
request = api_request.get("/")
response = view(request, pk=track.pk)
assert response.status_code == 200
assert response.data == expected
......@@ -164,3 +164,12 @@ def test_actor_can_manage_domain_service_actor(mocker, factories):
obj = mocker.Mock(fid="https://{}/hello".format(actor.domain_id))
assert actor.can_manage(obj) is True
def test_can_create_fetch_for_object(factories):
track = factories["music.Track"](fid="http://test.domain")
fetch = factories["federation.Fetch"](object=track)
assert fetch.url == "http://test.domain"
assert fetch.status == "pending"
assert fetch.detail == {}
assert fetch.object == track
......@@ -5,6 +5,7 @@ import pytest
from django.utils import timezone
from funkwhale_api.federation import jsonld
from funkwhale_api.federation import models
from funkwhale_api.federation import serializers
from funkwhale_api.federation import tasks
......@@ -332,3 +333,60 @@ def test_rotate_actor_key(factories, settings, mocker):
assert actor.public_key == "public"
assert actor.private_key == "private"
def test_fetch_skipped(factories, r_mock):
url = "https://fetch.object"
fetch = factories["federation.Fetch"](url=url)
payload = {"@context": jsonld.get_default_context(), "type": "Unhandled"}
r_mock.get(url, json=payload)
tasks.fetch(fetch_id=fetch.pk)
fetch.refresh_from_db()
assert fetch