diff --git a/api/funkwhale_api/federation/admin.py b/api/funkwhale_api/federation/admin.py
index 8c9bbe31c8062bda8fd9192b14170f5289620b13..263af80cb175e1235ebfd654a3daf142c19ea927 100644
--- a/api/funkwhale_api/federation/admin.py
+++ b/api/funkwhale_api/federation/admin.py
@@ -30,6 +30,14 @@ class DomainAdmin(admin.ModelAdmin):
search_fields = ["name"]
+@admin.register(models.Fetch)
+class FetchAdmin(admin.ModelAdmin):
+ list_display = ["url", "actor", "status", "creation_date", "fetch_date", "detail"]
+ search_fields = ["url", "actor__username"]
+ list_filter = ["status"]
+ list_select_related = True
+
+
@admin.register(models.Activity)
class ActivityAdmin(admin.ModelAdmin):
list_display = ["type", "fid", "url", "actor", "creation_date"]
diff --git a/api/funkwhale_api/federation/api_serializers.py b/api/funkwhale_api/federation/api_serializers.py
index 9041ed28a40b52a49c63c44b07d845077bc309d7..dbc655a47d11c677f7b98ec1a34cbbeacc6882c0 100644
--- a/api/funkwhale_api/federation/api_serializers.py
+++ b/api/funkwhale_api/federation/api_serializers.py
@@ -144,3 +144,19 @@ class InboxItemActionSerializer(common_serializers.ActionSerializer):
def handle_read(self, objects):
return objects.update(is_read=True)
+
+
+class FetchSerializer(serializers.ModelSerializer):
+ actor = federation_serializers.APIActorSerializer()
+
+ class Meta:
+ model = models.Fetch
+ fields = [
+ "id",
+ "url",
+ "actor",
+ "status",
+ "detail",
+ "creation_date",
+ "fetch_date",
+ ]
diff --git a/api/funkwhale_api/federation/api_urls.py b/api/funkwhale_api/federation/api_urls.py
index e1e451bff957d3d6dc78f4610bbdb0e77f0069c2..bd2258de961f8d0c80f8b2a3b73d5dd6a9a8ca07 100644
--- a/api/funkwhale_api/federation/api_urls.py
+++ b/api/funkwhale_api/federation/api_urls.py
@@ -3,6 +3,7 @@ from rest_framework import routers
from . import api_views
router = routers.SimpleRouter()
+router.register(r"fetches", api_views.FetchViewSet, "fetches")
router.register(r"follows/library", api_views.LibraryFollowViewSet, "library-follows")
router.register(r"inbox", api_views.InboxItemViewSet, "inbox")
router.register(r"libraries", api_views.LibraryViewSet, "libraries")
diff --git a/api/funkwhale_api/federation/api_views.py b/api/funkwhale_api/federation/api_views.py
index 0fe044ec1921e36fc452389b62b90d4121229b19..5f6f50d8fa5e273f7b678787f324422f1f3f9211 100644
--- a/api/funkwhale_api/federation/api_views.py
+++ b/api/funkwhale_api/federation/api_views.py
@@ -5,6 +5,7 @@ from django.db.models import Count
from rest_framework import decorators
from rest_framework import mixins
+from rest_framework import permissions
from rest_framework import response
from rest_framework import viewsets
@@ -189,3 +190,10 @@ class InboxItemViewSet(
serializer.is_valid(raise_exception=True)
result = serializer.save()
return response.Response(result, status=200)
+
+
+class FetchViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet):
+
+ queryset = models.Fetch.objects.select_related("actor")
+ serializer_class = api_serializers.FetchSerializer
+ permission_classes = [permissions.IsAuthenticated]
diff --git a/api/funkwhale_api/federation/decorators.py b/api/funkwhale_api/federation/decorators.py
new file mode 100644
index 0000000000000000000000000000000000000000..3d2d62567613f7899eea0c2a9356812150bcd160
--- /dev/null
+++ b/api/funkwhale_api/federation/decorators.py
@@ -0,0 +1,49 @@
+from django.db import transaction
+
+from rest_framework import decorators
+from rest_framework import permissions
+from rest_framework import response
+from rest_framework import status
+
+from funkwhale_api.common import utils as common_utils
+
+from . import api_serializers
+from . import filters
+from . import models
+from . import tasks
+from . import utils
+
+
+def fetches_route():
+ @transaction.atomic
+ def fetches(self, request, *args, **kwargs):
+ obj = self.get_object()
+ if request.method == "GET":
+ queryset = models.Fetch.objects.get_for_object(obj).select_related("actor")
+ queryset = queryset.order_by("-creation_date")
+ filterset = filters.FetchFilter(request.GET, queryset=queryset)
+ page = self.paginate_queryset(filterset.qs)
+ if page is not None:
+ serializer = api_serializers.FetchSerializer(page, many=True)
+ return self.get_paginated_response(serializer.data)
+
+ serializer = api_serializers.FetchSerializer(queryset, many=True)
+ return response.Response(serializer.data)
+ if request.method == "POST":
+ if utils.is_local(obj.fid):
+ return response.Response(
+ {"detail": "Cannot fetch a local object"}, status=400
+ )
+
+ fetch = models.Fetch.objects.create(
+ url=obj.fid, actor=request.user.actor, object=obj
+ )
+ common_utils.on_commit(tasks.fetch.delay, fetch_id=fetch.pk)
+ serializer = api_serializers.FetchSerializer(fetch)
+ return response.Response(serializer.data, status=status.HTTP_201_CREATED)
+
+ return decorators.action(
+ methods=["get", "post"],
+ detail=True,
+ permission_classes=[permissions.IsAuthenticated],
+ )(fetches)
diff --git a/api/funkwhale_api/federation/factories.py b/api/funkwhale_api/federation/factories.py
index cf9546447a66dc55edd7636f6c25b711b92aed8f..14bb4e8c96ea150a09fbf95bdbbbcb021f270a8c 100644
--- a/api/funkwhale_api/federation/factories.py
+++ b/api/funkwhale_api/federation/factories.py
@@ -166,7 +166,7 @@ class MusicLibraryFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
@registry.register
-class LibraryScan(NoUpdateOnCreate, factory.django.DjangoModelFactory):
+class LibraryScanFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
library = factory.SubFactory(MusicLibraryFactory)
actor = factory.SubFactory(ActorFactory)
total_files = factory.LazyAttribute(lambda o: o.library.uploads_count)
@@ -175,6 +175,14 @@ class LibraryScan(NoUpdateOnCreate, factory.django.DjangoModelFactory):
model = "music.LibraryScan"
+@registry.register
+class FetchFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
+ actor = factory.SubFactory(ActorFactory)
+
+ class Meta:
+ model = "federation.Fetch"
+
+
@registry.register
class ActivityFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
actor = factory.SubFactory(ActorFactory)
diff --git a/api/funkwhale_api/federation/filters.py b/api/funkwhale_api/federation/filters.py
index 3a8b76ceee9e706b806fddef3f82076f89077979..bfc48bcfbfcc6f9c4d3b33794523a5e4b833abc4 100644
--- a/api/funkwhale_api/federation/filters.py
+++ b/api/funkwhale_api/federation/filters.py
@@ -46,3 +46,14 @@ class InboxItemFilter(django_filters.FilterSet):
def filter_before(self, queryset, field_name, value):
return queryset.filter(pk__lte=value)
+
+
+class FetchFilter(django_filters.FilterSet):
+ ordering = django_filters.OrderingFilter(
+ # tuple-mapping retains order
+ fields=(("creation_date", "creation_date"), ("fetch_date", "fetch_date"))
+ )
+
+ class Meta:
+ model = models.Fetch
+ fields = ["status", "object_id", "url"]
diff --git a/api/funkwhale_api/federation/migrations/0018_fetch.py b/api/funkwhale_api/federation/migrations/0018_fetch.py
new file mode 100644
index 0000000000000000000000000000000000000000..11789024fd022cfe25a2cded4984eda34cde6c9d
--- /dev/null
+++ b/api/funkwhale_api/federation/migrations/0018_fetch.py
@@ -0,0 +1,33 @@
+# Generated by Django 2.1.7 on 2019-04-17 14:57
+
+import django.contrib.postgres.fields.jsonb
+import django.core.serializers.json
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import funkwhale_api.federation.models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('contenttypes', '0002_remove_content_type_name'),
+ ('federation', '0017_auto_20190130_0926'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='Fetch',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('url', models.URLField(db_index=True, max_length=500)),
+ ('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
+ ('fetch_date', models.DateTimeField(blank=True, null=True)),
+ ('object_id', models.IntegerField(null=True)),
+ ('status', models.CharField(choices=[('pending', 'Pending'), ('errored', 'Errored'), ('finished', 'Finished'), ('skipped', 'Skipped')], default='pending', max_length=20)),
+ ('detail', django.contrib.postgres.fields.jsonb.JSONField(default=funkwhale_api.federation.models.empty_dict, encoder=django.core.serializers.json.DjangoJSONEncoder, max_length=50000)),
+ ('actor', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='fetches', to='federation.Actor')),
+ ('object_content_type', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')),
+ ],
+ ),
+ ]
diff --git a/api/funkwhale_api/federation/models.py b/api/funkwhale_api/federation/models.py
index f465ea3ac086166ec6bfae0f06bf6c9d2fc5ef59..7d3d5639de3520a2647b773885868c6ff24fa66a 100644
--- a/api/funkwhale_api/federation/models.py
+++ b/api/funkwhale_api/federation/models.py
@@ -288,6 +288,55 @@ class Actor(models.Model):
)
+FETCH_STATUSES = [
+ ("pending", "Pending"),
+ ("errored", "Errored"),
+ ("finished", "Finished"),
+ ("skipped", "Skipped"),
+]
+
+
+class FetchQuerySet(models.QuerySet):
+ def get_for_object(self, object):
+ content_type = ContentType.objects.get_for_model(object)
+ return self.filter(object_content_type=content_type, object_id=object.pk)
+
+
+class Fetch(models.Model):
+ url = models.URLField(max_length=500, db_index=True)
+ creation_date = models.DateTimeField(default=timezone.now)
+ fetch_date = models.DateTimeField(null=True, blank=True)
+ object_id = models.IntegerField(null=True)
+ object_content_type = models.ForeignKey(
+ ContentType, null=True, on_delete=models.CASCADE
+ )
+ object = GenericForeignKey("object_content_type", "object_id")
+ status = models.CharField(default="pending", choices=FETCH_STATUSES, max_length=20)
+ detail = JSONField(default=empty_dict, max_length=50000, encoder=DjangoJSONEncoder)
+ actor = models.ForeignKey(Actor, related_name="fetches", on_delete=models.CASCADE)
+
+ objects = FetchQuerySet.as_manager()
+
+ def save(self, **kwargs):
+ if not self.url and self.object:
+ self.url = self.object.fid
+
+ super().save(**kwargs)
+
+ @property
+ def serializers(self):
+ from . import contexts
+ from . import serializers
+
+ return {
+ contexts.FW.Artist: serializers.ArtistSerializer,
+ contexts.FW.Album: serializers.AlbumSerializer,
+ contexts.FW.Track: serializers.TrackSerializer,
+ contexts.AS.Audio: serializers.UploadSerializer,
+ contexts.FW.Library: serializers.LibrarySerializer,
+ }
+
+
class InboxItem(models.Model):
"""
Store activities binding to local actors, with read/unread status.
diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py
index b32c09bdba9c4bdacc6b56d2f5739c2da4241fec..3e7618c9cf09745098a715f25dea01863fb32e21 100644
--- a/api/funkwhale_api/federation/serializers.py
+++ b/api/funkwhale_api/federation/serializers.py
@@ -771,6 +771,7 @@ class ArtistSerializer(MusicEntitySerializer):
]
class Meta:
+ model = music_models.Artist
jsonld_mapping = MUSIC_ENTITY_JSONLD_MAPPING
def to_representation(self, instance):
@@ -804,6 +805,7 @@ class AlbumSerializer(MusicEntitySerializer):
]
class Meta:
+ model = music_models.Album
jsonld_mapping = funkwhale_utils.concat_dicts(
MUSIC_ENTITY_JSONLD_MAPPING,
{
@@ -863,6 +865,7 @@ class TrackSerializer(MusicEntitySerializer):
]
class Meta:
+ model = music_models.Track
jsonld_mapping = funkwhale_utils.concat_dicts(
MUSIC_ENTITY_JSONLD_MAPPING,
{
@@ -970,6 +973,7 @@ class UploadSerializer(jsonld.JsonLdSerializer):
track = TrackSerializer(required=True)
class Meta:
+ model = music_models.Upload
jsonld_mapping = {
"track": jsonld.first_obj(contexts.FW.track),
"library": jsonld.first_id(contexts.FW.library),
diff --git a/api/funkwhale_api/federation/tasks.py b/api/funkwhale_api/federation/tasks.py
index 38e8eb6776a218d99cc109065913bd76f4bdba82..7a1c7d92bc7c3ce04524fae47e6ee8e51a48bcb8 100644
--- a/api/funkwhale_api/federation/tasks.py
+++ b/api/funkwhale_api/federation/tasks.py
@@ -1,9 +1,11 @@
import datetime
+import json
import logging
import os
import requests
from django.conf import settings
+from django.db import transaction
from django.db.models import Q, F
from django.utils import timezone
from dynamic_preferences.registries import global_preferences_registry
@@ -16,6 +18,7 @@ from funkwhale_api.music import models as music_models
from funkwhale_api.taskapp import celery
from . import actors
+from . import jsonld
from . import keys
from . import models, signing
from . import serializers
@@ -278,3 +281,83 @@ def rotate_actor_key(actor):
actor.private_key = pair[0].decode()
actor.public_key = pair[1].decode()
actor.save(update_fields=["private_key", "public_key"])
+
+
+@celery.app.task(name="federation.fetch")
+@transaction.atomic
+@celery.require_instance(
+ models.Fetch.objects.filter(status="pending").select_related("actor"), "fetch"
+)
+def fetch(fetch):
+ actor = fetch.actor
+ auth = signing.get_auth(actor.private_key, actor.private_key_id)
+
+ def error(code, **kwargs):
+ fetch.status = "errored"
+ fetch.fetch_date = timezone.now()
+ fetch.detail = {"error_code": code}
+ fetch.detail.update(kwargs)
+ fetch.save(update_fields=["fetch_date", "status", "detail"])
+
+ try:
+ response = session.get_session().get(
+ auth=auth,
+ url=fetch.url,
+ timeout=5,
+ verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
+ headers={"Content-Type": "application/activity+json"},
+ )
+ logger.debug("Remote answered with %s", response.status_code)
+ response.raise_for_status()
+ except requests.exceptions.HTTPError as e:
+ return error("http", status_code=e.response.status_code if e.response else None)
+ except requests.exceptions.Timeout:
+ return error("timeout")
+ except requests.exceptions.ConnectionError as e:
+ return error("connection", message=str(e))
+ except requests.RequestException as e:
+ return error("request", message=str(e))
+ except Exception as e:
+ return error("unhandled", message=str(e))
+
+ try:
+ payload = response.json()
+ except json.decoder.JSONDecodeError:
+ return error("invalid_json")
+
+ try:
+ doc = jsonld.expand(payload)
+ except ValueError:
+ return error("invalid_jsonld")
+
+ try:
+ type = doc.get("@type", [])[0]
+ except IndexError:
+ return error("missing_jsonld_type")
+ try:
+ serializer_class = fetch.serializers[type]
+ model = serializer_class.Meta.model
+ except (KeyError, AttributeError):
+ fetch.status = "skipped"
+ fetch.fetch_date = timezone.now()
+ fetch.detail = {"reason": "unhandled_type", "type": type}
+ return fetch.save(update_fields=["fetch_date", "status", "detail"])
+ try:
+ id = doc.get("@id")
+ except IndexError:
+ existing = None
+ else:
+ existing = model.objects.filter(fid=id).first()
+
+ serializer = serializer_class(existing, data=payload)
+ if not serializer.is_valid():
+ return error("validation", validation_errors=serializer.errors)
+ try:
+ serializer.save()
+ except Exception as e:
+ error("save", message=str(e))
+ raise
+
+ fetch.status = "finished"
+ fetch.fetch_date = timezone.now()
+ return fetch.save(update_fields=["fetch_date", "status"])
diff --git a/api/funkwhale_api/federation/utils.py b/api/funkwhale_api/federation/utils.py
index 2bbfdf7fadd023c3349961237f877677e29dc06a..8f73c57350a2290f4fea9e106859b460052a24c0 100644
--- a/api/funkwhale_api/federation/utils.py
+++ b/api/funkwhale_api/federation/utils.py
@@ -121,3 +121,13 @@ def get_domain_query_from_url(domain, url_field="fid"):
**{"{}__startswith".format(url_field): "https://{}/".format(domain)}
)
return query
+
+
+def is_local(url):
+ if not url:
+ return True
+
+ d = settings.FEDERATION_HOSTNAME
+ return url.startswith("http://{}/".format(d)) or url.startswith(
+ "https://{}/".format(d)
+ )
diff --git a/api/funkwhale_api/music/models.py b/api/funkwhale_api/music/models.py
index 4b166be7c3e01b2909375997a5fe5322edd918a6..7ad88d45f442ca3570c4ceaec266086821a1ed96 100644
--- a/api/funkwhale_api/music/models.py
+++ b/api/funkwhale_api/music/models.py
@@ -117,13 +117,7 @@ class APIModelMixin(models.Model):
@property
def is_local(self):
- if not self.fid:
- return True
-
- d = settings.FEDERATION_HOSTNAME
- return self.fid.startswith("http://{}/".format(d)) or self.fid.startswith(
- "https://{}/".format(d)
- )
+ return federation_utils.is_local(self.fid)
@property
def domain_name(self):
diff --git a/api/funkwhale_api/music/views.py b/api/funkwhale_api/music/views.py
index b6df2214351499858fec275da4f535d6827f18cd..336a87ce0cc46af663c1be350fcb0e7f86cb3df5 100644
--- a/api/funkwhale_api/music/views.py
+++ b/api/funkwhale_api/music/views.py
@@ -22,6 +22,7 @@ from funkwhale_api.common import views as common_views
from funkwhale_api.federation.authentication import SignatureAuthentication
from funkwhale_api.federation import actors
from funkwhale_api.federation import api_serializers as federation_api_serializers
+from funkwhale_api.federation import decorators as federation_decorators
from funkwhale_api.federation import routes
from funkwhale_api.users.oauth import permissions as oauth_permissions
@@ -70,6 +71,7 @@ class ArtistViewSet(common_views.SkipFilterForGetObject, viewsets.ReadOnlyModelV
filterset_class = filters.ArtistFilter
ordering_fields = ("id", "name", "creation_date")
+ fetches = federation_decorators.fetches_route()
mutations = common_decorators.mutations_route(types=["update"])
def get_queryset(self):
@@ -100,6 +102,7 @@ class AlbumViewSet(common_views.SkipFilterForGetObject, viewsets.ReadOnlyModelVi
ordering_fields = ("creation_date", "release_date", "title")
filterset_class = filters.AlbumFilter
+ fetches = federation_decorators.fetches_route()
mutations = common_decorators.mutations_route(types=["update"])
def get_queryset(self):
@@ -201,7 +204,7 @@ class TrackViewSet(
"disc_number",
"artist__name",
)
-
+ fetches = federation_decorators.fetches_route()
mutations = common_decorators.mutations_route(types=["update"])
def get_queryset(self):
@@ -437,6 +440,8 @@ class UploadViewSet(
"artist__name",
)
+ fetches = federation_decorators.fetches_route()
+
def get_queryset(self):
qs = super().get_queryset()
return qs.filter(library__actor=self.request.user.actor)
diff --git a/api/tests/federation/test_api_views.py b/api/tests/federation/test_api_views.py
index 75579d39a2d11d4ac2a1c4207ab30efdb477d470..c34c5e99aa1ec74ee2bbdd32bb33142297f54ff9 100644
--- a/api/tests/federation/test_api_views.py
+++ b/api/tests/federation/test_api_views.py
@@ -167,3 +167,15 @@ def test_user_can_update_read_status_of_inbox_item(factories, logged_in_api_clie
ii.refresh_from_db()
assert ii.is_read is True
+
+
+def test_can_detail_fetch(logged_in_api_client, factories):
+ fetch = factories["federation.Fetch"](url="http://test.object")
+ url = reverse("api:v1:federation:fetches-detail", kwargs={"pk": fetch.pk})
+
+ response = logged_in_api_client.get(url)
+
+ expected = api_serializers.FetchSerializer(fetch).data
+
+ assert response.status_code == 200
+ assert response.data == expected
diff --git a/api/tests/federation/test_decorators.py b/api/tests/federation/test_decorators.py
new file mode 100644
index 0000000000000000000000000000000000000000..fa50f5674a699378cb879e2764759e063bd29823
--- /dev/null
+++ b/api/tests/federation/test_decorators.py
@@ -0,0 +1,83 @@
+from rest_framework import viewsets
+
+from funkwhale_api.music import models as music_models
+
+from funkwhale_api.federation import api_serializers
+from funkwhale_api.federation import decorators
+from funkwhale_api.federation import models
+from funkwhale_api.federation import tasks
+
+
+class V(viewsets.ModelViewSet):
+ queryset = music_models.Track.objects.all()
+ fetches = decorators.fetches_route()
+ permission_classes = []
+
+
+def test_fetches_route_create(factories, api_request, mocker):
+ on_commit = mocker.patch("funkwhale_api.common.utils.on_commit")
+ user = factories["users.User"]()
+ actor = user.create_actor()
+ track = factories["music.Track"]()
+ view = V.as_view({"post": "fetches"})
+
+ request = api_request.post("/", format="json")
+ setattr(request, "user", user)
+ setattr(request, "session", {})
+ response = view(request, pk=track.pk)
+
+ assert response.status_code == 201
+
+ fetch = models.Fetch.objects.get_for_object(track).latest("id")
+ on_commit.assert_called_once_with(tasks.fetch.delay, fetch_id=fetch.pk)
+
+ assert fetch.url == track.fid
+ assert fetch.object == track
+ assert fetch.status == "pending"
+ assert fetch.actor == actor
+
+ expected = api_serializers.FetchSerializer(fetch).data
+ assert response.data == expected
+
+
+def test_fetches_route_create_local(factories, api_request, mocker, settings):
+ user = factories["users.User"]()
+ user.create_actor()
+ track = factories["music.Track"](
+ fid="https://{}/test".format(settings.FEDERATION_HOSTNAME)
+ )
+ view = V.as_view({"post": "fetches"})
+
+ request = api_request.post("/", format="json")
+ setattr(request, "user", user)
+ setattr(request, "session", {})
+ response = view(request, pk=track.pk)
+
+ assert response.status_code == 400
+
+
+def test_fetches_route_list(factories, api_request, mocker):
+ user = factories["users.User"]()
+ user.create_actor()
+ track = factories["music.Track"]()
+ fetches = [
+ factories["federation.Fetch"](object=track),
+ factories["federation.Fetch"](object=track),
+ ]
+ view = V.as_view({"get": "fetches"})
+
+ request = api_request.get("/", format="json")
+ setattr(request, "user", user)
+ setattr(request, "session", {})
+ expected = {
+ "next": None,
+ "previous": None,
+ "count": 2,
+ "results": api_serializers.FetchSerializer(reversed(fetches), many=True).data,
+ }
+
+ request = api_request.get("/")
+ response = view(request, pk=track.pk)
+
+ assert response.status_code == 200
+ assert response.data == expected
diff --git a/api/tests/federation/test_models.py b/api/tests/federation/test_models.py
index 68a4571421c6705eaf42c779728bd9b1b423dcb3..d6f862bb356300e9d93fb02c0300f266eef0a6bf 100644
--- a/api/tests/federation/test_models.py
+++ b/api/tests/federation/test_models.py
@@ -164,3 +164,12 @@ def test_actor_can_manage_domain_service_actor(mocker, factories):
obj = mocker.Mock(fid="https://{}/hello".format(actor.domain_id))
assert actor.can_manage(obj) is True
+
+
+def test_can_create_fetch_for_object(factories):
+ track = factories["music.Track"](fid="http://test.domain")
+ fetch = factories["federation.Fetch"](object=track)
+ assert fetch.url == "http://test.domain"
+ assert fetch.status == "pending"
+ assert fetch.detail == {}
+ assert fetch.object == track
diff --git a/api/tests/federation/test_tasks.py b/api/tests/federation/test_tasks.py
index 4428484b985b14c44a867e444d19d4940892f04a..5cfd228e5d2954fa097f1fdc49dfa08dd311cb4d 100644
--- a/api/tests/federation/test_tasks.py
+++ b/api/tests/federation/test_tasks.py
@@ -5,6 +5,7 @@ import pytest
from django.utils import timezone
+from funkwhale_api.federation import jsonld
from funkwhale_api.federation import models
from funkwhale_api.federation import serializers
from funkwhale_api.federation import tasks
@@ -332,3 +333,60 @@ def test_rotate_actor_key(factories, settings, mocker):
assert actor.public_key == "public"
assert actor.private_key == "private"
+
+
+def test_fetch_skipped(factories, r_mock):
+ url = "https://fetch.object"
+ fetch = factories["federation.Fetch"](url=url)
+ payload = {"@context": jsonld.get_default_context(), "type": "Unhandled"}
+ r_mock.get(url, json=payload)
+
+ tasks.fetch(fetch_id=fetch.pk)
+
+ fetch.refresh_from_db()
+
+ assert fetch.status == "skipped"
+ assert fetch.detail["reason"] == "unhandled_type"
+
+
+@pytest.mark.parametrize(
+ "r_mock_args, expected_error_code",
+ [
+ ({"json": {"type": "Unhandled"}}, "invalid_jsonld"),
+ ({"json": {"@context": jsonld.get_default_context()}}, "invalid_jsonld"),
+ ({"text": "invalidjson"}, "invalid_json"),
+ ({"status_code": 404}, "http"),
+ ({"status_code": 500}, "http"),
+ ],
+)
+def test_fetch_errored(factories, r_mock_args, expected_error_code, r_mock):
+ url = "https://fetch.object"
+ fetch = factories["federation.Fetch"](url=url)
+ r_mock.get(url, **r_mock_args)
+
+ tasks.fetch(fetch_id=fetch.pk)
+
+ fetch.refresh_from_db()
+
+ assert fetch.status == "errored"
+ assert fetch.detail["error_code"] == expected_error_code
+
+
+def test_fetch_success(factories, r_mock, mocker):
+ artist = factories["music.Artist"]()
+ fetch = factories["federation.Fetch"](url=artist.fid)
+ payload = serializers.ArtistSerializer(artist).data
+ init = mocker.spy(serializers.ArtistSerializer, "__init__")
+ save = mocker.spy(serializers.ArtistSerializer, "save")
+
+ r_mock.get(artist.fid, json=payload)
+
+ tasks.fetch(fetch_id=fetch.pk)
+
+ fetch.refresh_from_db()
+ payload["@context"].append("https://funkwhale.audio/ns")
+ assert fetch.status == "finished"
+ assert init.call_count == 1
+ assert init.call_args[0][1] == artist
+ assert init.call_args[1]["data"] == payload
+ assert save.call_count == 1
diff --git a/front/src/components/federation/FetchButton.vue b/front/src/components/federation/FetchButton.vue
new file mode 100644
index 0000000000000000000000000000000000000000..5c7f4b66e2638c7777d4d9783739bd6b11b7504c
--- /dev/null
+++ b/front/src/components/federation/FetchButton.vue
@@ -0,0 +1,157 @@
+<template>
+ <div @click="createFetch" role="button">
+ <div>
+ <slot></slot>
+ </div>
+ <modal class="small" :show.sync="showModal">
+ <h3 class="header">
+ <translate translate-context="Popup/*/Title">Refreshing object from remote…</translate>
+ </h3>
+ <div class="scrolling content">
+ <template v-if="fetch && fetch.status != 'pending'">
+ <div v-if="fetch.status === 'skipped'" class="ui message">
+ <div class="header"><translate translate-context="Popup/*/Message.Title">Refresh was skipped</translate></div>
+ <p><translate translate-context="Popup/*/Message.Content">The remote server answered, but returned data was unsupported by Funkwhale.</translate></p>
+ </div>
+ <div v-else-if="fetch.status === 'finished'" class="ui success message">
+ <div class="header"><translate translate-context="Popup/*/Message.Title">Refresh successful</translate></div>
+ <p><translate translate-context="Popup/*/Message.Content">Data was refreshed successfully from remote server.</translate></p>
+ </div>
+ <div v-else-if="fetch.status === 'errored'" class="ui error message">
+ <div class="header"><translate translate-context="Popup/*/Message.Title">Refresh error</translate></div>
+ <p><translate translate-context="Popup/*/Message.Content">An error occured while trying to refresh data:</translate></p>
+ <table class="ui very basic collapsing celled table">
+ <tbody>
+ <tr>
+ <td>
+ <translate translate-context="Popup/Import/Table.Label/Noun">Error type</translate>
+ </td>
+ <td>
+ {{ fetch.detail.error_code }}
+ </td>
+ </tr>
+ <tr>
+ <td>
+ <translate translate-context="Popup/Import/Table.Label/Noun">Error detail</translate>
+ </td>
+ <td>
+ <translate
+ v-if="fetch.detail.error_code === 'http' && fetch.detail.status_code"
+ :translate-params="{status: fetch.detail.status_code}"
+ translate-context="*/*/Error">The remote server answered with HTTP %{ status }</translate>
+ <translate
+ v-else-if="['http', 'request'].indexOf(fetch.detail.error_code) > -1"
+ translate-context="*/*/Error">An HTTP error occured while contacting the remote server</translate>
+ <translate
+ v-else-if="fetch.detail.error_code === 'timeout'"
+ translate-context="*/*/Error">The remote server didn't answered fast enough</translate>
+ <translate
+ v-else-if="fetch.detail.error_code === 'connection'"
+ translate-context="*/*/Error">Impossible to connect to the remote server</translate>
+ <translate
+ v-else-if="['invalid_json', 'invalid_jsonld', 'missing_jsonld_type'].indexOf(fetch.detail.error_code) > -1"
+ translate-context="*/*/Error">The return server returned invalid JSON or JSON-LD data</translate>
+ <translate v-else-if="fetch.detail.error_code === 'validation'" translate-context="*/*/Error">Data returned by the remote server had invalid or missing attributes</translate>
+ <translate v-else-if="fetch.detail.error_code === 'unhandled'" translate-context="*/*/Error">Unknowkn error</translate>
+ <translate v-else translate-context="*/*/Error">Unknowkn error</translate>
+ </td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+ </template>
+ <div v-else-if="isCreatingFetch" class="ui active inverted dimmer">
+ <div class="ui text loader">
+ <translate translate-context="Popup/*/Loading.Title">Requesting a fetch…</translate>
+ </div>
+ </div>
+ <div v-else-if="isWaitingFetch" class="ui active inverted dimmer">
+ <div class="ui text loader">
+ <translate translate-context="Popup/*/Loading.Title">Waiting for result…</translate>
+ </div>
+ </div>
+ <div v-if="errors.length > 0" class="ui negative message">
+ <div class="header"><translate translate-context="Content/*/Error message.Title">Error while saving settings</translate></div>
+ <ul class="list">
+ <li v-for="error in errors">{{ error }}</li>
+ </ul>
+ </div>
+ <div v-else-if="fetch && fetch.status === 'pending' && pollsCount >= maxPolls" class="ui warning message">
+ <div class="header"><translate translate-context="Popup/*/Message.Title">Refresh pending</translate></div>
+ <p><translate translate-context="Popup/*/Message.Content">Refresh request wasn't proceed in time by our server. It will be processed later.</translate></p>
+ </div>
+ </div>
+ <div class="actions">
+ <div role="button" class="ui cancel button">
+ <translate translate-context="*/*/Button.Label/Verb">Close</translate>
+ </div>
+ <div role="button" @click="showModal = false; $emit('refresh')" class="ui confirm green button" v-if="fetch && fetch.status === 'finished'">
+ <translate translate-context="*/*/Button.Label/Verb">Close and reload page</translate>
+ </div>
+ </div>
+ </modal>
+ </div>
+</template>
+
+<script>
+import axios from "axios"
+import Modal from '@/components/semantic/Modal'
+
+export default {
+ props: ['url'],
+ components: {
+ Modal
+ },
+ data () {
+ return {
+ fetch: null,
+ isCreatingFetch: false,
+ errors: [],
+ showModal: false,
+ isWaitingFetch: false,
+ maxPolls: 15,
+ pollsCount: 0,
+ }
+ },
+ methods: {
+ createFetch () {
+ let self = this
+ this.fetch = null
+ this.pollsCount = 0
+ this.errors = []
+ this.isCreatingFetch = true
+ this.isWaitingFetch = false
+ self.showModal = true
+ axios.post(this.url).then((response) => {
+ self.isCreatingFetch = false
+ self.fetch = response.data
+ self.pollFetch()
+ }, (error) => {
+ self.isCreatingFetch = false
+ self.errors = error.backendErrors
+ })
+ },
+ pollFetch () {
+ this.isWaitingFetch = true
+ this.pollsCount += 1
+ let url = `federation/fetches/${this.fetch.id}/`
+ let self = this
+ self.showModal = true
+ axios.get(url).then((response) => {
+ self.isCreatingFetch = false
+ self.fetch = response.data
+ if (self.fetch.status === 'pending' && self.pollsCount < self.maxPolls) {
+ setTimeout(() => {
+ self.pollFetch()
+ }, 1000)
+ } else {
+ self.isWaitingFetch = false
+ }
+ }, (error) => {
+ self.errors = error.backendErrors
+ self.isWaitingFetch = false
+ })
+ }
+ }
+}
+</script>
diff --git a/front/src/components/semantic/Modal.vue b/front/src/components/semantic/Modal.vue
index 75cc97a8888713e49b3e28d9e28810fbc0985fc6..076b3e466dac8b487924d06f0ae24d4d04a0b854 100644
--- a/front/src/components/semantic/Modal.vue
+++ b/front/src/components/semantic/Modal.vue
@@ -23,6 +23,7 @@ export default {
if (this.control) {
$(this.$el).modal('hide')
}
+ $(this.$el).remove()
},
methods: {
initModal () {
diff --git a/front/src/views/admin/library/AlbumDetail.vue b/front/src/views/admin/library/AlbumDetail.vue
index 3215da10113c8c3945e0ea132c111dea04c7ff99..b5d802d98591e32dbde7148271535286000470c1 100644
--- a/front/src/views/admin/library/AlbumDetail.vue
+++ b/front/src/views/admin/library/AlbumDetail.vue
@@ -46,6 +46,10 @@
<i class="external icon"></i>
<translate translate-context="Content/Moderation/Link/Verb">Open on MusicBrainz</translate>
</a>
+ <fetch-button @refresh="fetchData" v-if="!object.is_local" class="basic item" :url="`albums/${object.id}/fetches/`">
+ <i class="refresh icon"></i>
+ <translate translate-context="Content/Moderation/Button/Verb">Refresh from remote server</translate>
+ </fetch-button>
<a class="basic item" :href="object.url || object.fid" target="_blank" rel="noopener noreferrer">
<i class="external icon"></i>
<translate translate-context="Content/Moderation/Link/Verb">Open remote profile</translate>
@@ -264,10 +268,14 @@
<script>
import axios from "axios"
import logger from "@/logging"
+import FetchButton from "@/components/federation/FetchButton"
export default {
props: ["id"],
+ components: {
+ FetchButton
+ },
data() {
return {
isLoading: true,
diff --git a/front/src/views/admin/library/ArtistDetail.vue b/front/src/views/admin/library/ArtistDetail.vue
index bfbd414a171900fe11a709f4a5624aaa1e5abd00..d509f7394ad516492e17f4bd625b5e78866b5001 100644
--- a/front/src/views/admin/library/ArtistDetail.vue
+++ b/front/src/views/admin/library/ArtistDetail.vue
@@ -45,6 +45,10 @@
<i class="external icon"></i>
<translate translate-context="Content/Moderation/Link/Verb">Open on MusicBrainz</translate>
</a>
+ <fetch-button @refresh="fetchData" v-if="!object.is_local" class="basic item" :url="`artists/${object.id}/fetches/`">
+ <i class="refresh icon"></i>
+ <translate translate-context="Content/Moderation/Button/Verb">Refresh from remote server</translate>
+ </fetch-button>
<a class="basic item" :href="object.url || object.fid" target="_blank" rel="noopener noreferrer">
<i class="external icon"></i>
<translate translate-context="Content/Moderation/Link/Verb">Open remote profile</translate>
@@ -264,9 +268,13 @@
import axios from "axios"
import logger from "@/logging"
+import FetchButton from "@/components/federation/FetchButton"
export default {
props: ["id"],
+ components: {
+ FetchButton
+ },
data() {
return {
isLoading: true,
diff --git a/front/src/views/admin/library/TrackDetail.vue b/front/src/views/admin/library/TrackDetail.vue
index 6e0c9f65e29b73ee89552a6b773324759a934a7c..15e08f7a8f169537e9eb373446e66f251385d420 100644
--- a/front/src/views/admin/library/TrackDetail.vue
+++ b/front/src/views/admin/library/TrackDetail.vue
@@ -45,6 +45,10 @@
<i class="external icon"></i>
<translate translate-context="Content/Moderation/Link/Verb">Open on MusicBrainz</translate>
</a>
+ <fetch-button @refresh="fetchData" v-if="!object.is_local" class="basic item" :url="`tracks/${object.id}/fetches/`">
+ <i class="refresh icon"></i>
+ <translate translate-context="Content/Moderation/Button/Verb">Refresh from remote server</translate>
+ </fetch-button>
<a class="basic item" :href="object.url || object.fid" target="_blank" rel="noopener noreferrer">
<i class="external icon"></i>
<translate translate-context="Content/Moderation/Link/Verb">Open remote profile</translate>
@@ -306,10 +310,14 @@
<script>
import axios from "axios"
import logger from "@/logging"
+import FetchButton from "@/components/federation/FetchButton"
export default {
props: ["id"],
+ components: {
+ FetchButton
+ },
data() {
return {
isLoading: true,