diff --git a/api/funkwhale_api/federation/library.py b/api/funkwhale_api/federation/library.py
index 177f1475451cee06f7e0f0a091730651fa98c55c..d7c2f817c8ad7a3584a4051e0c9c75def5c35b52 100644
--- a/api/funkwhale_api/federation/library.py
+++ b/api/funkwhale_api/federation/library.py
@@ -144,3 +144,24 @@ def get_library_data(library_url):
}
return serializer.validated_data
+
+
+def get_library_page(library, page_url):
+ actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
+ auth = signing.get_auth(actor.private_key, actor.private_key_id)
+ response = session.get_session().get(
+ page_url,
+ auth=auth,
+ timeout=5,
+ verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
+ headers={
+ 'Content-Type': 'application/activity+json'
+ }
+ )
+ serializer = serializers.CollectionPageSerializer(
+ data=response.json(),
+ context={
+ 'library': library,
+ 'item_serializer': serializers.AudioSerializer})
+ serializer.is_valid(raise_exception=True)
+ return serializer.validated_data
diff --git a/api/funkwhale_api/federation/models.py b/api/funkwhale_api/federation/models.py
index 7dc9c46e4fa98970357e18ebdf77a61761db186d..76dbfd1ade0779965b09f7575107e9b4bebaf7f4 100644
--- a/api/funkwhale_api/federation/models.py
+++ b/api/funkwhale_api/federation/models.py
@@ -2,6 +2,7 @@ import uuid
from django.conf import settings
from django.contrib.postgres.fields import JSONField
+from django.core.serializers.json import DjangoJSONEncoder
from django.db import models
from django.utils import timezone
@@ -160,4 +161,5 @@ class LibraryTrack(models.Model):
artist_name = models.CharField(max_length=500)
album_title = models.CharField(max_length=500)
title = models.CharField(max_length=500)
- metadata = JSONField(default={}, max_length=10000)
+ metadata = JSONField(
+ default={}, max_length=10000, encoder=DjangoJSONEncoder)
diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py
index c717e679b0a4621333af4214f12c91970700f518..42054c7c4b349a834828f263523e66d625fba23a 100644
--- a/api/funkwhale_api/federation/serializers.py
+++ b/api/funkwhale_api/federation/serializers.py
@@ -494,6 +494,8 @@ class PaginatedCollectionSerializer(serializers.Serializer):
totalItems = serializers.IntegerField(min_value=0)
actor = serializers.URLField()
id = serializers.URLField()
+ first = serializers.URLField()
+ last = serializers.URLField()
def to_representation(self, conf):
paginator = Paginator(
@@ -524,10 +526,22 @@ class CollectionPageSerializer(serializers.Serializer):
items = serializers.ListField()
actor = serializers.URLField()
id = serializers.URLField()
- prev = serializers.URLField(required=False)
+ first = serializers.URLField()
+ last = serializers.URLField()
next = serializers.URLField(required=False)
+ prev = serializers.URLField(required=False)
partOf = serializers.URLField()
+ def validate_items(self, v):
+ item_serializer = self.context.get('item_serializer')
+ if not item_serializer:
+ return v
+ raw_items = [item_serializer(data=i, context=self.context) for i in v]
+ for i in raw_items:
+ i.is_valid(raise_exception=True)
+
+ return raw_items
+
def to_representation(self, conf):
page = conf['page']
first = funkwhale_utils.set_query_parameter(
diff --git a/api/funkwhale_api/federation/tasks.py b/api/funkwhale_api/federation/tasks.py
new file mode 100644
index 0000000000000000000000000000000000000000..fd03d3290cc41d9b023113a1b37b8359315b866b
--- /dev/null
+++ b/api/funkwhale_api/federation/tasks.py
@@ -0,0 +1,27 @@
+from funkwhale_api.taskapp import celery
+
+from . import library as lb
+from . import models
+
+
+@celery.app.task(name='federation.scan_library')
+@celery.require_instance(models.Library, 'library')
+def scan_library(library):
+ if not library.federation_enabled:
+ return
+
+ data = lb.get_library_data(library.url)
+ scan_library_page.delay(
+ library_id=library.id, page_url=data['first'])
+
+
+@celery.app.task(name='federation.scan_library_page')
+@celery.require_instance(models.Library, 'library')
+def scan_library_page(library, page_url):
+ if not library.federation_enabled:
+ return
+
+ data = lb.get_library_page(library, page_url)
+ lts = []
+ for item_serializer in data['items']:
+ lts.append(item_serializer.save())
diff --git a/api/funkwhale_api/radios/models.py b/api/funkwhale_api/radios/models.py
index d9c12534c03f779d497a1a9e7ae43e12cc3bb32f..0273b53871b4536a52b34870ef5ba2e1e93ae078 100644
--- a/api/funkwhale_api/radios/models.py
+++ b/api/funkwhale_api/radios/models.py
@@ -4,6 +4,7 @@ from django.core.exceptions import ValidationError
from django.contrib.postgres.fields import JSONField
from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
+from django.core.serializers.json import DjangoJSONEncoder
from funkwhale_api.music.models import Track
@@ -23,7 +24,7 @@ class Radio(models.Model):
creation_date = models.DateTimeField(default=timezone.now)
is_public = models.BooleanField(default=False)
version = models.PositiveIntegerField(default=0)
- config = JSONField()
+ config = JSONField(encoder=DjangoJSONEncoder)
def get_candidates(self):
return filters.run(self.config)
diff --git a/api/tests/federation/test_serializers.py b/api/tests/federation/test_serializers.py
index 8086a00592c710b2b8ee59e00112cb89bd563c9a..6d33a529de7d197d0b88557cc38ee8c17ff46ce4 100644
--- a/api/tests/federation/test_serializers.py
+++ b/api/tests/federation/test_serializers.py
@@ -386,6 +386,8 @@ def test_paginated_collection_serializer_validation():
'id': 'https://test.federation/test',
'totalItems': 5,
'actor': 'http://test.actor',
+ 'first': 'https://test.federation/test?page=1',
+ 'last': 'https://test.federation/test?page=1',
'items': []
}
@@ -407,6 +409,8 @@ def test_collection_page_serializer_validation():
'totalItems': 5,
'actor': 'https://test.actor',
'items': [],
+ 'first': 'https://test.federation/test?page=1',
+ 'last': 'https://test.federation/test?page=3',
'prev': base + '?page=1',
'next': base + '?page=3',
'partOf': base,
@@ -426,6 +430,21 @@ def test_collection_page_serializer_validation():
assert serializer.validated_data['partOf'] == data['partOf']
+def test_collection_page_serializer_can_validate_child():
+ base = 'https://test.federation/test'
+ data = {
+ 'items': [{'in': 'valid'}],
+ }
+
+ serializer = serializers.CollectionPageSerializer(
+ data=data,
+ context={'item_serializer': serializers.AudioSerializer}
+ )
+
+ assert serializer.is_valid() is False
+ assert 'items' in serializer.errors
+
+
def test_collection_page_serializer(factories):
tfs = factories['music.TrackFile'].create_batch(size=5)
actor = factories['federation.Actor'](local=True)
diff --git a/api/tests/federation/test_tasks.py b/api/tests/federation/test_tasks.py
new file mode 100644
index 0000000000000000000000000000000000000000..85684c1a37a56e96f8c6ccacecdeb1383913077c
--- /dev/null
+++ b/api/tests/federation/test_tasks.py
@@ -0,0 +1,61 @@
+from django.core.paginator import Paginator
+
+from funkwhale_api.federation import serializers
+from funkwhale_api.federation import tasks
+
+
+def test_scan_library_does_nothing_if_federation_disabled(mocker, factories):
+ library = factories['federation.Library'](federation_enabled=False)
+ tasks.scan_library(library_id=library.pk)
+
+ assert library.tracks.count() == 0
+
+
+def test_scan_library_page_does_nothing_if_federation_disabled(
+ mocker, factories):
+ library = factories['federation.Library'](federation_enabled=False)
+ tasks.scan_library_page(library_id=library.pk, page_url=None)
+
+ assert library.tracks.count() == 0
+
+
+def test_scan_library_fetches_page_and_calls_scan_page(
+ mocker, factories, r_mock):
+ library = factories['federation.Library'](federation_enabled=True)
+ collection_conf = {
+ 'actor': library.actor,
+ 'id': library.url,
+ 'page_size': 10,
+ 'items': range(10),
+ }
+ collection = serializers.PaginatedCollectionSerializer(collection_conf)
+ scan_page = mocker.patch(
+ 'funkwhale_api.federation.tasks.scan_library_page.delay')
+ r_mock.get(collection_conf['id'], json=collection.data)
+ tasks.scan_library(library_id=library.pk)
+
+ scan_page.assert_called_once_with(
+ library_id=library.id,
+ page_url=collection.data['first'],
+ )
+
+
+def test_scan_page_fetches_page_and_creates_tracks(
+ mocker, factories, r_mock):
+ library = factories['federation.Library'](federation_enabled=True)
+ tfs = factories['music.TrackFile'].create_batch(size=5)
+ page_conf = {
+ 'actor': library.actor,
+ 'id': library.url,
+ 'page': Paginator(tfs, 5).page(1),
+ 'item_serializer': serializers.AudioSerializer,
+ }
+ page = serializers.CollectionPageSerializer(page_conf)
+ #scan_page = mocker.patch(
+ # 'funkwhale_api.federation.tasks.scan_library_page.delay')
+ r_mock.get(page.data['id'], json=page.data)
+
+ tasks.scan_library_page(library_id=library.pk, page_url=page.data['id'])
+
+ lts = list(library.tracks.all().order_by('-published_date'))
+ assert len(lts) == 5
diff --git a/front/src/components/federation/LibraryForm.vue b/front/src/components/federation/LibraryForm.vue
index 5cf6dabb2f4dd8323af9972cfd21bdeebdac0175..5da46dc170bdfa05951e127ab67679a5735156e8 100644
--- a/front/src/components/federation/LibraryForm.vue
+++ b/front/src/components/federation/LibraryForm.vue
@@ -43,7 +43,7 @@ export default {
data () {
return {
isLoading: false,
- libraryUsername: 'library@node2.funkwhale.test',
+ libraryUsername: '',
result: null,
errors: []
}
diff --git a/front/src/views/federation/LibraryDetail.vue b/front/src/views/federation/LibraryDetail.vue
index d2430bda525082cc0c35fe4ca05483dd0ae82e9f..d33fcc212829e411d7d51e9650437cc3ca4ddade 100644
--- a/front/src/views/federation/LibraryDetail.vue
+++ b/front/src/views/federation/LibraryDetail.vue
@@ -77,6 +77,14 @@
</td>
<td></td>
</tr>
+ <tr>
+ <td>Last fetched</td>
+ <td>
+ <human-date v-if="object.fetched_date" :date="object.fetched_date"></human-date>
+ <template v-else>Never</template>
+ </td>
+ <td></td>
+ </tr>
</tbody>
</table>
</div>