Started work on library scanning

api/funkwhale_api/federation/library.py

@@ -144,3 +144,24 @@ def get_library_data(library_url):
         }
 
     return serializer.validated_data
+
+
+def get_library_page(library, page_url):
+    actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
+    auth = signing.get_auth(actor.private_key, actor.private_key_id)
+    response = session.get_session().get(
+        page_url,
+        auth=auth,
+        timeout=5,
+        verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
+        headers={
+            'Content-Type': 'application/activity+json'
+        }
+    )
+    serializer = serializers.CollectionPageSerializer(
+        data=response.json(),
+        context={
+            'library': library,
+            'item_serializer': serializers.AudioSerializer})
+    serializer.is_valid(raise_exception=True)
+    return serializer.validated_data

api/funkwhale_api/federation/models.py

@@ -2,6 +2,7 @@ import uuid
 
 from django.conf import settings
 from django.contrib.postgres.fields import JSONField
+from django.core.serializers.json import DjangoJSONEncoder
 from django.db import models
 from django.utils import timezone
 
@@ -160,4 +161,5 @@ class LibraryTrack(models.Model):
     artist_name = models.CharField(max_length=500)
     album_title = models.CharField(max_length=500)
     title = models.CharField(max_length=500)
-    metadata = JSONField(default={}, max_length=10000)
+    metadata = JSONField(
+        default={}, max_length=10000, encoder=DjangoJSONEncoder)

api/funkwhale_api/federation/serializers.py

@@ -494,6 +494,8 @@ class PaginatedCollectionSerializer(serializers.Serializer):
     totalItems = serializers.IntegerField(min_value=0)
     actor = serializers.URLField()
     id = serializers.URLField()
+    first = serializers.URLField()
+    last = serializers.URLField()
 
     def to_representation(self, conf):
         paginator = Paginator(
@@ -524,10 +526,22 @@ class CollectionPageSerializer(serializers.Serializer):
     items = serializers.ListField()
     actor = serializers.URLField()
     id = serializers.URLField()
-    prev = serializers.URLField(required=False)
+    first = serializers.URLField()
+    last = serializers.URLField()
     next = serializers.URLField(required=False)
+    prev = serializers.URLField(required=False)
     partOf = serializers.URLField()
 
+    def validate_items(self, v):
+        item_serializer = self.context.get('item_serializer')
+        if not item_serializer:
+            return v
+        raw_items = [item_serializer(data=i, context=self.context) for i in v]
+        for i in raw_items:
+            i.is_valid(raise_exception=True)
+
+        return raw_items
+
     def to_representation(self, conf):
         page = conf['page']
         first = funkwhale_utils.set_query_parameter(

api/funkwhale_api/federation/tasks.py

+from funkwhale_api.taskapp import celery
+
+from . import library as lb
+from . import models
+
+
+@celery.app.task(name='federation.scan_library')
+@celery.require_instance(models.Library, 'library')
+def scan_library(library):
+    if not library.federation_enabled:
+        return
+
+    data = lb.get_library_data(library.url)
+    scan_library_page.delay(
+        library_id=library.id, page_url=data['first'])
+
+
+@celery.app.task(name='federation.scan_library_page')
+@celery.require_instance(models.Library, 'library')
+def scan_library_page(library, page_url):
+    if not library.federation_enabled:
+        return
+
+    data = lb.get_library_page(library, page_url)
+    lts = []
+    for item_serializer in data['items']:
+        lts.append(item_serializer.save())

api/funkwhale_api/radios/models.py

@@ -4,6 +4,7 @@ from django.core.exceptions import ValidationError
 from django.contrib.postgres.fields import JSONField
 from django.contrib.contenttypes.fields import GenericForeignKey
 from django.contrib.contenttypes.models import ContentType
+from django.core.serializers.json import DjangoJSONEncoder
 
 from funkwhale_api.music.models import Track
 
@@ -23,7 +24,7 @@ class Radio(models.Model):
     creation_date = models.DateTimeField(default=timezone.now)
     is_public = models.BooleanField(default=False)
     version = models.PositiveIntegerField(default=0)
-    config = JSONField()
+    config = JSONField(encoder=DjangoJSONEncoder)
 
     def get_candidates(self):
         return filters.run(self.config)

api/tests/federation/test_serializers.py

@@ -386,6 +386,8 @@ def test_paginated_collection_serializer_validation():
         'id': 'https://test.federation/test',
         'totalItems': 5,
         'actor': 'http://test.actor',
+        'first': 'https://test.federation/test?page=1',
+        'last': 'https://test.federation/test?page=1',
         'items': []
     }
 
@@ -407,6 +409,8 @@ def test_collection_page_serializer_validation():
         'totalItems': 5,
         'actor': 'https://test.actor',
         'items': [],
+        'first': 'https://test.federation/test?page=1',
+        'last': 'https://test.federation/test?page=3',
         'prev': base + '?page=1',
         'next': base + '?page=3',
         'partOf': base,
@@ -426,6 +430,21 @@ def test_collection_page_serializer_validation():
     assert serializer.validated_data['partOf'] == data['partOf']
 
 
+def test_collection_page_serializer_can_validate_child():
+    base = 'https://test.federation/test'
+    data = {
+        'items': [{'in': 'valid'}],
+    }
+
+    serializer = serializers.CollectionPageSerializer(
+        data=data,
+        context={'item_serializer': serializers.AudioSerializer}
+    )
+
+    assert serializer.is_valid() is False
+    assert 'items' in serializer.errors
+
+
 def test_collection_page_serializer(factories):
     tfs = factories['music.TrackFile'].create_batch(size=5)
     actor = factories['federation.Actor'](local=True)

api/tests/federation/test_tasks.py

+from django.core.paginator import Paginator
+
+from funkwhale_api.federation import serializers
+from funkwhale_api.federation import tasks
+
+
+def test_scan_library_does_nothing_if_federation_disabled(mocker, factories):
+    library = factories['federation.Library'](federation_enabled=False)
+    tasks.scan_library(library_id=library.pk)
+
+    assert library.tracks.count() == 0
+
+
+def test_scan_library_page_does_nothing_if_federation_disabled(
+        mocker, factories):
+    library = factories['federation.Library'](federation_enabled=False)
+    tasks.scan_library_page(library_id=library.pk, page_url=None)
+
+    assert library.tracks.count() == 0
+
+
+def test_scan_library_fetches_page_and_calls_scan_page(
+        mocker, factories, r_mock):
+    library = factories['federation.Library'](federation_enabled=True)
+    collection_conf = {
+        'actor': library.actor,
+        'id': library.url,
+        'page_size': 10,
+        'items': range(10),
+    }
+    collection = serializers.PaginatedCollectionSerializer(collection_conf)
+    scan_page = mocker.patch(
+        'funkwhale_api.federation.tasks.scan_library_page.delay')
+    r_mock.get(collection_conf['id'], json=collection.data)
+    tasks.scan_library(library_id=library.pk)
+
+    scan_page.assert_called_once_with(
+        library_id=library.id,
+        page_url=collection.data['first'],
+    )
+
+
+def test_scan_page_fetches_page_and_creates_tracks(
+        mocker, factories, r_mock):
+    library = factories['federation.Library'](federation_enabled=True)
+    tfs = factories['music.TrackFile'].create_batch(size=5)
+    page_conf = {
+        'actor': library.actor,
+        'id': library.url,
+        'page': Paginator(tfs, 5).page(1),
+        'item_serializer': serializers.AudioSerializer,
+    }
+    page = serializers.CollectionPageSerializer(page_conf)
+    #scan_page = mocker.patch(
+    #    'funkwhale_api.federation.tasks.scan_library_page.delay')
+    r_mock.get(page.data['id'], json=page.data)
+
+    tasks.scan_library_page(library_id=library.pk, page_url=page.data['id'])
+
+    lts = list(library.tracks.all().order_by('-published_date'))
+    assert len(lts) == 5

front/src/components/federation/LibraryForm.vue

@@ -43,7 +43,7 @@ export default {
   data () {
     return {
       isLoading: false,
-      libraryUsername: 'library@node2.funkwhale.test',
+      libraryUsername: '',
       result: null,
       errors: []
     }

front/src/views/federation/LibraryDetail.vue

@@ -77,6 +77,14 @@
                 </td>
                 <td></td>
               </tr>
+              <tr>
+                <td>Last fetched</td>
+                <td>
+                  <human-date v-if="object.fetched_date" :date="object.fetched_date"></human-date>
+                  <template v-else>Never</template>
+                </td>
+                <td></td>
+              </tr>
             </tbody>
           </table>
         </div>