Verified Commit 520fb9d0 authored by Eliot Berriot's avatar Eliot Berriot
Browse files

Started work on library scanning

parent 472cc7e2
......@@ -144,3 +144,24 @@ def get_library_data(library_url):
}
return serializer.validated_data
def get_library_page(library, page_url):
actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
auth = signing.get_auth(actor.private_key, actor.private_key_id)
response = session.get_session().get(
page_url,
auth=auth,
timeout=5,
verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
headers={
'Content-Type': 'application/activity+json'
}
)
serializer = serializers.CollectionPageSerializer(
data=response.json(),
context={
'library': library,
'item_serializer': serializers.AudioSerializer})
serializer.is_valid(raise_exception=True)
return serializer.validated_data
......@@ -2,6 +2,7 @@ import uuid
from django.conf import settings
from django.contrib.postgres.fields import JSONField
from django.core.serializers.json import DjangoJSONEncoder
from django.db import models
from django.utils import timezone
......@@ -160,4 +161,5 @@ class LibraryTrack(models.Model):
artist_name = models.CharField(max_length=500)
album_title = models.CharField(max_length=500)
title = models.CharField(max_length=500)
metadata = JSONField(default={}, max_length=10000)
metadata = JSONField(
default={}, max_length=10000, encoder=DjangoJSONEncoder)
......@@ -494,6 +494,8 @@ class PaginatedCollectionSerializer(serializers.Serializer):
totalItems = serializers.IntegerField(min_value=0)
actor = serializers.URLField()
id = serializers.URLField()
first = serializers.URLField()
last = serializers.URLField()
def to_representation(self, conf):
paginator = Paginator(
......@@ -524,10 +526,22 @@ class CollectionPageSerializer(serializers.Serializer):
items = serializers.ListField()
actor = serializers.URLField()
id = serializers.URLField()
prev = serializers.URLField(required=False)
first = serializers.URLField()
last = serializers.URLField()
next = serializers.URLField(required=False)
prev = serializers.URLField(required=False)
partOf = serializers.URLField()
def validate_items(self, v):
item_serializer = self.context.get('item_serializer')
if not item_serializer:
return v
raw_items = [item_serializer(data=i, context=self.context) for i in v]
for i in raw_items:
i.is_valid(raise_exception=True)
return raw_items
def to_representation(self, conf):
page = conf['page']
first = funkwhale_utils.set_query_parameter(
......
from funkwhale_api.taskapp import celery
from . import library as lb
from . import models
@celery.app.task(name='federation.scan_library')
@celery.require_instance(models.Library, 'library')
def scan_library(library):
if not library.federation_enabled:
return
data = lb.get_library_data(library.url)
scan_library_page.delay(
library_id=library.id, page_url=data['first'])
@celery.app.task(name='federation.scan_library_page')
@celery.require_instance(models.Library, 'library')
def scan_library_page(library, page_url):
if not library.federation_enabled:
return
data = lb.get_library_page(library, page_url)
lts = []
for item_serializer in data['items']:
lts.append(item_serializer.save())
......@@ -4,6 +4,7 @@ from django.core.exceptions import ValidationError
from django.contrib.postgres.fields import JSONField
from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
from django.core.serializers.json import DjangoJSONEncoder
from funkwhale_api.music.models import Track
......@@ -23,7 +24,7 @@ class Radio(models.Model):
creation_date = models.DateTimeField(default=timezone.now)
is_public = models.BooleanField(default=False)
version = models.PositiveIntegerField(default=0)
config = JSONField()
config = JSONField(encoder=DjangoJSONEncoder)
def get_candidates(self):
return filters.run(self.config)
......
......@@ -386,6 +386,8 @@ def test_paginated_collection_serializer_validation():
'id': 'https://test.federation/test',
'totalItems': 5,
'actor': 'http://test.actor',
'first': 'https://test.federation/test?page=1',
'last': 'https://test.federation/test?page=1',
'items': []
}
......@@ -407,6 +409,8 @@ def test_collection_page_serializer_validation():
'totalItems': 5,
'actor': 'https://test.actor',
'items': [],
'first': 'https://test.federation/test?page=1',
'last': 'https://test.federation/test?page=3',
'prev': base + '?page=1',
'next': base + '?page=3',
'partOf': base,
......@@ -426,6 +430,21 @@ def test_collection_page_serializer_validation():
assert serializer.validated_data['partOf'] == data['partOf']
def test_collection_page_serializer_can_validate_child():
base = 'https://test.federation/test'
data = {
'items': [{'in': 'valid'}],
}
serializer = serializers.CollectionPageSerializer(
data=data,
context={'item_serializer': serializers.AudioSerializer}
)
assert serializer.is_valid() is False
assert 'items' in serializer.errors
def test_collection_page_serializer(factories):
tfs = factories['music.TrackFile'].create_batch(size=5)
actor = factories['federation.Actor'](local=True)
......
from django.core.paginator import Paginator
from funkwhale_api.federation import serializers
from funkwhale_api.federation import tasks
def test_scan_library_does_nothing_if_federation_disabled(mocker, factories):
library = factories['federation.Library'](federation_enabled=False)
tasks.scan_library(library_id=library.pk)
assert library.tracks.count() == 0
def test_scan_library_page_does_nothing_if_federation_disabled(
mocker, factories):
library = factories['federation.Library'](federation_enabled=False)
tasks.scan_library_page(library_id=library.pk, page_url=None)
assert library.tracks.count() == 0
def test_scan_library_fetches_page_and_calls_scan_page(
mocker, factories, r_mock):
library = factories['federation.Library'](federation_enabled=True)
collection_conf = {
'actor': library.actor,
'id': library.url,
'page_size': 10,
'items': range(10),
}
collection = serializers.PaginatedCollectionSerializer(collection_conf)
scan_page = mocker.patch(
'funkwhale_api.federation.tasks.scan_library_page.delay')
r_mock.get(collection_conf['id'], json=collection.data)
tasks.scan_library(library_id=library.pk)
scan_page.assert_called_once_with(
library_id=library.id,
page_url=collection.data['first'],
)
def test_scan_page_fetches_page_and_creates_tracks(
mocker, factories, r_mock):
library = factories['federation.Library'](federation_enabled=True)
tfs = factories['music.TrackFile'].create_batch(size=5)
page_conf = {
'actor': library.actor,
'id': library.url,
'page': Paginator(tfs, 5).page(1),
'item_serializer': serializers.AudioSerializer,
}
page = serializers.CollectionPageSerializer(page_conf)
#scan_page = mocker.patch(
# 'funkwhale_api.federation.tasks.scan_library_page.delay')
r_mock.get(page.data['id'], json=page.data)
tasks.scan_library_page(library_id=library.pk, page_url=page.data['id'])
lts = list(library.tracks.all().order_by('-published_date'))
assert len(lts) == 5
......@@ -43,7 +43,7 @@ export default {
data () {
return {
isLoading: false,
libraryUsername: 'library@node2.funkwhale.test',
libraryUsername: '',
result: null,
errors: []
}
......
......@@ -77,6 +77,14 @@
</td>
<td></td>
</tr>
<tr>
<td>Last fetched</td>
<td>
<human-date v-if="object.fetched_date" :date="object.fetched_date"></human-date>
<template v-else>Never</template>
</td>
<td></td>
</tr>
</tbody>
</table>
</div>
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment