Verified Commit 520fb9d0 authored by Eliot Berriot's avatar Eliot Berriot
Browse files

Started work on library scanning

parent 472cc7e2
...@@ -144,3 +144,24 @@ def get_library_data(library_url): ...@@ -144,3 +144,24 @@ def get_library_data(library_url):
} }
return serializer.validated_data return serializer.validated_data
def get_library_page(library, page_url):
actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
auth = signing.get_auth(actor.private_key, actor.private_key_id)
response = session.get_session().get(
page_url,
auth=auth,
timeout=5,
verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
headers={
'Content-Type': 'application/activity+json'
}
)
serializer = serializers.CollectionPageSerializer(
data=response.json(),
context={
'library': library,
'item_serializer': serializers.AudioSerializer})
serializer.is_valid(raise_exception=True)
return serializer.validated_data
...@@ -2,6 +2,7 @@ import uuid ...@@ -2,6 +2,7 @@ import uuid
from django.conf import settings from django.conf import settings
from django.contrib.postgres.fields import JSONField from django.contrib.postgres.fields import JSONField
from django.core.serializers.json import DjangoJSONEncoder
from django.db import models from django.db import models
from django.utils import timezone from django.utils import timezone
...@@ -160,4 +161,5 @@ class LibraryTrack(models.Model): ...@@ -160,4 +161,5 @@ class LibraryTrack(models.Model):
artist_name = models.CharField(max_length=500) artist_name = models.CharField(max_length=500)
album_title = models.CharField(max_length=500) album_title = models.CharField(max_length=500)
title = models.CharField(max_length=500) title = models.CharField(max_length=500)
metadata = JSONField(default={}, max_length=10000) metadata = JSONField(
default={}, max_length=10000, encoder=DjangoJSONEncoder)
...@@ -494,6 +494,8 @@ class PaginatedCollectionSerializer(serializers.Serializer): ...@@ -494,6 +494,8 @@ class PaginatedCollectionSerializer(serializers.Serializer):
totalItems = serializers.IntegerField(min_value=0) totalItems = serializers.IntegerField(min_value=0)
actor = serializers.URLField() actor = serializers.URLField()
id = serializers.URLField() id = serializers.URLField()
first = serializers.URLField()
last = serializers.URLField()
def to_representation(self, conf): def to_representation(self, conf):
paginator = Paginator( paginator = Paginator(
...@@ -524,10 +526,22 @@ class CollectionPageSerializer(serializers.Serializer): ...@@ -524,10 +526,22 @@ class CollectionPageSerializer(serializers.Serializer):
items = serializers.ListField() items = serializers.ListField()
actor = serializers.URLField() actor = serializers.URLField()
id = serializers.URLField() id = serializers.URLField()
prev = serializers.URLField(required=False) first = serializers.URLField()
last = serializers.URLField()
next = serializers.URLField(required=False) next = serializers.URLField(required=False)
prev = serializers.URLField(required=False)
partOf = serializers.URLField() partOf = serializers.URLField()
def validate_items(self, v):
item_serializer = self.context.get('item_serializer')
if not item_serializer:
return v
raw_items = [item_serializer(data=i, context=self.context) for i in v]
for i in raw_items:
i.is_valid(raise_exception=True)
return raw_items
def to_representation(self, conf): def to_representation(self, conf):
page = conf['page'] page = conf['page']
first = funkwhale_utils.set_query_parameter( first = funkwhale_utils.set_query_parameter(
......
from funkwhale_api.taskapp import celery
from . import library as lb
from . import models
@celery.app.task(name='federation.scan_library')
@celery.require_instance(models.Library, 'library')
def scan_library(library):
if not library.federation_enabled:
return
data = lb.get_library_data(library.url)
scan_library_page.delay(
library_id=library.id, page_url=data['first'])
@celery.app.task(name='federation.scan_library_page')
@celery.require_instance(models.Library, 'library')
def scan_library_page(library, page_url):
if not library.federation_enabled:
return
data = lb.get_library_page(library, page_url)
lts = []
for item_serializer in data['items']:
lts.append(item_serializer.save())
...@@ -4,6 +4,7 @@ from django.core.exceptions import ValidationError ...@@ -4,6 +4,7 @@ from django.core.exceptions import ValidationError
from django.contrib.postgres.fields import JSONField from django.contrib.postgres.fields import JSONField
from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType from django.contrib.contenttypes.models import ContentType
from django.core.serializers.json import DjangoJSONEncoder
from funkwhale_api.music.models import Track from funkwhale_api.music.models import Track
...@@ -23,7 +24,7 @@ class Radio(models.Model): ...@@ -23,7 +24,7 @@ class Radio(models.Model):
creation_date = models.DateTimeField(default=timezone.now) creation_date = models.DateTimeField(default=timezone.now)
is_public = models.BooleanField(default=False) is_public = models.BooleanField(default=False)
version = models.PositiveIntegerField(default=0) version = models.PositiveIntegerField(default=0)
config = JSONField() config = JSONField(encoder=DjangoJSONEncoder)
def get_candidates(self): def get_candidates(self):
return filters.run(self.config) return filters.run(self.config)
......
...@@ -386,6 +386,8 @@ def test_paginated_collection_serializer_validation(): ...@@ -386,6 +386,8 @@ def test_paginated_collection_serializer_validation():
'id': 'https://test.federation/test', 'id': 'https://test.federation/test',
'totalItems': 5, 'totalItems': 5,
'actor': 'http://test.actor', 'actor': 'http://test.actor',
'first': 'https://test.federation/test?page=1',
'last': 'https://test.federation/test?page=1',
'items': [] 'items': []
} }
...@@ -407,6 +409,8 @@ def test_collection_page_serializer_validation(): ...@@ -407,6 +409,8 @@ def test_collection_page_serializer_validation():
'totalItems': 5, 'totalItems': 5,
'actor': 'https://test.actor', 'actor': 'https://test.actor',
'items': [], 'items': [],
'first': 'https://test.federation/test?page=1',
'last': 'https://test.federation/test?page=3',
'prev': base + '?page=1', 'prev': base + '?page=1',
'next': base + '?page=3', 'next': base + '?page=3',
'partOf': base, 'partOf': base,
...@@ -426,6 +430,21 @@ def test_collection_page_serializer_validation(): ...@@ -426,6 +430,21 @@ def test_collection_page_serializer_validation():
assert serializer.validated_data['partOf'] == data['partOf'] assert serializer.validated_data['partOf'] == data['partOf']
def test_collection_page_serializer_can_validate_child():
base = 'https://test.federation/test'
data = {
'items': [{'in': 'valid'}],
}
serializer = serializers.CollectionPageSerializer(
data=data,
context={'item_serializer': serializers.AudioSerializer}
)
assert serializer.is_valid() is False
assert 'items' in serializer.errors
def test_collection_page_serializer(factories): def test_collection_page_serializer(factories):
tfs = factories['music.TrackFile'].create_batch(size=5) tfs = factories['music.TrackFile'].create_batch(size=5)
actor = factories['federation.Actor'](local=True) actor = factories['federation.Actor'](local=True)
......
from django.core.paginator import Paginator
from funkwhale_api.federation import serializers
from funkwhale_api.federation import tasks
def test_scan_library_does_nothing_if_federation_disabled(mocker, factories):
library = factories['federation.Library'](federation_enabled=False)
tasks.scan_library(library_id=library.pk)
assert library.tracks.count() == 0
def test_scan_library_page_does_nothing_if_federation_disabled(
mocker, factories):
library = factories['federation.Library'](federation_enabled=False)
tasks.scan_library_page(library_id=library.pk, page_url=None)
assert library.tracks.count() == 0
def test_scan_library_fetches_page_and_calls_scan_page(
mocker, factories, r_mock):
library = factories['federation.Library'](federation_enabled=True)
collection_conf = {
'actor': library.actor,
'id': library.url,
'page_size': 10,
'items': range(10),
}
collection = serializers.PaginatedCollectionSerializer(collection_conf)
scan_page = mocker.patch(
'funkwhale_api.federation.tasks.scan_library_page.delay')
r_mock.get(collection_conf['id'], json=collection.data)
tasks.scan_library(library_id=library.pk)
scan_page.assert_called_once_with(
library_id=library.id,
page_url=collection.data['first'],
)
def test_scan_page_fetches_page_and_creates_tracks(
mocker, factories, r_mock):
library = factories['federation.Library'](federation_enabled=True)
tfs = factories['music.TrackFile'].create_batch(size=5)
page_conf = {
'actor': library.actor,
'id': library.url,
'page': Paginator(tfs, 5).page(1),
'item_serializer': serializers.AudioSerializer,
}
page = serializers.CollectionPageSerializer(page_conf)
#scan_page = mocker.patch(
# 'funkwhale_api.federation.tasks.scan_library_page.delay')
r_mock.get(page.data['id'], json=page.data)
tasks.scan_library_page(library_id=library.pk, page_url=page.data['id'])
lts = list(library.tracks.all().order_by('-published_date'))
assert len(lts) == 5
...@@ -43,7 +43,7 @@ export default { ...@@ -43,7 +43,7 @@ export default {
data () { data () {
return { return {
isLoading: false, isLoading: false,
libraryUsername: 'library@node2.funkwhale.test', libraryUsername: '',
result: null, result: null,
errors: [] errors: []
} }
......
...@@ -77,6 +77,14 @@ ...@@ -77,6 +77,14 @@
</td> </td>
<td></td> <td></td>
</tr> </tr>
<tr>
<td>Last fetched</td>
<td>
<human-date v-if="object.fetched_date" :date="object.fetched_date"></human-date>
<template v-else>Never</template>
</td>
<td></td>
</tr>
</tbody> </tbody>
</table> </table>
</div> </div>
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment