Skip to content
GitLab
Verified Commit b29ca447 authored by Eliot Berriot's avatar Eliot Berriot
Browse files

Now store remote library tracks in a dedicated model, this is much simpler

parent f273faf9
Hide whitespace changes
Inline Side-by-side
api/funkwhale_api/federation/actors.py
View file @ b29ca447
......@@ -4,6 +4,7 @@ import uuid
import xml
from django.conf import settings
from django.db import transaction
from django.urls import reverse
from django.utils import timezone
......@@ -192,10 +193,8 @@ class LibraryActor(SystemActor):
def manually_approves_followers(self):
return settings.FEDERATION_MUSIC_NEEDS_APPROVAL
@transaction.atomic
def handle_create(self, ac, sender):
from funkwhale_api.music.serializers import (
AudioCollectionImportSerializer)
try:
remote_library = models.Library.objects.get(
actor=sender,
......@@ -212,18 +211,28 @@ class LibraryActor(SystemActor):
if ac['object']['totalItems'] <= 0:
return
items = ac['object']['items']
serializer = AudioCollectionImportSerializer(
data=ac['object'],
context={'library': remote_library})
if not serializer.is_valid():
logger.error(
'Cannot import audio collection: %s', serializer.errors)
try:
items = ac['object']['items']
except KeyError:
logger.warning('No items in collection!')
return
serializer.save()
item_serializers = [
serializers.AudioSerializer(
data=i, context={'library': remote_library})
for i in items
]
valid_serializers = []
for s in item_serializers:
if s.is_valid():
valid_serializers.append(s)
else:
logger.debug(
'Skipping invalid item %s, %s', s.initial_data, s.errors)
for s in valid_serializers:
s.save()
class TestActor(SystemActor):
......
api/funkwhale_api/federation/factories.py
View file @ b29ca447
......@@ -128,11 +128,73 @@ class LibraryFactory(factory.DjangoModelFactory):
url = factory.Faker('url')
federation_enabled = True
download_files = False
autoimport = False
class Meta:
model = models.Library
class ArtistMetadataFactory(factory.Factory):
name = factory.Faker('name')
class Meta:
model = dict
class Params:
musicbrainz = factory.Trait(
musicbrainz_id=factory.Faker('uuid4')
)
class ReleaseMetadataFactory(factory.Factory):
title = factory.Faker('sentence')
class Meta:
model = dict
class Params:
musicbrainz = factory.Trait(
musicbrainz_id=factory.Faker('uuid4')
)
class RecordingMetadataFactory(factory.Factory):
title = factory.Faker('sentence')
class Meta:
model = dict
class Params:
musicbrainz = factory.Trait(
musicbrainz_id=factory.Faker('uuid4')
)
@registry.register(name='federation.LibraryTrackMetadata')
class LibraryTrackMetadataFactory(factory.Factory):
artist = factory.SubFactory(ArtistMetadataFactory)
recording = factory.SubFactory(RecordingMetadataFactory)
release = factory.SubFactory(ReleaseMetadataFactory)
class Meta:
model = dict
@registry.register
class LibraryTrackFactory(factory.DjangoModelFactory):
library = factory.SubFactory(LibraryFactory)
url = factory.Faker('url')
title = factory.Faker('sentence')
artist_name = factory.Faker('sentence')
album_title = factory.Faker('sentence')
audio_url = factory.Faker('url')
audio_mimetype = 'audio/ogg'
metadata = factory.SubFactory(LibraryTrackMetadataFactory)
class Meta:
model = models.LibraryTrack
@registry.register(name='federation.Note')
class NoteFactory(factory.Factory):
type = 'Note'
......@@ -189,7 +251,7 @@ class AudioFactory(factory.Factory):
)
actor = factory.Faker('url')
url = factory.SubFactory(LinkFactory, audio=True)
metadata = factory.SubFactory(AudioMetadataFactory)
metadata = factory.SubFactory(LibraryTrackMetadataFactory)
class Meta:
model = dict
api/funkwhale_api/federation/migrations/0003_auto_20180406_1621.py → api/funkwhale_api/federation/migrations/0003_auto_20180407_0852.py
View file @ b29ca447
# Generated by Django 2.0.3 on 2018-04-06 16:21
# Generated by Django 2.0.3 on 2018-04-07 08:52
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
import django.db.models.deletion
import django.utils.timezone
......@@ -9,6 +10,7 @@ import uuid
class Migration(migrations.Migration):
dependencies = [
('music', '0022_importbatch_import_request'),
('federation', '0002_auto_20180403_1620'),
]
......@@ -47,10 +49,30 @@ class Migration(migrations.Migration):
('url', models.URLField()),
('federation_enabled', models.BooleanField()),
('download_files', models.BooleanField()),
('files_count', models.PositiveIntegerField(blank=True, null=True)),
('autoimport', models.BooleanField()),
('tracks_count', models.PositiveIntegerField(blank=True, null=True)),
('actor', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='library', to='federation.Actor')),
],
),
migrations.CreateModel(
name='LibraryTrack',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('url', models.URLField(unique=True)),
('audio_url', models.URLField()),
('audio_mimetype', models.CharField(max_length=200)),
('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
('modification_date', models.DateTimeField(auto_now=True)),
('fetched_date', models.DateTimeField(blank=True, null=True)),
('published_date', models.DateTimeField(blank=True, null=True)),
('artist_name', models.CharField(max_length=500)),
('album_title', models.CharField(max_length=500)),
('title', models.CharField(max_length=500)),
('metadata', django.contrib.postgres.fields.jsonb.JSONField(default={}, max_length=10000)),
('library', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tracks', to='federation.Library')),
('local_track_file', models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='library_track', to='music.TrackFile')),
],
),
migrations.AddField(
model_name='actor',
name='followers',
......
api/funkwhale_api/federation/models.py
View file @ b29ca447
import uuid
from django.conf import settings
from django.contrib.postgres.fields import JSONField
from django.db import models
from django.utils import timezone
......@@ -170,8 +171,35 @@ class Library(models.Model):
related_name='library')
uuid = models.UUIDField(default=uuid.uuid4)
url = models.URLField()
# use this flag to disable federation with a library
federation_enabled = models.BooleanField()
# should we mirror files locally or hotlink them?
download_files = models.BooleanField()
files_count = models.PositiveIntegerField(null=True, blank=True)
# should we automatically import new files from this library?
autoimport = models.BooleanField()
tracks_count = models.PositiveIntegerField(null=True, blank=True)
class LibraryTrack(models.Model):
url = models.URLField(unique=True)
audio_url = models.URLField()
audio_mimetype = models.CharField(max_length=200)
creation_date = models.DateTimeField(default=timezone.now)
modification_date = models.DateTimeField(
auto_now=True)
fetched_date = models.DateTimeField(null=True, blank=True)
published_date = models.DateTimeField(null=True, blank=True)
library = models.ForeignKey(
Library, related_name='tracks', on_delete=models.CASCADE)
local_track_file = models.OneToOneField(
'music.TrackFile',
related_name='library_track',
on_delete=models.CASCADE,
null=True,
blank=True,
)
artist_name = models.CharField(max_length=500)
album_title = models.CharField(max_length=500)
title = models.CharField(max_length=500)
metadata = JSONField(default={}, max_length=10000)
api/funkwhale_api/federation/serializers.py
View file @ b29ca447
......@@ -3,6 +3,7 @@ import urllib.parse
from django.urls import reverse
from django.conf import settings
from django.core.paginator import Paginator
from django.db import transaction
from rest_framework import serializers
from dynamic_preferences.registries import global_preferences_registry
......@@ -265,3 +266,149 @@ class CollectionPageSerializer(serializers.Serializer):
if self.context.get('include_ap_context', True):
d['@context'] = AP_CONTEXT
return d
class ArtistMetadataSerializer(serializers.Serializer):
musicbrainz_id = serializers.UUIDField(required=False)
name = serializers.CharField()
class ReleaseMetadataSerializer(serializers.Serializer):
musicbrainz_id = serializers.UUIDField(required=False)
title = serializers.CharField()
class RecordingMetadataSerializer(serializers.Serializer):
musicbrainz_id = serializers.UUIDField(required=False)
title = serializers.CharField()
class AudioMetadataSerializer(serializers.Serializer):
artist = ArtistMetadataSerializer()
release = ReleaseMetadataSerializer()
recording = RecordingMetadataSerializer()
class AudioSerializer(serializers.Serializer):
type = serializers.CharField()
id = serializers.URLField()
url = serializers.JSONField()
published = serializers.DateTimeField()
updated = serializers.DateTimeField(required=False)
metadata = AudioMetadataSerializer()
def validate_type(self, v):
if v != 'Audio':
raise serializers.ValidationError('Invalid type for audio')
return v
def validate_url(self, v):
try:
url = v['href']
except (KeyError, TypeError):
raise serializers.ValidationError('Missing href')
try:
media_type = v['mediaType']
except (KeyError, TypeError):
raise serializers.ValidationError('Missing mediaType')
if not media_type.startswith('audio/'):
raise serializers.ValidationError('Invalid mediaType')
return url
def validate_url(self, v):
try:
url = v['href']
except (KeyError, TypeError):
raise serializers.ValidationError('Missing href')
try:
media_type = v['mediaType']
except (KeyError, TypeError):
raise serializers.ValidationError('Missing mediaType')
if not media_type.startswith('audio/'):
raise serializers.ValidationError('Invalid mediaType')
return v
def create(self, validated_data):
defaults = {
'audio_mimetype': validated_data['url']['mediaType'],
'audio_url': validated_data['url']['href'],
'metadata': validated_data['metadata'],
'artist_name': validated_data['metadata']['artist']['name'],
'album_title': validated_data['metadata']['release']['title'],
'title': validated_data['metadata']['recording']['title'],
'published_date': validated_data['published'],
'modification_date': validated_data.get('updated'),
}
return models.LibraryTrack.objects.get_or_create(
library=self.context['library'],
url=validated_data['id'],
defaults=defaults
)[0]
def to_representation(self, instance):
track = instance.track
album = instance.track.album
artist = instance.track.artist
d = {
'type': 'Audio',
'id': instance.get_federation_url(),
'name': instance.track.full_name,
'published': instance.creation_date.isoformat(),
'updated': instance.modification_date.isoformat(),
'metadata': {
'artist': {
'musicbrainz_id': str(artist.mbid) if artist.mbid else None,
'name': artist.name,
},
'release': {
'musicbrainz_id': str(album.mbid) if album.mbid else None,
'title': album.title,
},
'recording': {
'musicbrainz_id': str(track.mbid) if track.mbid else None,
'title': track.title,
},
},
'url': {
'href': utils.full_url(instance.path),
'type': 'Link',
'mediaType': instance.mimetype
},
'attributedTo': [
self.context['actor'].url
]
}
if self.context.get('include_ap_context', True):
d['@context'] = AP_CONTEXT
return d
class CollectionSerializer(serializers.Serializer):
def to_representation(self, conf):
d = {
'id': conf['id'],
'actor': conf['actor'].url,
'totalItems': len(conf['items']),
'type': 'Collection',
'items': [
conf['item_serializer'](
i,
context={
'actor': conf['actor'],
'include_ap_context': False}
).data
for i in conf['items']
]
}
if self.context.get('include_ap_context', True):
d['@context'] = AP_CONTEXT
return d
api/funkwhale_api/federation/urls.py
View file @ b29ca447
......@@ -15,10 +15,10 @@ router.register(
'well-known')
music_router.register(
r'federation/files',
r'files',
views.MusicFilesViewSet,
'files',
)
urlpatterns = router.urls + [
url('music/', include((music_router.urls, 'music'), namespace='music'))
url('federation/music/', include((music_router.urls, 'music'), namespace='music'))
]
api/funkwhale_api/federation/views.py
View file @ b29ca447
......@@ -10,7 +10,6 @@ from rest_framework import response
from rest_framework.decorators import list_route, detail_route
from funkwhale_api.music.models import TrackFile
from funkwhale_api.music.serializers import AudioSerializer
from . import actors
from . import authentication
......@@ -119,13 +118,16 @@ class MusicFilesViewSet(FederationMixin, viewsets.GenericViewSet):
def list(self, request, *args, **kwargs):
page = request.GET.get('page')
library = actors.SYSTEM_ACTORS['library'].get_actor_instance()
qs = TrackFile.objects.order_by('-creation_date')
qs = TrackFile.objects.order_by('-creation_date').select_related(
'track__artist',
'track__album__artist'
)
if page is None:
conf = {
'id': utils.full_url(reverse('federation:music:files-list')),
'page_size': settings.FEDERATION_COLLECTION_PAGE_SIZE,
'items': qs,
'item_serializer': AudioSerializer,
'item_serializer': serializers.AudioSerializer,
'actor': library,
}
serializer = serializers.PaginatedCollectionSerializer(conf)
......@@ -140,15 +142,15 @@ class MusicFilesViewSet(FederationMixin, viewsets.GenericViewSet):
qs, settings.FEDERATION_COLLECTION_PAGE_SIZE)
try:
page = p.page(page_number)
conf = {
'id': utils.full_url(reverse('federation:music:files-list')),
'page': page,
'item_serializer': serializers.AudioSerializer,
'actor': library,
}
serializer = serializers.CollectionPageSerializer(conf)
data = serializer.data
except paginator.EmptyPage:
return response.Response(status=404)
conf = {
'id': utils.full_url(reverse('federation:music:files-list')),
'page': page,
'item_serializer': AudioSerializer,
'actor': library,
}
serializer = serializers.CollectionPageSerializer(conf)
data = serializer.data
return response.Response(data)
api/funkwhale_api/music/factories.py
View file @ b29ca447
......@@ -3,9 +3,7 @@ import os
from funkwhale_api.factories import registry, ManyToManyFromList
from funkwhale_api.federation.factories import (
AudioMetadataFactory,
ActorFactory,
LibraryFactory,
LibraryTrackFactory,
)
from funkwhale_api.users.factories import UserFactory
......@@ -69,8 +67,6 @@ class ImportBatchFactory(factory.django.DjangoModelFactory):
class Params:
federation = factory.Trait(
submitted_by=None,
source_library=factory.SubFactory(LibraryFactory),
source_library_url=factory.Faker('url'),
source='federation',
)
......@@ -86,9 +82,9 @@ class ImportJobFactory(factory.django.DjangoModelFactory):
class Params:
federation = factory.Trait(
mbid=None,
library_track=factory.SubFactory(LibraryTrackFactory),
batch=factory.SubFactory(ImportBatchFactory, federation=True),
source_library_url=factory.Faker('url'),
metadata=factory.SubFactory(AudioMetadataFactory),
)
......
api/funkwhale_api/music/migrations/0023_auto_20180406_1621.py → api/funkwhale_api/music/migrations/0023_auto_20180407_0852.py
View file @ b29ca447
# Generated by Django 2.0.3 on 2018-04-06 16:21
# Generated by Django 2.0.3 on 2018-04-07 08:52
from django.conf import settings
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
import django.db.models.deletion
import django.utils.timezone
......@@ -11,7 +10,7 @@ import uuid
class Migration(migrations.Migration):
dependencies = [
('federation', '0003_auto_20180406_1621'),
('federation', '0003_auto_20180407_0852'),
('music', '0022_importbatch_import_request'),
]
......@@ -26,16 +25,6 @@ class Migration(migrations.Migration):
name='uuid',
field=models.UUIDField(db_index=True, default=uuid.uuid4, unique=True),
),
migrations.AddField(
model_name='importbatch',
name='source_library',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='import_batches', to='federation.Library'),
),
migrations.AddField(
model_name='importbatch',
name='source_library_url',
field=models.URLField(blank=True, null=True),
),
migrations.AddField(
model_name='importbatch',
name='uuid',
......@@ -43,13 +32,8 @@ class Migration(migrations.Migration):
),
migrations.AddField(
model_name='importjob',
name='metadata',
field=django.contrib.postgres.fields.jsonb.JSONField(default={}),
),
migrations.AddField(
model_name='importjob',
name='source_library_url',
field=models.URLField(blank=True, null=True),
name='library_track',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='import_jobs', to='federation.LibraryTrack'),
),
migrations.AddField(
model_name='importjob',
......@@ -76,16 +60,6 @@ class Migration(migrations.Migration):
name='modification_date',
field=models.DateTimeField(auto_now=True),
),
migrations.AddField(
model_name='trackfile',
name='source_library',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='track_files', to='federation.Library'),
),
migrations.AddField(
model_name='trackfile',
name='source_library_url',
field=models.URLField(blank=True, null=True),
),
migrations.AddField(
model_name='trackfile',
name='uuid',
......
api/funkwhale_api/music/models.py
View file @ b29ca447
......@@ -9,7 +9,6 @@ import uuid
from django.conf import settings
from django.db import models
from django.contrib.postgres.fields import JSONField
from django.core.files.base import ContentFile
from django.core.files import File
from django.db.models.signals import post_save
......@@ -416,16 +415,6 @@ class TrackFile(models.Model):
source = models.URLField(null=True, blank=True)
creation_date = models.DateTimeField(default=timezone.now)
modification_date = models.DateTimeField(auto_now=True)
source_library = models.ForeignKey(
'federation.Library',
null=True,
blank=True,
on_delete=models.SET_NULL,
related_name='track_files')
# points to the URL of the original trackfile ActivityPub Object
source_library_url = models.URLField(null=True, blank=True)
duration = models.IntegerField(null=True, blank=True)
acoustid_track_id = models.UUIDField(null=True, blank=True)
mimetype = models.CharField(null=True, blank=True, max_length=200)
......@@ -503,14 +492,6 @@ class ImportBatch(models.Model):
blank=True,
on_delete=models.CASCADE)
source_library = models.ForeignKey(
'federation.Library',
null=True,
blank=True,
on_delete=models.SET_NULL,
related_name='import_batches')
source_library_url = models.URLField(null=True, blank=True)
class Meta: