Skip to content
GitLab
Commit 2bc71eec authored by Eliot Berriot's avatar Eliot Berriot
Browse files

See #170: add a description field on tracks, albums, tracks

parent 424b9f13
Hide whitespace changes
Inline Side-by-side
api/funkwhale_api/common/factories.py
View file @ 2bc71eec
......@@ -26,3 +26,12 @@ class AttachmentFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
class Meta:
model = "common.Attachment"
@registry.register
class CommonFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
text = factory.Faker("paragraph")
content_type = "text/plain"
class Meta:
model = "common.Content"
api/funkwhale_api/common/migrations/0006_content.py 0 → 100644
View file @ 2bc71eec
# Generated by Django 2.2.7 on 2020-01-13 10:14
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('common', '0005_auto_20191125_1421'),
]
operations = [
migrations.CreateModel(
name='Content',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('text', models.CharField(blank=True, max_length=5000, null=True)),
('content_type', models.CharField(max_length=100)),
],
),
]
api/funkwhale_api/common/models.py
View file @ 2bc71eec
......@@ -24,6 +24,14 @@ from . import utils
from . import validators
CONTENT_TEXT_MAX_LENGTH = 5000
CONTENT_TEXT_SUPPORTED_TYPES = [
"text/html",
"text/markdown",
"text/plain",
]
@Field.register_lookup
class NotEqual(Lookup):
lookup_name = "ne"
......@@ -273,6 +281,15 @@ class MutationAttachment(models.Model):
unique_together = ("attachment", "mutation")
class Content(models.Model):
"""
A text content that can be associated to other models, like a description, a summary, etc.
"""
text = models.CharField(max_length=CONTENT_TEXT_MAX_LENGTH, blank=True, null=True)
content_type = models.CharField(max_length=100)
@receiver(models.signals.post_save, sender=Attachment)
def warm_attachment_thumbnails(sender, instance, **kwargs):
if not instance.file or not settings.CREATE_IMAGE_THUMBNAILS:
......@@ -302,3 +319,18 @@ def trigger_mutation_post_init(sender, instance, created, **kwargs):
except AttributeError:
return
handler(instance)
CONTENT_FKS = {
"music.Track": ["description"],
"music.Album": ["description"],
"music.Artist": ["description"],
}
@receiver(models.signals.post_delete, sender=None)
def remove_attached_content(sender, instance, **kwargs):
fk_fields = CONTENT_FKS.get(instance._meta.label, [])
for field in fk_fields:
if getattr(instance, "{}_id".format(field)):
getattr(instance, field).delete()
api/funkwhale_api/common/mutations.py
View file @ 2bc71eec
......@@ -86,7 +86,6 @@ class MutationSerializer(serializers.Serializer):
class UpdateMutationSerializer(serializers.ModelSerializer, MutationSerializer):
serialized_relations = {}
previous_state_handlers = {}
def __init__(self, *args, **kwargs):
# we force partial mode, because update mutations are partial
......@@ -141,9 +140,12 @@ class UpdateMutationSerializer(serializers.ModelSerializer, MutationSerializer):
obj,
*list(validated_data.keys()),
serialized_relations=self.serialized_relations,
handlers=self.previous_state_handlers,
handlers=self.get_previous_state_handlers(),
)
def get_previous_state_handlers(self):
return {}
def get_update_previous_state(obj, *fields, serialized_relations={}, handlers={}):
if not fields:
......
api/funkwhale_api/common/serializers.py
View file @ 2bc71eec
......@@ -11,6 +11,7 @@ from django.utils.encoding import smart_text
from django.utils.translation import ugettext_lazy as _
from . import models
from . import utils
class RelatedField(serializers.RelatedField):
......@@ -308,3 +309,12 @@ class AttachmentSerializer(serializers.Serializer):
return models.Attachment.objects.create(
file=validated_data["file"], actor=validated_data["actor"]
)
class ContentSerializer(serializers.Serializer):
text = serializers.CharField(max_length=models.CONTENT_TEXT_MAX_LENGTH)
content_type = serializers.ChoiceField(choices=models.CONTENT_TEXT_SUPPORTED_TYPES,)
html = serializers.SerializerMethodField()
def get_html(self, o):
return utils.render_html(o.text, o.content_type)
api/funkwhale_api/common/utils.py
View file @ 2bc71eec
from django.utils.deconstruct import deconstructible
import bleach.sanitizer
import markdown
import os
import shutil
import uuid
......@@ -241,3 +243,65 @@ def join_queries_or(left, right):
return left | right
else:
return right
def render_markdown(text):
return markdown.markdown(text, extensions=["nl2br"])
HTMl_CLEANER = bleach.sanitizer.Cleaner(
strip=True,
tags=[
"p",
"a",
"abbr",
"acronym",
"b",
"blockquote",
"code",
"em",
"i",
"li",
"ol",
"strong",
"ul",
],
)
HTML_LINKER = bleach.linkifier.Linker()
def clean_html(html):
return HTMl_CLEANER.clean(html)
def render_html(text, content_type):
rendered = render_markdown(text)
if content_type == "text/html":
rendered = text
elif content_type == "text/markdown":
rendered = render_markdown(text)
else:
rendered = render_markdown(text)
rendered = HTML_LINKER.linkify(rendered)
return clean_html(rendered).strip().replace("\n", "")
@transaction.atomic
def attach_content(obj, field, content_data):
from . import models
existing = getattr(obj, "{}_id".format(field))
if existing:
getattr(obj, field).delete()
if not content_data:
return
content_obj = models.Content.objects.create(
text=content_data["text"][: models.CONTENT_TEXT_MAX_LENGTH],
content_type=content_data["content_type"],
)
setattr(obj, field, content_obj)
obj.save(update_fields=[field])
api/funkwhale_api/federation/serializers.py
View file @ 2bc71eec
......@@ -9,7 +9,7 @@ from django.db import transaction
from rest_framework import serializers
from funkwhale_api.common import utils as funkwhale_utils
from funkwhale_api.common import utils as common_utils
from funkwhale_api.common import models as common_models
from funkwhale_api.music import licenses
from funkwhale_api.music import models as music_models
......@@ -611,9 +611,9 @@ class PaginatedCollectionSerializer(jsonld.JsonLdSerializer):
def to_representation(self, conf):
paginator = Paginator(conf["items"], conf.get("page_size", 20))
first = funkwhale_utils.set_query_parameter(conf["id"], page=1)
first = common_utils.set_query_parameter(conf["id"], page=1)
current = first
last = funkwhale_utils.set_query_parameter(conf["id"], page=paginator.num_pages)
last = common_utils.set_query_parameter(conf["id"], page=paginator.num_pages)
d = {
"id": conf["id"],
# XXX Stable release: remove the obsolete actor field
......@@ -646,7 +646,7 @@ class LibrarySerializer(PaginatedCollectionSerializer):
)
class Meta:
jsonld_mapping = funkwhale_utils.concat_dicts(
jsonld_mapping = common_utils.concat_dicts(
PAGINATED_COLLECTION_JSONLD_MAPPING,
{
"name": jsonld.first_val(contexts.AS.name),
......@@ -740,11 +740,11 @@ class CollectionPageSerializer(jsonld.JsonLdSerializer):
def to_representation(self, conf):
page = conf["page"]
first = funkwhale_utils.set_query_parameter(conf["id"], page=1)
last = funkwhale_utils.set_query_parameter(
first = common_utils.set_query_parameter(conf["id"], page=1)
last = common_utils.set_query_parameter(
conf["id"], page=page.paginator.num_pages
)
id = funkwhale_utils.set_query_parameter(conf["id"], page=page.number)
id = common_utils.set_query_parameter(conf["id"], page=page.number)
d = {
"id": id,
"partOf": conf["id"],
......@@ -764,12 +764,12 @@ class CollectionPageSerializer(jsonld.JsonLdSerializer):
}
if page.has_previous():
d["prev"] = funkwhale_utils.set_query_parameter(
d["prev"] = common_utils.set_query_parameter(
conf["id"], page=page.previous_page_number()
)
if page.has_next():
d["next"] = funkwhale_utils.set_query_parameter(
d["next"] = common_utils.set_query_parameter(
conf["id"], page=page.next_page_number()
)
d.update(get_additional_fields(conf))
......@@ -784,6 +784,8 @@ MUSIC_ENTITY_JSONLD_MAPPING = {
"musicbrainzId": jsonld.first_val(contexts.FW.musicbrainzId),
"attributedTo": jsonld.first_id(contexts.AS.attributedTo),
"tags": jsonld.raw(contexts.AS.tag),
"mediaType": jsonld.first_val(contexts.AS.mediaType),
"content": jsonld.first_val(contexts.AS.content),
}
......@@ -805,6 +807,28 @@ def repr_tag(tag_name):
return {"type": "Hashtag", "name": "#{}".format(tag_name)}
def include_content(repr, content_obj):
if not content_obj:
return
repr["content"] = common_utils.render_html(
content_obj.text, content_obj.content_type
)
repr["mediaType"] = "text/html"
class TruncatedCharField(serializers.CharField):
def __init__(self, *args, **kwargs):
self.truncate_length = kwargs.pop("truncate_length")
super().__init__(*args, **kwargs)
def to_internal_value(self, v):
v = super().to_internal_value(v)
if v:
v = v[: self.truncate_length]
return v
class MusicEntitySerializer(jsonld.JsonLdSerializer):
id = serializers.URLField(max_length=500)
published = serializers.DateTimeField()
......@@ -815,13 +839,23 @@ class MusicEntitySerializer(jsonld.JsonLdSerializer):
tags = serializers.ListField(
child=TagSerializer(), min_length=0, required=False, allow_null=True
)
mediaType = serializers.ChoiceField(
choices=common_models.CONTENT_TEXT_SUPPORTED_TYPES,
default="text/html",
required=False,
)
content = TruncatedCharField(
truncate_length=common_models.CONTENT_TEXT_MAX_LENGTH,
required=False,
allow_null=True,
)
@transaction.atomic
def update(self, instance, validated_data):
attributed_to_fid = validated_data.get("attributedTo")
if attributed_to_fid:
validated_data["attributedTo"] = actors.get_actor(attributed_to_fid)
updated_fields = funkwhale_utils.get_updated_fields(
updated_fields = common_utils.get_updated_fields(
self.updateable_fields, validated_data, instance
)
updated_fields = self.validate_updated_data(instance, updated_fields)
......@@ -831,6 +865,9 @@ class MusicEntitySerializer(jsonld.JsonLdSerializer):
tags = [t["name"] for t in validated_data.get("tags", []) or []]
tags_models.set_tags(instance, *tags)
common_utils.attach_content(
instance, "description", validated_data.get("description")
)
return instance
def get_tags_repr(self, instance):
......@@ -842,6 +879,15 @@ class MusicEntitySerializer(jsonld.JsonLdSerializer):
def validate_updated_data(self, instance, validated_data):
return validated_data
def validate(self, data):
validated_data = super().validate(data)
if data.get("content"):
validated_data["description"] = {
"content_type": data["mediaType"],
"text": data["content"],
}
return validated_data
class ArtistSerializer(MusicEntitySerializer):
updateable_fields = [
......@@ -866,7 +912,7 @@ class ArtistSerializer(MusicEntitySerializer):
else None,
"tag": self.get_tags_repr(instance),
}
include_content(d, instance.description)
if self.context.get("include_ap_context", self.parent is None):
d["@context"] = jsonld.get_default_context()
return d
......@@ -888,7 +934,7 @@ class AlbumSerializer(MusicEntitySerializer):
class Meta:
model = music_models.Album
jsonld_mapping = funkwhale_utils.concat_dicts(
jsonld_mapping = common_utils.concat_dicts(
MUSIC_ENTITY_JSONLD_MAPPING,
{
"released": jsonld.first_val(contexts.FW.released),
......@@ -917,6 +963,7 @@ class AlbumSerializer(MusicEntitySerializer):
else None,
"tag": self.get_tags_repr(instance),
}
include_content(d, instance.description)
if instance.attachment_cover:
d["cover"] = {
"type": "Link",
......@@ -968,7 +1015,7 @@ class TrackSerializer(MusicEntitySerializer):
class Meta:
model = music_models.Track
jsonld_mapping = funkwhale_utils.concat_dicts(
jsonld_mapping = common_utils.concat_dicts(
MUSIC_ENTITY_JSONLD_MAPPING,
{
"album": jsonld.first_obj(contexts.FW.album),
......@@ -1006,7 +1053,7 @@ class TrackSerializer(MusicEntitySerializer):
else None,
"tag": self.get_tags_repr(instance),
}
include_content(d, instance.description)
if self.context.get("include_ap_context", self.parent is None):
d["@context"] = jsonld.get_default_context()
return d
......@@ -1017,23 +1064,21 @@ class TrackSerializer(MusicEntitySerializer):
references = {}
actors_to_fetch = set()
actors_to_fetch.add(
funkwhale_utils.recursive_getattr(
common_utils.recursive_getattr(
validated_data, "attributedTo", permissive=True
)
)
actors_to_fetch.add(
funkwhale_utils.recursive_getattr(
common_utils.recursive_getattr(
validated_data, "album.attributedTo", permissive=True
)
)
artists = (
funkwhale_utils.recursive_getattr(
validated_data, "artists", permissive=True
)
common_utils.recursive_getattr(validated_data, "artists", permissive=True)
or []
)
album_artists = (
funkwhale_utils.recursive_getattr(
common_utils.recursive_getattr(
validated_data, "album.artists", permissive=True
)
or []
......@@ -1244,6 +1289,7 @@ class ChannelUploadSerializer(serializers.Serializer):
},
],
}
include_content(data, upload.track.description)
tags = [item.tag.name for item in upload.get_all_tagged_items()]
if tags:
data["tag"] = [repr_tag(name) for name in tags]
......
api/funkwhale_api/federation/views.py
View file @ 2bc71eec
......@@ -225,11 +225,14 @@ class MusicLibraryViewSet(
"album__attributed_to",
"attributed_to",
"album__attachment_cover",
"description",
).prefetch_related(
"tagged_items__tag",
"album__tagged_items__tag",
"album__artist__tagged_items__tag",
"artist__tagged_items__tag",
"artist__description",
"album__description",
),
)
),
......@@ -278,6 +281,7 @@ class MusicUploadViewSet(
"library__actor",
"track__artist",
"track__album__artist",
"track__description",
"track__album__attachment_cover",
)
serializer_class = serializers.UploadSerializer
......@@ -299,7 +303,7 @@ class MusicArtistViewSet(
):
authentication_classes = [authentication.SignatureAuthentication]
renderer_classes = renderers.get_ap_renderers()
queryset = music_models.Artist.objects.local()
queryset = music_models.Artist.objects.local().select_related("description")
serializer_class = serializers.ArtistSerializer
lookup_field = "uuid"
......@@ -309,7 +313,9 @@ class MusicAlbumViewSet(
):
authentication_classes = [authentication.SignatureAuthentication]
renderer_classes = renderers.get_ap_renderers()
queryset = music_models.Album.objects.local().select_related("artist")
queryset = music_models.Album.objects.local().select_related(
"artist__description", "description"
)
serializer_class = serializers.AlbumSerializer
lookup_field = "uuid"
......@@ -320,7 +326,7 @@ class MusicTrackViewSet(
authentication_classes = [authentication.SignatureAuthentication]
renderer_classes = renderers.get_ap_renderers()
queryset = music_models.Track.objects.local().select_related(
"album__artist", "artist"
"album__artist", "album__description", "artist__description", "description"
)
serializer_class = serializers.TrackSerializer
lookup_field = "uuid"
api/funkwhale_api/manage/serializers.py
View file @ 2bc71eec
......@@ -383,7 +383,9 @@ class ManageNestedAlbumSerializer(ManageBaseAlbumSerializer):
return getattr(obj, "tracks_count", None)
class ManageArtistSerializer(ManageBaseArtistSerializer):
class ManageArtistSerializer(
music_serializers.OptionalDescriptionMixin, ManageBaseArtistSerializer
):
albums = ManageNestedAlbumSerializer(many=True)
tracks = ManageNestedTrackSerializer(many=True)
attributed_to = ManageBaseActorSerializer()
......@@ -407,7 +409,9 @@ class ManageNestedArtistSerializer(ManageBaseArtistSerializer):
pass
class ManageAlbumSerializer(ManageBaseAlbumSerializer):
class ManageAlbumSerializer(
music_serializers.OptionalDescriptionMixin, ManageBaseAlbumSerializer
):
tracks = ManageNestedTrackSerializer(many=True)
attributed_to = ManageBaseActorSerializer()
artist = ManageNestedArtistSerializer()
......@@ -435,7 +439,9 @@ class ManageTrackAlbumSerializer(ManageBaseAlbumSerializer):
fields = ManageBaseAlbumSerializer.Meta.fields + ["artist"]
class ManageTrackSerializer(ManageNestedTrackSerializer):
class ManageTrackSerializer(
music_serializers.OptionalDescriptionMixin, ManageNestedTrackSerializer
):
artist = ManageNestedArtistSerializer()
album = ManageTrackAlbumSerializer()
attributed_to = ManageBaseActorSerializer()
......
api/funkwhale_api/manage/views.py
View file @ 2bc71eec
......@@ -100,6 +100,11 @@ class ManageArtistViewSet(
result = serializer.save()
return response.Response(result, status=200)
def get_serializer_context(self):
context = super().get_serializer_context()
context["description"] = self.action in ["retrieve", "create", "update"]
return context
class ManageAlbumViewSet(
mixins.ListModelMixin,
......@@ -134,6 +139,11 @@ class ManageAlbumViewSet(
result = serializer.save()
return response.Response(result, status=200)
def get_serializer_context(self):
context = super().get_serializer_context()
context["description"] = self.action in ["retrieve", "create", "update"]
return context
uploads_subquery = (
music_models.Upload.objects.filter(track_id=OuterRef("pk"))
......@@ -186,6 +196,11 @@ class ManageTrackViewSet(
result = serializer.save()
return response.Response(result, status=200)
def get_serializer_context(self):
context = super().get_serializer_context()
context["description"] = self.action in ["retrieve", "create", "update"]
return context
uploads_subquery = (
music_models.Upload.objects.filter(library_id=OuterRef("pk"))
......
api/funkwhale_api/music/metadata.py
View file @ 2bc71eec
......@@ -168,6 +168,17 @@ def get_mp3_recording_id(f, k):
raise TagNotFound(k)
def get_mp3_comment(f, k):
keys_to_try = ["COMM", "COMM::eng"]
for key in keys_to_try:
try:
return get_id3_tag(f, key)
except TagNotFound:
pass
raise TagNotFound("COMM")
VALIDATION = {}
CONF = {
......@@ -192,6 +203,7 @@ CONF = {
"field": "metadata_block_picture",
"to_application": clean_ogg_pictures,
},
"comment": {"field": "comment"},
},
},
"OggVorbis": {
......@@ -215,6 +227,7 @@ CONF = {
"field": "metadata_block_picture",
"to_application": clean_ogg_pictures,
},
"comment": {"field": "comment"},
},
},
"OggTheora": {
......@@ -234,6 +247,7 @@ CONF = {
"license": {},
"copyright": {},
"genre": {},
"comment": {"field": "comment"},
},
},
"MP3": {
......@@ -255,6 +269,7 @@ CONF = {
"pictures": {},
"license": {"field": "WCOP"},
"copyright": {"field": "TCOP"},
"comment": {"field": "COMM", "getter": get_mp3_comment},
},
},
"MP4": {
......@@ -282,6 +297,7 @@ CONF = {
"pictures": {},
"license": {"field": "----:com.apple.iTunes:LICENSE"},
"copyright": {"field": "cprt"},
"comment": {"field": "©cmt"},
},
},
"FLAC": {
......@@ -304,6 +320,7 @@ CONF = {
"pictures": {},
"license": {},
"copyright": {},
"comment": {},
},
},
}
......@@ -322,6 +339,7 @@ ALL_FIELDS = [