Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found
Select Git revision

Target

Select target project
  • funkwhale/funkwhale
  • Luclu7/funkwhale
  • mbothorel/funkwhale
  • EorlBruder/funkwhale
  • tcit/funkwhale
  • JocelynDelalande/funkwhale
  • eneiluj/funkwhale
  • reg/funkwhale
  • ButterflyOfFire/funkwhale
  • m4sk1n/funkwhale
  • wxcafe/funkwhale
  • andybalaam/funkwhale
  • jcgruenhage/funkwhale
  • pblayo/funkwhale
  • joshuaboniface/funkwhale
  • n3ddy/funkwhale
  • gegeweb/funkwhale
  • tohojo/funkwhale
  • emillumine/funkwhale
  • Te-k/funkwhale
  • asaintgenis/funkwhale
  • anoadragon453/funkwhale
  • Sakada/funkwhale
  • ilianaw/funkwhale
  • l4p1n/funkwhale
  • pnizet/funkwhale
  • dante383/funkwhale
  • interfect/funkwhale
  • akhardya/funkwhale
  • svfusion/funkwhale
  • noplanman/funkwhale
  • nykopol/funkwhale
  • roipoussiere/funkwhale
  • Von/funkwhale
  • aurieh/funkwhale
  • icaria36/funkwhale
  • floreal/funkwhale
  • paulwalko/funkwhale
  • comradekingu/funkwhale
  • FurryJulie/funkwhale
  • Legolars99/funkwhale
  • Vierkantor/funkwhale
  • zachhats/funkwhale
  • heyjake/funkwhale
  • sn0w/funkwhale
  • jvoisin/funkwhale
  • gordon/funkwhale
  • Alexander/funkwhale
  • bignose/funkwhale
  • qasim.ali/funkwhale
  • fakegit/funkwhale
  • Kxze/funkwhale
  • stenstad/funkwhale
  • creak/funkwhale
  • Kaze/funkwhale
  • Tixie/funkwhale
  • IISergII/funkwhale
  • lfuelling/funkwhale
  • nhaddag/funkwhale
  • yoasif/funkwhale
  • ifischer/funkwhale
  • keslerm/funkwhale
  • flupe/funkwhale
  • petitminion/funkwhale
  • ariasuni/funkwhale
  • ollie/funkwhale
  • ngaumont/funkwhale
  • techknowlogick/funkwhale
  • Shleeble/funkwhale
  • theflyingfrog/funkwhale
  • jonatron/funkwhale
  • neobrain/funkwhale
  • eorn/funkwhale
  • KokaKiwi/funkwhale
  • u1-liquid/funkwhale
  • marzzzello/funkwhale
  • sirenwatcher/funkwhale
  • newer027/funkwhale
  • codl/funkwhale
  • Zwordi/funkwhale
  • gisforgabriel/funkwhale
  • iuriatan/funkwhale
  • simon/funkwhale
  • bheesham/funkwhale
  • zeoses/funkwhale
  • accraze/funkwhale
  • meliurwen/funkwhale
  • divadsn/funkwhale
  • Etua/funkwhale
  • sdrik/funkwhale
  • Soran/funkwhale
  • kuba-orlik/funkwhale
  • cristianvogel/funkwhale
  • Forceu/funkwhale
  • jeff/funkwhale
  • der_scheibenhacker/funkwhale
  • owlnical/funkwhale
  • jovuit/funkwhale
  • SilverFox15/funkwhale
  • phw/funkwhale
  • mayhem/funkwhale
  • sridhar/funkwhale
  • stromlin/funkwhale
  • rrrnld/funkwhale
  • nitaibezerra/funkwhale
  • jaller94/funkwhale
  • pcouy/funkwhale
  • eduxstad/funkwhale
  • codingHahn/funkwhale
  • captain/funkwhale
  • polyedre/funkwhale
  • leishenailong/funkwhale
  • ccritter/funkwhale
  • lnceballosz/funkwhale
  • fpiesche/funkwhale
  • Fanyx/funkwhale
  • markusblogde/funkwhale
  • Firobe/funkwhale
  • devilcius/funkwhale
  • freaktechnik/funkwhale
  • blopware/funkwhale
  • cone/funkwhale
  • thanksd/funkwhale
  • vachan-maker/funkwhale
  • bbenti/funkwhale
  • tarator/funkwhale
  • prplecake/funkwhale
  • DMarzal/funkwhale
  • lullis/funkwhale
  • hanacgr/funkwhale
  • albjeremias/funkwhale
  • xeruf/funkwhale
  • llelite/funkwhale
  • RoiArthurB/funkwhale
  • cloo/funkwhale
  • nztvar/funkwhale
  • Keunes/funkwhale
  • petitminion/funkwhale-petitminion
  • m-idler/funkwhale
  • SkyLeite/funkwhale
140 results
Select Git revision
Show changes
Showing
with 567 additions and 188 deletions
......@@ -36,7 +36,6 @@ def set_all_artists_credit(apps, schema_editor):
class Migration(migrations.Migration):
dependencies = [
("music", "0058_upload_quality"),
("playlists", "0008_playlist_library_drop"),
]
operations = [
......
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("music", "0059_remove_album_artist_remove_track_artist_artistcredit_and_more"),
("playlists", "0007_alter_playlist_actor_alter_playlisttrack_uuid_and_more"),
]
operations = []
# Generated by Django 4.2.9 on 2025-01-03 16:12
from django.db import migrations, models
from django.db import IntegrityError
from django.db import migrations, models, transaction
from django.conf import settings
from funkwhale_api.federation import utils as federation_utils
from django.urls import reverse
......@@ -10,8 +10,9 @@ import uuid
def insert_tracks_to_playlist(apps, playlist, uploads):
PlaylistTrack = apps.get_model("playlists", "PlaylistTrack")
plts = [
PlaylistTrack(
for i, upload in enumerate(uploads):
if upload.track:
PlaylistTrack.objects.create(
creation_date=playlist.creation_date,
playlist=playlist,
track=upload.track,
......@@ -19,90 +20,189 @@ def insert_tracks_to_playlist(apps, playlist, uploads):
uuid=(new_uuid := uuid.uuid4()),
fid=federation_utils.full_url(
reverse(
f"federation:music:playlists-detail",
"federation:music:playlist-tracks-detail",
kwargs={"uuid": new_uuid},
)
),
)
for i, upload in enumerate(uploads)
if upload.track
]
upload.library = None
upload.save()
return PlaylistTrack.objects.bulk_create(plts)
playlist.library.playlist_uploads.set(uploads)
@transaction.atomic
def migrate_libraries_to_playlist(apps, schema_editor):
Playlist = apps.get_model("playlists", "Playlist")
Library = apps.get_model("music", "Library")
LibraryFollow = apps.get_model("federation", "LibraryFollow")
Follow = apps.get_model("federation", "Follow")
User = apps.get_model("users", "User")
Actor = apps.get_model("federation", "Actor")
Channel = apps.get_model("audio", "Channel")
# library to playlist
to_instance_libs = []
to_public_libs = []
to_me_libs = []
for library in Library.objects.all():
playlist = Playlist.objects.create(
if (
not federation_utils.is_local(library.actor.fid)
or library.actor.name == "service"
):
continue
if (
hasattr(library, "playlist")
and library.playlist
and library.uploads.all().exists()
):
uploads = library.uploads.all()
with transaction.atomic():
insert_tracks_to_playlist(apps, library.playlist, uploads)
continue
if (
Channel.objects.filter(library=library).exists()
or Playlist.objects.filter(library=library).exists()
or not federation_utils.is_local(library.fid)
or library.name in ["me", "instance", "everyone"]
):
continue
try:
playlist, created = Playlist.objects.get_or_create(
name=library.name,
library=library,
actor=library.actor,
creation_date=library.creation_date,
privacy_level=library.privacy_level,
uuid=(new_uuid := uuid.uuid4()),
fid=federation_utils.full_url(
description=library.description,
defaults={
"uuid": (new_uuid := uuid.uuid4()),
"fid": federation_utils.full_url(
reverse(
f"federation:music:playlists-detail",
"federation:music:playlists-detail",
kwargs={"uuid": new_uuid},
)
),
},
)
playlist.save()
if library.uploads.all().exists():
insert_tracks_to_playlist(apps, playlist, library.uploads.all())
uploads = library.uploads.all()
with transaction.atomic():
insert_tracks_to_playlist(apps, playlist, uploads)
# library follows to user follow
for lib_follow in LibraryFollow.objects.filter(target=library):
try:
Follow.objects.create(
uuid=lib_follow.uuid,
target=library.actor,
actor=lib_follow.actor,
approved=lib_follow.approved,
creation_date=lib_follow.creation_date,
modification_date=lib_follow.modification_date,
)
except IntegrityError:
pass
if library.privacy_level == "me":
to_me_libs.append(library)
if library.privacy_level == "instance":
to_instance_libs.append(library)
if library.privacy_level == "everyone":
to_public_libs.append(library)
LibraryFollow.objects.all().delete()
library.privacy_level = "me"
library.playlist = playlist
library.save()
except Exception as e:
print(f"An error occurred during library.playlist creation : {e}")
continue
# migrate uploads to new library
# migrate uploads to new built-in libraries
for actor in Actor.objects.all():
if (
not federation_utils.is_local(actor.fid)
or actor.name == "service"
or hasattr(actor, "channel")
):
continue
privacy_levels = ["me", "instance", "everyone"]
for privacy_level in privacy_levels:
build_in_lib = Library.objects.create(
build_in_lib, created = Library.objects.filter(
channel__isnull=True
).get_or_create(
actor=actor,
privacy_level=privacy_level,
name=privacy_level,
uuid=(new_uuid := uuid.uuid4()),
fid=federation_utils.full_url(
defaults={
"uuid": (new_uuid := uuid.uuid4()),
"fid": federation_utils.full_url(
reverse(
f"federation:music:playlists-detail",
"federation:music:libraries-detail",
kwargs={"uuid": new_uuid},
)
),
},
)
for library in actor.libraries.filter(privacy_level=privacy_level):
library.uploads.all().update(library=build_in_lib)
if library.pk is not build_in_lib.pk:
library.delete()
library.delete
if privacy_level == "everyone":
for lib in to_public_libs:
lib.uploads.all().update(library=build_in_lib)
if privacy_level == "instance":
for lib in to_instance_libs:
lib.uploads.all().update(library=build_in_lib)
if privacy_level == "me":
for lib in to_me_libs:
lib.uploads.all().update(library=build_in_lib)
def check_succefull_migration(apps, schema_editor):
Actor = apps.get_model("federation", "Actor")
Playlist = apps.get_model("playlists", "Playlist")
for actor in Actor.objects.all():
not_build_in_libs = (
actor.playlists.count()
+ actor.libraries.filter(channel__isnull=False).count()
)
if actor.name == "service" or not federation_utils.is_local(actor.fid):
continue
elif actor.playlists.filter(library__isnull=True).count() > 0:
raise Exception(
f"Incoherent playlist database state : all local playlists do not have lib or too many libs"
)
elif (
not hasattr(actor, "channel")
and actor.libraries.count() - 3 != not_build_in_libs
or (hasattr(actor, "channel") and actor.libraries.count() > 1)
):
raise Exception(
f"Incoherent library database state, check for errors in log and share them to the funkwhale team. Migration was abordted to prevent data loss.\
actor libs = {actor.libraries.count()} and acto not built-in lib = {not_build_in_libs} \
and acto pl ={actor.playlists.count()} and not channel lib = {actor.libraries.filter(channel__isnull=False).count()} \
and actor.name = {actor.name}"
)
for playlist in Playlist.objects.all():
if not federation_utils.is_local(playlist.fid):
continue
elif playlist.library.privacy_level != "me":
raise Exception(
"Incoherent playlist database state, check for errors in log and share them to the funkwhale team. Migration was abordted to prevent data loss"
)
class Migration(migrations.Migration):
dependencies = [
("music", "0060_remove_library_description_and_more"),
("music", "0060_empty_for_test"),
("playlists", "0009_playlist_library"),
]
operations = [
migrations.AddField(
model_name="upload",
name="playlist_libraries",
field=models.ManyToManyField(
blank=True,
related_name="playlist_uploads",
to="music.library",
),
),
migrations.RunPython(
migrate_libraries_to_playlist, reverse_code=migrations.RunPython.noop
),
migrations.RunPython(
check_succefull_migration, reverse_code=migrations.RunPython.noop
),
]
# Generated by Django 4.2.9 on 2025-01-03 20:43
from django.db import migrations, models
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("music", "0059_remove_album_artist_remove_track_artist_artistcredit_and_more"),
("playlists", "0007_alter_playlist_actor_alter_playlisttrack_uuid_and_more"),
("music", "0061_migrate_libraries_to_playlist"),
]
operations = [
migrations.RemoveField(
model_name="library",
......
......@@ -5,7 +5,7 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("music", "0061_migrate_libraries_to_playlist"),
("music", "0062_del_lib_description"),
]
operations = [
......
......@@ -5,6 +5,7 @@ import os
import tempfile
import urllib.parse
import uuid
from random import randint
import arrow
import slugify
......@@ -16,7 +17,7 @@ from django.core.exceptions import ObjectDoesNotExist
from django.core.files.base import ContentFile
from django.core.serializers.json import DjangoJSONEncoder
from django.db import models, transaction
from django.db.models import Count, JSONField, Prefetch
from django.db.models import Count, JSONField, Max, Min, Prefetch
from django.db.models.expressions import OuterRef, Subquery
from django.db.models.query_utils import Q
from django.db.models.signals import post_save, pre_save
......@@ -24,7 +25,6 @@ from django.dispatch import receiver
from django.urls import reverse
from django.utils import timezone
from config import plugins
from funkwhale_api import musicbrainz
from funkwhale_api.common import fields
from funkwhale_api.common import models as common_models
......@@ -523,25 +523,36 @@ class TrackQuerySet(common_models.LocalFromFidQuerySet, models.QuerySet):
def with_playable_uploads(self, actor):
uploads = Upload.objects.playable_by(actor)
queryset = self.prefetch_related(
return self.prefetch_related(
models.Prefetch("uploads", queryset=uploads, to_attr="playable_uploads")
)
if queryset and queryset[0].uploads.count() > 0:
return queryset
else:
plugins.trigger_hook(
plugins.TRIGGER_THIRD_PARTY_UPLOAD,
track=self.first(),
)
return queryset
def order_for_album(self):
"""
Order by disc number then position
"""
return self.order_by("disc_number", "position", "title")
def random(self, batch_size):
bounds = self.aggregate(min_id=Min("id"), max_id=Max("id"))
min_id, max_id = bounds["min_id"], bounds["max_id"]
if min_id is None or max_id is None:
return self.none()
tries = 0
max_tries = 10
found_ids = set()
while len(found_ids) < batch_size and tries < max_tries:
candidate_ids = [randint(min_id, max_id) for _ in range(batch_size * 2)]
found_ids.update(
self.filter(id__in=candidate_ids).values_list("id", flat=True)
)
tries += 1
return self.filter(id__in=list(found_ids)[:batch_size]).order_by("?")
def get_artist(release_list):
return Artist.get_or_create_from_api(
......@@ -719,7 +730,7 @@ class Track(APIModelMixin):
@property
def listen_url(self) -> str:
# Not using reverse because this is slow
return f"/api/v1/listen/{self.uuid}/"
return f"/api/v2/listen/{self.uuid}/"
@property
def local_license(self):
......@@ -744,13 +755,14 @@ class UploadQuerySet(common_models.NullsLastQuerySet):
def playable_by(self, actor, include=True):
libraries = Library.objects.viewable_by(actor)
if include:
return self.filter(
library__in=libraries, import_status__in=["finished", "skipped"]
Q(library__in=libraries) | Q(playlist_libraries__in=libraries),
import_status__in=["finished", "skipped"],
)
return self.exclude(
library__in=libraries, import_status__in=["finished", "skipped"]
Q(library__in=libraries) | Q(playlist_libraries__in=libraries),
import_status__in=["finished", "skipped"],
)
def local(self, include=True):
......@@ -826,6 +838,11 @@ class Upload(models.Model):
related_name="uploads",
on_delete=models.CASCADE,
)
playlist_libraries = models.ManyToManyField(
"library",
blank=True,
related_name="playlist_uploads",
)
# metadata from federation
metadata = JSONField(
......
import os
import pathlib
import urllib.parse
from django import urls
......@@ -140,10 +142,11 @@ class ArtistWithAlbumsSerializer(OptionalDescriptionMixin, serializers.Serialize
return getattr(o, "_tracks_count", 0)
class SimpleArtistSerializer(serializers.ModelSerializer):
attachment_cover = CoverField(allow_null=True, required=False)
class ArtistSerializer(serializers.ModelSerializer):
cover = CoverField(allow_null=True, required=False)
description = common_serializers.ContentSerializer(allow_null=True, required=False)
channel = serializers.UUIDField(allow_null=True, required=False)
tags = serializers.SerializerMethodField()
class Meta:
model = models.Artist
......@@ -157,63 +160,26 @@ class SimpleArtistSerializer(serializers.ModelSerializer):
"is_local",
"content_category",
"description",
"attachment_cover",
"cover",
"channel",
"attributed_to",
"tags",
)
@extend_schema_field({"type": "array", "items": {"type": "string"}})
def get_tags(self, obj):
tagged_items = getattr(obj, "_prefetched_tagged_items", [])
return [ti.tag.name for ti in tagged_items]
class ArtistCreditSerializer(serializers.ModelSerializer):
artist = SimpleArtistSerializer()
artist = ArtistSerializer()
class Meta:
model = models.ArtistCredit
fields = ["artist", "credit", "joinphrase", "index"]
class AlbumSerializer(OptionalDescriptionMixin, serializers.Serializer):
artist_credit = ArtistCreditSerializer(many=True)
cover = CoverField(allow_null=True)
is_playable = serializers.SerializerMethodField()
tags = serializers.SerializerMethodField()
tracks_count = serializers.SerializerMethodField()
attributed_to = APIActorSerializer()
id = serializers.IntegerField()
fid = serializers.URLField()
mbid = serializers.UUIDField()
title = serializers.CharField()
release_date = serializers.DateField()
creation_date = serializers.DateTimeField()
is_local = serializers.BooleanField()
duration = serializers.SerializerMethodField(read_only=True)
def get_tracks_count(self, o) -> int:
return len(o.tracks.all())
def get_is_playable(self, obj) -> bool:
try:
return any(
[
bool(getattr(t, "is_playable_by_actor", None))
for t in obj.tracks.all()
]
)
except AttributeError:
return None
@extend_schema_field({"type": "array", "items": {"type": "string"}})
def get_tags(self, obj):
tagged_items = getattr(obj, "_prefetched_tagged_items", [])
return [ti.tag.name for ti in tagged_items]
def get_duration(self, obj) -> int:
try:
return obj.duration
except AttributeError:
# no annotation?
return 0
class TrackAlbumSerializer(serializers.ModelSerializer):
artist_credit = ArtistCreditSerializer(many=True)
cover = CoverField(allow_null=True)
......@@ -275,7 +241,7 @@ class TrackSerializer(OptionalDescriptionMixin, serializers.Serializer):
listen_url = serializers.SerializerMethodField()
tags = serializers.SerializerMethodField()
attributed_to = APIActorSerializer(allow_null=True)
description = common_serializers.ContentSerializer(allow_null=True, required=False)
id = serializers.IntegerField()
fid = serializers.URLField()
mbid = serializers.UUIDField()
......@@ -317,6 +283,51 @@ class TrackSerializer(OptionalDescriptionMixin, serializers.Serializer):
return bool(getattr(obj, "playable_uploads", []))
class AlbumSerializer(OptionalDescriptionMixin, serializers.Serializer):
artist_credit = ArtistCreditSerializer(many=True)
cover = CoverField(allow_null=True)
is_playable = serializers.SerializerMethodField()
tags = serializers.SerializerMethodField()
tracks_count = serializers.SerializerMethodField()
attributed_to = APIActorSerializer()
id = serializers.IntegerField()
fid = serializers.URLField()
mbid = serializers.UUIDField()
title = serializers.CharField()
release_date = serializers.DateField()
creation_date = serializers.DateTimeField()
is_local = serializers.BooleanField()
duration = serializers.SerializerMethodField(read_only=True)
tracks = TrackSerializer(many=True, allow_null=True)
description = common_serializers.ContentSerializer(allow_null=True, required=False)
def get_tracks_count(self, o) -> int:
return len(o.tracks.all())
def get_is_playable(self, obj) -> bool:
try:
return any(
[
bool(getattr(t, "is_playable_by_actor", None))
for t in obj.tracks.all()
]
)
except AttributeError:
return None
@extend_schema_field({"type": "array", "items": {"type": "string"}})
def get_tags(self, obj):
tagged_items = getattr(obj, "_prefetched_tagged_items", [])
return [ti.tag.name for ti in tagged_items]
def get_duration(self, obj) -> int:
try:
return obj.duration
except AttributeError:
# no annotation?
return 0
@common_serializers.track_fields_for_update("name", "description", "privacy_level")
class LibraryForOwnerSerializer(serializers.ModelSerializer):
uploads_count = serializers.SerializerMethodField()
......@@ -363,6 +374,9 @@ class UploadSerializer(serializers.ModelSerializer):
required=False,
filters=lambda context: {"actor": context["user"].actor},
)
privacy_level = serializers.ChoiceField(
choices=models.LIBRARY_PRIVACY_LEVEL_CHOICES, required=False
)
channel = common_serializers.RelatedField(
"uuid",
ChannelSerializer(),
......@@ -386,6 +400,7 @@ class UploadSerializer(serializers.ModelSerializer):
"size",
"import_date",
"import_status",
"privacy_level",
]
read_only_fields = [
......@@ -450,6 +465,7 @@ class ImportMetadataSerializer(serializers.Serializer):
)
@extend_schema_field(ImportMetadataSerializer)
class ImportMetadataField(serializers.JSONField):
def to_internal_value(self, v):
v = super().to_internal_value(v)
......@@ -486,6 +502,7 @@ class UploadForOwnerSerializer(UploadSerializer):
r = super().to_representation(obj)
if "audio_file" in r:
del r["audio_file"]
r["privacy_level"] = obj.library.privacy_level
return r
def validate(self, validated_data):
......@@ -540,10 +557,14 @@ class UploadBulkUpdateSerializer(serializers.Serializer):
raise serializers.ValidationError(
f"Upload with uuid {data['uuid']} does not exist"
)
lib = upload.library.actor.libraries.filter(
privacy_level=data["privacy_level"], name=data["privacy_level"]
).exclude(playlist__isnull=False)
upload.library = upload.library.actor.libraries.get(
privacy_level=data["privacy_level"]
)
if len(lib) == 1:
upload.library = lib[0]
else:
raise serializers.ValidationError("Built-in library not found or too many")
return upload
......@@ -884,13 +905,17 @@ class FSImportSerializer(serializers.Serializer):
prune = serializers.BooleanField(required=False, default=True)
outbox = serializers.BooleanField(required=False, default=False)
broadcast = serializers.BooleanField(required=False, default=False)
replace = serializers.BooleanField(required=False, default=False)
batch_size = serializers.IntegerField(required=False, default=1000)
verbosity = serializers.IntegerField(required=False, default=1)
def validate_path(self, value):
try:
utils.browse_dir(settings.MUSIC_DIRECTORY_PATH, value)
except (NotADirectoryError, FileNotFoundError, ValueError):
except NotADirectoryError:
if not os.path.isfile(pathlib.Path(settings.MUSIC_DIRECTORY_PATH) / value):
raise serializers.ValidationError("Invalid path")
except (FileNotFoundError, ValueError):
raise serializers.ValidationError("Invalid path")
return value
......
......@@ -1209,6 +1209,7 @@ def fs_import(
prune=True,
outbox=False,
broadcast=False,
replace=False,
batch_size=1000,
verbosity=1,
):
......@@ -1229,7 +1230,7 @@ def fs_import(
"batch_size": batch_size,
"async_": False,
"prune": prune,
"replace": False,
"replace": replace,
"verbosity": verbosity,
"exit_on_failure": False,
"outbox": outbox,
......
......@@ -386,6 +386,7 @@ class LibraryViewSet(
prune=serializer.validated_data["prune"],
outbox=serializer.validated_data["outbox"],
broadcast=serializer.validated_data["broadcast"],
replace=serializer.validated_data["replace"],
batch_size=serializer.validated_data["batch_size"],
verbosity=serializer.validated_data["verbosity"],
)
......@@ -798,6 +799,9 @@ class UploadViewSet(
cover_data["content"] = base64.b64encode(cover_data["content"])
return Response(payload, status=200)
@extend_schema(
request=serializers.UploadBulkUpdateSerializer(many=True),
)
@action(detail=False, methods=["patch"])
def bulk_update(self, request, *args, **kwargs):
"""
......@@ -811,7 +815,9 @@ class UploadViewSet(
models.Upload.objects.bulk_update(serializer.validated_data, ["library"])
return Response(
serializers.UploadForOwnerSerializer(serializer.validated_data).data,
serializers.UploadForOwnerSerializer(
serializer.validated_data, many=True
).data,
status=200,
)
......
......@@ -15,3 +15,22 @@ class PlaylistTrackAdmin(admin.ModelAdmin):
list_display = ["playlist", "track", "index"]
search_fields = ["track__name", "playlist__name"]
list_select_related = True
@admin.register(models.PlaylistScan)
class LibraryScanAdmin(admin.ModelAdmin):
list_display = [
"id",
"playlist",
"actor",
"status",
"creation_date",
"modification_date",
"status",
"total_files",
"processed_files",
"errored_files",
]
list_select_related = True
search_fields = ["actor__username", "playlist__name"]
list_filter = ["status"]
......@@ -3,7 +3,7 @@ from django.conf import settings
from funkwhale_api.factories import NoUpdateOnCreate, registry
from funkwhale_api.federation import models
from funkwhale_api.federation.factories import ActorFactory
from funkwhale_api.federation.factories import ActorFactory, MusicLibraryFactory
from funkwhale_api.music.factories import TrackFactory
......@@ -13,6 +13,7 @@ class PlaylistFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
actor = factory.SubFactory(ActorFactory)
fid = factory.Faker("federation_url")
uuid = factory.Faker("uuid4")
library = factory.SubFactory(MusicLibraryFactory)
class Meta:
model = "playlists.Playlist"
......
......@@ -25,7 +25,7 @@ def gen_uuid(apps, schema_editor):
unique_uuid = uuid.uuid4()
fid = utils.full_url(
reverse("federation:music:playlist-detail", kwargs={"uuid": unique_uuid})
reverse("federation:music:playlists-detail", kwargs={"uuid": unique_uuid})
)
row.uuid = unique_uuid
row.fid = fid
......@@ -42,7 +42,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name="playlist",
name="fid",
field=models.URLField(max_length=500 ),
field=models.URLField(max_length=500, null=True),
),
migrations.AddField(
model_name="playlist",
......@@ -63,8 +63,13 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name="playlist",
name="fid",
field=models.URLField(max_length=500, unique=True, db_index=True,
),),
field=models.URLField(
max_length=500,
unique=True,
db_index=True,
null=False,
),
),
migrations.AddField(
model_name="playlist",
name="actor",
......
......@@ -9,14 +9,14 @@ from funkwhale_api.federation import utils
from django.urls import reverse
def gen_uuid(apps, schema_editor):
MyModel = apps.get_model("playlists", "Playlist")
MyModel = apps.get_model("playlists", "PlaylistTrack")
for row in MyModel.objects.all():
unique_uuid = uuid.uuid4()
while MyModel.objects.filter(uuid=unique_uuid).exists():
unique_uuid = uuid.uuid4()
fid = utils.full_url(
reverse("federation:music:playlist-detail", kwargs={"uuid": unique_uuid})
reverse("federation:music:playlists-detail", kwargs={"uuid": unique_uuid})
)
row.uuid = unique_uuid
row.fid = fid
......@@ -38,8 +38,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name="playlisttrack",
name="fid",
field=models.URLField(max_length=500
),
field=models.URLField(max_length=500, null=True),
),
migrations.AddField(
model_name="playlisttrack",
......@@ -48,7 +47,7 @@ class Migration(migrations.Migration):
),
migrations.RunPython(gen_uuid, reverse_code=migrations.RunPython.noop),
migrations.AlterField(
model_name="playlist",
model_name="playlisttrack",
name="uuid",
field=models.UUIDField(default=uuid.uuid4, null=False, unique=True),
),
......@@ -56,6 +55,7 @@ class Migration(migrations.Migration):
model_name="playlisttrack",
name="fid",
field=models.URLField(
db_index=True, max_length=500, unique=True
),),
db_index=True, max_length=500, unique=True, null=False
),
),
]
import django.db.models.deletion
from django.db import migrations, models, transaction
from funkwhale_api.federation import utils as federation_utils
from django.urls import reverse
import uuid
from django.conf import settings
def add_uploads_to_pl_library(playlist, library):
for plt in playlist.playlist_tracks.all():
for upload in plt.track.uploads.filter(library__actor=playlist.actor):
library.uploads.add(upload)
@transaction.atomic
def create_playlist_libraries(apps, schema_editor):
Playlist = apps.get_model("playlists", "Playlist")
Library = apps.get_model("music", "Library")
Actor = apps.get_model("federation", "Actor")
playlist_with_lib_count = 0
for playlist in Playlist.objects.all():
if not federation_utils.is_local(playlist.actor.fid):
continue
library = playlist.library
if not library:
try:
# we don't want to get_or_create in case it's a channel lib
library = Library.objects.create(
name="playlist_" + playlist.name,
privacy_level="me",
actor=playlist.actor,
uuid=(new_uuid := uuid.uuid4()),
fid=federation_utils.full_url(
reverse(
"federation:music:libraries-detail",
kwargs={"uuid": new_uuid},
)
),
)
library.save()
playlist.library = library
playlist.save()
with transaction.atomic():
add_uploads_to_pl_library(playlist, library)
except Exception as e:
print(
f"An error occurred during playlist.library creation, raising since we want\
to enforce one lib per playlist"
)
raise e
playlist_with_lib_count = playlist_with_lib_count + 1
local_actors = Actor.objects.filter(domain_id=settings.FEDERATION_HOSTNAME)
if (
Library.objects.filter(
playlist__isnull=False, actor__in=local_actors
).count()
!= playlist_with_lib_count
):
raise Exception(
"Should have the same amount of local playlist and libraries with playlist"
)
class Migration(migrations.Migration):
dependencies = [
("playlists", "0008_playlist_library_drop"),
]
operations = [
migrations.AddField(
model_name="playlist",
name="library",
field=models.OneToOneField(
blank=True,
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="playlist",
to="music.library",
),
),
migrations.RunPython(
create_playlist_libraries, reverse_code=migrations.RunPython.noop
),
]
......@@ -88,12 +88,19 @@ class Playlist(federation_models.FederationMixin):
description = models.TextField(max_length=5000, null=True, blank=True)
objects = PlaylistQuerySet.as_manager()
federation_namespace = "playlists"
library = models.OneToOneField(
"music.Library",
null=True,
blank=True,
on_delete=models.CASCADE,
related_name="playlist",
)
def __str__(self):
return self.name
def get_absolute_url(self):
return f"/library/playlists/{self.pk}"
return f"/library/playlists/{self.uuid}"
def get_federation_id(self):
if self.fid:
......@@ -109,6 +116,19 @@ class Playlist(federation_models.FederationMixin):
if not self.pk and not self.fid:
self.fid = self.get_federation_id()
if not self.pk and not self.library_id:
self.library = music_models.Library.objects.create(
actor=self.actor,
name="playlist_" + self.name,
privacy_level="me",
uuid=(new_uuid := uuid.uuid4()),
fid=federation_utils.full_url(
reverse(
"federation:music:libraries-detail", kwargs={"uuid": new_uuid}
),
),
)
return super().save(**kwargs)
@transaction.atomic
......@@ -232,7 +252,7 @@ class Playlist(federation_models.FederationMixin):
latest_scan = (
self.scans.exclude(status="errored").order_by("-creation_date").first()
)
delay_between_scans = datetime.timedelta(seconds=3600 * 24)
delay_between_scans = datetime.timedelta(seconds=1)
now = timezone.now()
if (
not force
......@@ -287,7 +307,7 @@ class PlaylistTrackQuerySet(models.QuerySet, common_models.LocalFromFidQuerySet)
return self.annotate(is_playable_by_actor=subquery)
def playable_by(self, actor, include=True):
tracks = music_models.Track.objects.playable_by(actor, include)
tracks = music_models.Track.objects.playable_by(actor)
if include:
return self.filter(track__pk__in=tracks).distinct()
else:
......@@ -345,6 +365,9 @@ class PlaylistTrack(federation_models.FederationMixin):
return super().save(**kwargs)
def get_absolute_url(self):
return f"/library/tracks/{self.track.pk}"
class PlaylistScan(models.Model):
actor = models.ForeignKey(
......
......@@ -15,7 +15,7 @@ class PlaylistXspfRenderer(renderers.BaseRenderer):
if isinstance(data, bytes):
return data
fw_playlist = Playlist.objects.get(id=data["id"])
fw_playlist = Playlist.objects.get(uuid=data["uuid"])
plt_tracks = fw_playlist.playlist_tracks.prefetch_related("track")
top = Element("playlist", version="1", xmlns="http://xspf.org/ns/0/")
title_xspf = SubElement(top, "title")
......
......@@ -34,11 +34,14 @@ class PlaylistSerializer(serializers.ModelSerializer):
album_covers = serializers.SerializerMethodField(read_only=True)
is_playable = serializers.SerializerMethodField()
actor = APIActorSerializer(read_only=True)
library = serializers.SerializerMethodField()
library_followed = serializers.SerializerMethodField()
class Meta:
model = models.Playlist
fields = (
"id",
"uuid",
"fid",
"name",
"actor",
"modification_date",
......@@ -49,8 +52,37 @@ class PlaylistSerializer(serializers.ModelSerializer):
"duration",
"is_playable",
"actor",
"description",
"library",
"library_followed",
)
read_only_fields = ["id", "modification_date", "creation_date"]
read_only_fields = ["uuid", "fid", "modification_date", "creation_date"]
@extend_schema_field(OpenApiTypes.URI)
def get_library(self, obj):
if obj.library:
return obj.library.fid
else:
return None
@extend_schema_field(OpenApiTypes.BOOL)
def get_library_followed(self, obj):
if (
self.context.get("request", False)
and hasattr(self.context["request"], "user")
and hasattr(self.context["request"].user, "actor")
):
actor = self.context["request"].user.actor
lib_qs = obj.library.received_follows.filter(actor=actor)
if lib_qs.exists():
if lib_qs[0].approved is None:
return False
else:
return lib_qs[0].approved
else:
return None
return None
@extend_schema_field(OpenApiTypes.BOOL)
def get_is_playable(self, obj):
......
import logging
import requests
from django.db.models import F
from django.utils import timezone
......@@ -9,6 +11,8 @@ from funkwhale_api.taskapp import celery
from . import models
logger = logging.getLogger(__name__)
def get_playlist_data(playlist_url, actor):
auth = signing.get_auth(actor.private_key, actor.private_key_id)
......@@ -24,7 +28,11 @@ def get_playlist_data(playlist_url, actor):
if scode == 401:
return {"errors": ["This playlist requires authentication"]}
elif scode == 403:
return {"errors": ["Permission denied while scanning playlist"]}
return {
"errors": [
f"Permission denied while scanning playlist. Error : {scode}. PLaylist url = {playlist_url}"
]
}
elif scode >= 400:
return {"errors": [f"Error {scode} while fetching the playlist"]}
serializer = serializers.PlaylistCollectionSerializer(data=response.json())
......@@ -47,6 +55,7 @@ def get_playlist_page(playlist, page_url, actor):
context={
"playlist": playlist,
"item_serializer": serializers.PlaylistTrackSerializer,
"conf": {"library": playlist.library},
},
)
serializer.is_valid(raise_exception=True)
......@@ -60,6 +69,7 @@ def get_playlist_page(playlist, page_url, actor):
)
def start_playlist_scan(playlist_scan):
playlist_scan.playlist.playlist_tracks.all().delete()
try:
data = get_playlist_data(playlist_scan.playlist.fid, actor=playlist_scan.actor)
except Exception:
......@@ -90,13 +100,30 @@ def start_playlist_scan(playlist_scan):
)
def scan_playlist_page(playlist_scan, page_url):
data = get_playlist_page(playlist_scan.playlist, page_url, playlist_scan.actor)
tracks = []
plts = []
for item_serializer in data["items"]:
print(" item_serializer is " + str(item_serializer))
track = item_serializer.save(playlist=playlist_scan.playlist.fid)
tracks.append(track)
try:
plt = item_serializer.save(playlist=playlist_scan.playlist.fid)
# we get any upload owned by the playlist.actor and add a m2m with playlist_libraries
upload_qs = plt.track.uploads.filter(
library__actor=playlist_scan.playlist.actor
)
if not upload_qs:
logger.debug(
f"Could not find a upload for the playlist track {plt.track.title}. Probably the \
playlist.library library_scan failed or was not launched by inbox_update_playlist ?"
)
else:
upload_qs[0].playlist_libraries.add(playlist_scan.playlist.library)
logger.debug(f"Added {plt.track.title} to playlist library")
plts.append(plt)
except Exception as e:
logger.info(
f"Error while saving track to playlist {playlist_scan.playlist}: {e}"
)
continue
playlist_scan.processed_files = F("processed_files") + len(tracks)
playlist_scan.processed_files = F("processed_files") + len(plts)
playlist_scan.modification_date = timezone.now()
update_fields = ["modification_date", "processed_files"]
......
import logging
from itertools import chain
from django.conf import settings
from django.db import transaction
from django.db.models import Count
from drf_spectacular.utils import extend_schema
from rest_framework import exceptions, mixins, status, viewsets
from rest_framework.decorators import action
from rest_framework.pagination import PageNumberPagination
from rest_framework.parsers import FormParser, JSONParser, MultiPartParser
from rest_framework.renderers import JSONRenderer
from rest_framework.response import Response
from config import plugins
from funkwhale_api.common import fields, permissions
from funkwhale_api.federation import routes
from funkwhale_api.music import models as music_models
......@@ -29,6 +33,7 @@ class PlaylistViewSet(
mixins.ListModelMixin,
viewsets.GenericViewSet,
):
lookup_field = "uuid"
serializer_class = serializers.PlaylistSerializer
queryset = (
models.Playlist.objects.all()
......@@ -125,12 +130,25 @@ class PlaylistViewSet(
@action(methods=["get"], detail=True)
def tracks(self, request, *args, **kwargs):
playlist = self.get_object()
plts = playlist.playlist_tracks.all().for_nested_serialization(
music_utils.get_actor_from_request(request)
actor = music_utils.get_actor_from_request(request)
plts = playlist.playlist_tracks.all().for_nested_serialization(actor)
for plt in plts.playable_by(actor, include=False)[
: settings.THIRD_PARTY_UPLOAD_MAX_UPLOADS
]:
plugins.trigger_hook(
plugins.TRIGGER_THIRD_PARTY_UPLOAD,
track=plt.track,
actor=actor,
)
serializer = serializers.PlaylistTrackSerializer(plts, many=True)
data = {"count": len(plts), "results": serializer.data}
return Response(data, status=200)
# Apply pagination
paginator = PageNumberPagination()
paginator.page_size = 50 # Set the page size (number of items per page)
paginated_plts = paginator.paginate_queryset(plts, request)
# Serialize the paginated data
serializer = serializers.PlaylistTrackSerializer(paginated_plts, many=True)
return paginator.get_paginated_response(serializer.data)
@extend_schema(
operation_id="add_to_playlist", request=serializers.PlaylistAddManySerializer
......@@ -157,6 +175,7 @@ class PlaylistViewSet(
)
serializer = serializers.PlaylistTrackSerializer(plts, many=True)
data = {"count": len(plts), "results": serializer.data}
update_playlist_library_uploads(playlist, plts)
playlist.schedule_scan(playlist.actor, force=True)
return Response(data, status=201)
......@@ -167,7 +186,8 @@ class PlaylistViewSet(
playlist = self.get_object()
playlist.playlist_tracks.all().delete()
playlist.save(update_fields=["modification_date"])
playlist.schedule_scan(playlist.actor)
playlist.library.uploads.filter().delete()
playlist.schedule_scan(playlist.actor, force=True)
return Response(status=204)
def get_queryset(self):
......@@ -200,6 +220,8 @@ class PlaylistViewSet(
plt = playlist.playlist_tracks.by_index(index)
except models.PlaylistTrack.DoesNotExist:
return Response(status=404)
for upload in plt.track.uploads.filter(playlist_libraries=playlist.library):
upload.playlist_libraries.remove(playlist.library)
plt.delete(update_indexes=True)
plt.playlist.schedule_scan(playlist.actor)
return Response(status=204)
......@@ -244,7 +266,7 @@ class PlaylistViewSet(
serializer = music_serializers.AlbumSerializer(releases, many=True)
return Response(serializer.data, status=200)
@extend_schema(operation_id="get_playlist_artits")
@extend_schema(operation_id="get_playlist_artists")
@action(methods=["get"], detail=True)
@transaction.atomic
def artists(self, request, *args, **kwargs):
......@@ -256,5 +278,15 @@ class PlaylistViewSet(
except models.PlaylistTrack.DoesNotExist:
return Response(status=404)
artists = music_models.Artist.objects.filter(pk__in=artists_pks)
serializer = music_serializers.SimpleArtistSerializer(artists, many=True)
serializer = music_serializers.ArtistSerializer(artists, many=True)
return Response(serializer.data, status=200)
def update_playlist_library_uploads(playlist, plts):
uploads = list(
chain(
*[plt.track.uploads.filter(library__actor=playlist.actor) for plt in plts]
)
)
for upload in uploads:
upload.playlist_libraries.add(playlist.library)
......@@ -109,7 +109,7 @@ class SessionRadio(SimpleRadio):
queryset = self.filter_queryset(queryset)
# select a random batch of the qs
sliced_queryset = queryset.order_by("?")[:BATCH_SIZE]
sliced_queryset = queryset.random(BATCH_SIZE)
if len(sliced_queryset) <= 0 and not cached_evaluated_radio_tracks:
raise ValueError("No more radio candidates")
......@@ -166,7 +166,7 @@ class SessionRadio(SimpleRadio):
class RandomRadio(SessionRadio):
def get_queryset(self, **kwargs):
qs = super().get_queryset(**kwargs)
return qs.filter(artist_credit__artist__content_category="music").order_by("?")
return qs.filter(artist_credit__artist__content_category="music").random(100)
@registry.register(name="random_library")
......@@ -179,7 +179,7 @@ class RandomLibraryRadio(SessionRadio):
query = Q(artist_credit__artist__content_category="music") & Q(
pk__in=tracks_ids
)
return qs.filter(query).order_by("?")
return qs.filter(query).random(100)
@registry.register(name="favorites")
......@@ -390,7 +390,7 @@ class LessListenedRadio(SessionRadio):
return (
qs.filter(artist_credit__artist__content_category="music")
.exclude(pk__in=listened)
.order_by("?")
.random(100)
)
......@@ -411,7 +411,7 @@ class LessListenedLibraryRadio(SessionRadio):
query = Q(artist_credit__artist__content_category="music") & Q(
pk__in=tracks_ids
)
return qs.filter(query).exclude(pk__in=listened).order_by("?")
return qs.filter(query).exclude(pk__in=listened).random(100)
@registry.register(name="actor-content")
......