api/funkwhale_api/music/migrations/0059_remove_album_artist_remove_track_artist_artistcredit_and_more.py

@@ -36,7 +36,6 @@ def set_all_artists_credit(apps, schema_editor):
 class Migration(migrations.Migration):
     dependencies = [
         ("music", "0058_upload_quality"),
-        ("playlists", "0008_playlist_library_drop"),
     ]
 
     operations = [

api/funkwhale_api/music/migrations/0060_empty_for_test.py

+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("music", "0059_remove_album_artist_remove_track_artist_artistcredit_and_more"),
+        ("playlists", "0007_alter_playlist_actor_alter_playlisttrack_uuid_and_more"),
+    ]
+
+    operations = []

api/funkwhale_api/music/migrations/0061_migrate_libraries_to_playlist.py

 # Generated by Django 4.2.9 on 2025-01-03 16:12
 
-from django.db import migrations, models
-from django.db import IntegrityError
+from django.db import migrations, models, transaction
+from django.conf import settings
 
 from funkwhale_api.federation import utils as federation_utils
 from django.urls import reverse
@@ -10,8 +10,9 @@ import uuid
 
 def insert_tracks_to_playlist(apps, playlist, uploads):
     PlaylistTrack = apps.get_model("playlists", "PlaylistTrack")
-    plts = [
-        PlaylistTrack(
+    for i, upload in enumerate(uploads):
+        if upload.track:
+            PlaylistTrack.objects.create(
                 creation_date=playlist.creation_date,
                 playlist=playlist,
                 track=upload.track,
@@ -19,90 +20,189 @@ def insert_tracks_to_playlist(apps, playlist, uploads):
                 uuid=(new_uuid := uuid.uuid4()),
                 fid=federation_utils.full_url(
                     reverse(
-                    f"federation:music:playlists-detail",
+                        "federation:music:playlist-tracks-detail",
                         kwargs={"uuid": new_uuid},
                     )
                 ),
             )
-        for i, upload in enumerate(uploads)
-        if upload.track
-    ]
+            upload.library = None
+            upload.save()
 
-    return PlaylistTrack.objects.bulk_create(plts)
+    playlist.library.playlist_uploads.set(uploads)
 
 
+@transaction.atomic
 def migrate_libraries_to_playlist(apps, schema_editor):
     Playlist = apps.get_model("playlists", "Playlist")
     Library = apps.get_model("music", "Library")
-    LibraryFollow = apps.get_model("federation", "LibraryFollow")
-    Follow = apps.get_model("federation", "Follow")
-    User = apps.get_model("users", "User")
     Actor = apps.get_model("federation", "Actor")
+    Channel = apps.get_model("audio", "Channel")
 
-    # library to playlist
+    to_instance_libs = []
+    to_public_libs = []
+    to_me_libs = []
     for library in Library.objects.all():
-        playlist = Playlist.objects.create(
+        if (
+            not federation_utils.is_local(library.actor.fid)
+            or library.actor.name == "service"
+        ):
+            continue
+
+        if (
+            hasattr(library, "playlist")
+            and library.playlist
+            and library.uploads.all().exists()
+        ):
+            uploads = library.uploads.all()
+            with transaction.atomic():
+                insert_tracks_to_playlist(apps, library.playlist, uploads)
+            continue
+
+        if (
+            Channel.objects.filter(library=library).exists()
+            or Playlist.objects.filter(library=library).exists()
+            or not federation_utils.is_local(library.fid)
+            or library.name in ["me", "instance", "everyone"]
+        ):
+            continue
+
+        try:
+            playlist, created = Playlist.objects.get_or_create(
                 name=library.name,
+                library=library,
                 actor=library.actor,
                 creation_date=library.creation_date,
                 privacy_level=library.privacy_level,
-            uuid=(new_uuid := uuid.uuid4()),
-            fid=federation_utils.full_url(
+                description=library.description,
+                defaults={
+                    "uuid": (new_uuid := uuid.uuid4()),
+                    "fid": federation_utils.full_url(
                         reverse(
-                    f"federation:music:playlists-detail",
+                            "federation:music:playlists-detail",
                             kwargs={"uuid": new_uuid},
                         )
                     ),
+                },
             )
             playlist.save()
 
             if library.uploads.all().exists():
-            insert_tracks_to_playlist(apps, playlist, library.uploads.all())
+                uploads = library.uploads.all()
+                with transaction.atomic():
+                    insert_tracks_to_playlist(apps, playlist, uploads)
 
-        # library follows to user follow
-        for lib_follow in LibraryFollow.objects.filter(target=library):
-            try:
-                Follow.objects.create(
-                    uuid=lib_follow.uuid,
-                    target=library.actor,
-                    actor=lib_follow.actor,
-                    approved=lib_follow.approved,
-                    creation_date=lib_follow.creation_date,
-                    modification_date=lib_follow.modification_date,
-                )
-            except IntegrityError:
-                pass
+            if library.privacy_level == "me":
+                to_me_libs.append(library)
+            if library.privacy_level == "instance":
+                to_instance_libs.append(library)
+            if library.privacy_level == "everyone":
+                to_public_libs.append(library)
 
-        LibraryFollow.objects.all().delete()
+            library.privacy_level = "me"
+            library.playlist = playlist
+            library.save()
+        except Exception as e:
+            print(f"An error occurred during library.playlist creation : {e}")
+            continue
 
-    # migrate uploads to new library
+    # migrate uploads to new built-in libraries
     for actor in Actor.objects.all():
+        if (
+            not federation_utils.is_local(actor.fid)
+            or actor.name == "service"
+            or hasattr(actor, "channel")
+        ):
+            continue
+
         privacy_levels = ["me", "instance", "everyone"]
         for privacy_level in privacy_levels:
-            build_in_lib = Library.objects.create(
+            build_in_lib, created = Library.objects.filter(
+                channel__isnull=True
+            ).get_or_create(
                 actor=actor,
                 privacy_level=privacy_level,
                 name=privacy_level,
-                uuid=(new_uuid := uuid.uuid4()),
-                fid=federation_utils.full_url(
+                defaults={
+                    "uuid": (new_uuid := uuid.uuid4()),
+                    "fid": federation_utils.full_url(
                         reverse(
-                        f"federation:music:playlists-detail",
+                            "federation:music:libraries-detail",
                             kwargs={"uuid": new_uuid},
                         )
                     ),
+                },
             )
             for library in actor.libraries.filter(privacy_level=privacy_level):
                 library.uploads.all().update(library=build_in_lib)
-                if library.pk is not build_in_lib.pk:
-                    library.delete()
+                library.delete
+
+            if privacy_level == "everyone":
+                for lib in to_public_libs:
+                    lib.uploads.all().update(library=build_in_lib)
+            if privacy_level == "instance":
+                for lib in to_instance_libs:
+                    lib.uploads.all().update(library=build_in_lib)
+            if privacy_level == "me":
+                for lib in to_me_libs:
+                    lib.uploads.all().update(library=build_in_lib)
+
+
+def check_succefull_migration(apps, schema_editor):
+    Actor = apps.get_model("federation", "Actor")
+    Playlist = apps.get_model("playlists", "Playlist")
+
+    for actor in Actor.objects.all():
+        not_build_in_libs = (
+            actor.playlists.count()
+            + actor.libraries.filter(channel__isnull=False).count()
+        )
+
+        if actor.name == "service" or not federation_utils.is_local(actor.fid):
+            continue
+
+        elif actor.playlists.filter(library__isnull=True).count() > 0:
+            raise Exception(
+                f"Incoherent playlist database state : all local playlists do not have lib or too many libs"
+            )
+        elif (
+            not hasattr(actor, "channel")
+            and actor.libraries.count() - 3 != not_build_in_libs
+            or (hasattr(actor, "channel") and actor.libraries.count() > 1)
+        ):
+            raise Exception(
+                f"Incoherent library database state, check for errors in log and share them to the funkwhale team. Migration was abordted to prevent data loss.\
+                    actor libs = {actor.libraries.count()} and acto not built-in lib = {not_build_in_libs} \
+                    and acto pl ={actor.playlists.count()} and not channel lib = {actor.libraries.filter(channel__isnull=False).count()} \
+                    and actor.name = {actor.name}"
+            )
+    for playlist in Playlist.objects.all():
+        if not federation_utils.is_local(playlist.fid):
+            continue
+        elif playlist.library.privacy_level != "me":
+            raise Exception(
+                "Incoherent playlist database state, check for errors in log and share them to the funkwhale team. Migration was abordted to prevent data loss"
+            )
 
 
 class Migration(migrations.Migration):
     dependencies = [
-        ("music", "0060_remove_library_description_and_more"),
+        ("music", "0060_empty_for_test"),
+        ("playlists", "0009_playlist_library"),
     ]
     operations = [
+        migrations.AddField(
+            model_name="upload",
+            name="playlist_libraries",
+            field=models.ManyToManyField(
+                blank=True,
+                related_name="playlist_uploads",
+                to="music.library",
+            ),
+        ),
         migrations.RunPython(
             migrate_libraries_to_playlist, reverse_code=migrations.RunPython.noop
         ),
+        migrations.RunPython(
+            check_succefull_migration, reverse_code=migrations.RunPython.noop
+        ),
     ]

api/funkwhale_api/music/migrations/0060_remove_library_description_and_more.py → api/funkwhale_api/music/migrations/0062_del_lib_description.py

-# Generated by Django 4.2.9 on 2025-01-03 20:43
-
-from django.db import migrations, models
+from django.db import migrations
 
 
 class Migration(migrations.Migration):
     dependencies = [
-        ("music", "0059_remove_album_artist_remove_track_artist_artistcredit_and_more"),
-        ("playlists", "0007_alter_playlist_actor_alter_playlisttrack_uuid_and_more"),
+        ("music", "0061_migrate_libraries_to_playlist"),
     ]
-
     operations = [
         migrations.RemoveField(
             model_name="library",

api/funkwhale_api/music/migrations/0062_upload_third_party_provider.py → api/funkwhale_api/music/migrations/0063_upload_third_party_provider.py

@@ -5,7 +5,7 @@ from django.db import migrations, models
 
 class Migration(migrations.Migration):
     dependencies = [
-        ("music", "0061_migrate_libraries_to_playlist"),
+        ("music", "0062_del_lib_description"),
     ]
 
     operations = [

api/funkwhale_api/music/models.py

@@ -5,6 +5,7 @@ import os
 import tempfile
 import urllib.parse
 import uuid
+from random import randint
 
 import arrow
 import slugify
@@ -16,7 +17,7 @@ from django.core.exceptions import ObjectDoesNotExist
 from django.core.files.base import ContentFile
 from django.core.serializers.json import DjangoJSONEncoder
 from django.db import models, transaction
-from django.db.models import Count, JSONField, Prefetch
+from django.db.models import Count, JSONField, Max, Min, Prefetch
 from django.db.models.expressions import OuterRef, Subquery
 from django.db.models.query_utils import Q
 from django.db.models.signals import post_save, pre_save
@@ -24,7 +25,6 @@ from django.dispatch import receiver
 from django.urls import reverse
 from django.utils import timezone
 
-from config import plugins
 from funkwhale_api import musicbrainz
 from funkwhale_api.common import fields
 from funkwhale_api.common import models as common_models
@@ -523,25 +523,36 @@ class TrackQuerySet(common_models.LocalFromFidQuerySet, models.QuerySet):
 
     def with_playable_uploads(self, actor):
         uploads = Upload.objects.playable_by(actor)
-        queryset = self.prefetch_related(
+        return self.prefetch_related(
             models.Prefetch("uploads", queryset=uploads, to_attr="playable_uploads")
         )
 
-        if queryset and queryset[0].uploads.count() > 0:
-            return queryset
-        else:
-            plugins.trigger_hook(
-                plugins.TRIGGER_THIRD_PARTY_UPLOAD,
-                track=self.first(),
-            )
-            return queryset
-
     def order_for_album(self):
         """
         Order by disc number then position
         """
         return self.order_by("disc_number", "position", "title")
 
+    def random(self, batch_size):
+        bounds = self.aggregate(min_id=Min("id"), max_id=Max("id"))
+        min_id, max_id = bounds["min_id"], bounds["max_id"]
+
+        if min_id is None or max_id is None:
+            return self.none()
+
+        tries = 0
+        max_tries = 10
+        found_ids = set()
+
+        while len(found_ids) < batch_size and tries < max_tries:
+            candidate_ids = [randint(min_id, max_id) for _ in range(batch_size * 2)]
+            found_ids.update(
+                self.filter(id__in=candidate_ids).values_list("id", flat=True)
+            )
+            tries += 1
+
+        return self.filter(id__in=list(found_ids)[:batch_size]).order_by("?")
+
 
 def get_artist(release_list):
     return Artist.get_or_create_from_api(
@@ -719,7 +730,7 @@ class Track(APIModelMixin):
     @property
     def listen_url(self) -> str:
         # Not using reverse because this is slow
-        return f"/api/v1/listen/{self.uuid}/"
+        return f"/api/v2/listen/{self.uuid}/"
 
     @property
     def local_license(self):
@@ -744,13 +755,14 @@ class UploadQuerySet(common_models.NullsLastQuerySet):
 
     def playable_by(self, actor, include=True):
         libraries = Library.objects.viewable_by(actor)
-
         if include:
             return self.filter(
-                library__in=libraries, import_status__in=["finished", "skipped"]
+                Q(library__in=libraries) | Q(playlist_libraries__in=libraries),
+                import_status__in=["finished", "skipped"],
             )
         return self.exclude(
-            library__in=libraries, import_status__in=["finished", "skipped"]
+            Q(library__in=libraries) | Q(playlist_libraries__in=libraries),
+            import_status__in=["finished", "skipped"],
         )
 
     def local(self, include=True):
@@ -826,6 +838,11 @@ class Upload(models.Model):
         related_name="uploads",
         on_delete=models.CASCADE,
     )
+    playlist_libraries = models.ManyToManyField(
+        "library",
+        blank=True,
+        related_name="playlist_uploads",
+    )
 
     # metadata from federation
     metadata = JSONField(

api/funkwhale_api/music/serializers.py

+import os
+import pathlib
 import urllib.parse
 
 from django import urls
@@ -140,10 +142,11 @@ class ArtistWithAlbumsSerializer(OptionalDescriptionMixin, serializers.Serialize
         return getattr(o, "_tracks_count", 0)
 
 
-class SimpleArtistSerializer(serializers.ModelSerializer):
-    attachment_cover = CoverField(allow_null=True, required=False)
+class ArtistSerializer(serializers.ModelSerializer):
+    cover = CoverField(allow_null=True, required=False)
     description = common_serializers.ContentSerializer(allow_null=True, required=False)
     channel = serializers.UUIDField(allow_null=True, required=False)
+    tags = serializers.SerializerMethodField()
 
     class Meta:
         model = models.Artist
@@ -157,63 +160,26 @@ class SimpleArtistSerializer(serializers.ModelSerializer):
             "is_local",
             "content_category",
             "description",
-            "attachment_cover",
+            "cover",
             "channel",
             "attributed_to",
+            "tags",
         )
 
+    @extend_schema_field({"type": "array", "items": {"type": "string"}})
+    def get_tags(self, obj):
+        tagged_items = getattr(obj, "_prefetched_tagged_items", [])
+        return [ti.tag.name for ti in tagged_items]
+
 
 class ArtistCreditSerializer(serializers.ModelSerializer):
-    artist = SimpleArtistSerializer()
+    artist = ArtistSerializer()
 
     class Meta:
         model = models.ArtistCredit
         fields = ["artist", "credit", "joinphrase", "index"]
 
 
-class AlbumSerializer(OptionalDescriptionMixin, serializers.Serializer):
-    artist_credit = ArtistCreditSerializer(many=True)
-    cover = CoverField(allow_null=True)
-    is_playable = serializers.SerializerMethodField()
-    tags = serializers.SerializerMethodField()
-    tracks_count = serializers.SerializerMethodField()
-    attributed_to = APIActorSerializer()
-    id = serializers.IntegerField()
-    fid = serializers.URLField()
-    mbid = serializers.UUIDField()
-    title = serializers.CharField()
-    release_date = serializers.DateField()
-    creation_date = serializers.DateTimeField()
-    is_local = serializers.BooleanField()
-    duration = serializers.SerializerMethodField(read_only=True)
-
-    def get_tracks_count(self, o) -> int:
-        return len(o.tracks.all())
-
-    def get_is_playable(self, obj) -> bool:
-        try:
-            return any(
-                [
-                    bool(getattr(t, "is_playable_by_actor", None))
-                    for t in obj.tracks.all()
-                ]
-            )
-        except AttributeError:
-            return None
-
-    @extend_schema_field({"type": "array", "items": {"type": "string"}})
-    def get_tags(self, obj):
-        tagged_items = getattr(obj, "_prefetched_tagged_items", [])
-        return [ti.tag.name for ti in tagged_items]
-
-    def get_duration(self, obj) -> int:
-        try:
-            return obj.duration
-        except AttributeError:
-            # no annotation?
-            return 0
-
-
 class TrackAlbumSerializer(serializers.ModelSerializer):
     artist_credit = ArtistCreditSerializer(many=True)
     cover = CoverField(allow_null=True)
@@ -275,7 +241,7 @@ class TrackSerializer(OptionalDescriptionMixin, serializers.Serializer):
     listen_url = serializers.SerializerMethodField()
     tags = serializers.SerializerMethodField()
     attributed_to = APIActorSerializer(allow_null=True)
-
+    description = common_serializers.ContentSerializer(allow_null=True, required=False)
     id = serializers.IntegerField()
     fid = serializers.URLField()
     mbid = serializers.UUIDField()
@@ -317,6 +283,51 @@ class TrackSerializer(OptionalDescriptionMixin, serializers.Serializer):
         return bool(getattr(obj, "playable_uploads", []))
 
 
+class AlbumSerializer(OptionalDescriptionMixin, serializers.Serializer):
+    artist_credit = ArtistCreditSerializer(many=True)
+    cover = CoverField(allow_null=True)
+    is_playable = serializers.SerializerMethodField()
+    tags = serializers.SerializerMethodField()
+    tracks_count = serializers.SerializerMethodField()
+    attributed_to = APIActorSerializer()
+    id = serializers.IntegerField()
+    fid = serializers.URLField()
+    mbid = serializers.UUIDField()
+    title = serializers.CharField()
+    release_date = serializers.DateField()
+    creation_date = serializers.DateTimeField()
+    is_local = serializers.BooleanField()
+    duration = serializers.SerializerMethodField(read_only=True)
+    tracks = TrackSerializer(many=True, allow_null=True)
+    description = common_serializers.ContentSerializer(allow_null=True, required=False)
+
+    def get_tracks_count(self, o) -> int:
+        return len(o.tracks.all())
+
+    def get_is_playable(self, obj) -> bool:
+        try:
+            return any(
+                [
+                    bool(getattr(t, "is_playable_by_actor", None))
+                    for t in obj.tracks.all()
+                ]
+            )
+        except AttributeError:
+            return None
+
+    @extend_schema_field({"type": "array", "items": {"type": "string"}})
+    def get_tags(self, obj):
+        tagged_items = getattr(obj, "_prefetched_tagged_items", [])
+        return [ti.tag.name for ti in tagged_items]
+
+    def get_duration(self, obj) -> int:
+        try:
+            return obj.duration
+        except AttributeError:
+            # no annotation?
+            return 0
+
+
 @common_serializers.track_fields_for_update("name", "description", "privacy_level")
 class LibraryForOwnerSerializer(serializers.ModelSerializer):
     uploads_count = serializers.SerializerMethodField()
@@ -363,6 +374,9 @@ class UploadSerializer(serializers.ModelSerializer):
         required=False,
         filters=lambda context: {"actor": context["user"].actor},
     )
+    privacy_level = serializers.ChoiceField(
+        choices=models.LIBRARY_PRIVACY_LEVEL_CHOICES, required=False
+    )
     channel = common_serializers.RelatedField(
         "uuid",
         ChannelSerializer(),
@@ -386,6 +400,7 @@ class UploadSerializer(serializers.ModelSerializer):
             "size",
             "import_date",
             "import_status",
+            "privacy_level",
         ]
 
         read_only_fields = [
@@ -450,6 +465,7 @@ class ImportMetadataSerializer(serializers.Serializer):
     )
 
 
+@extend_schema_field(ImportMetadataSerializer)
 class ImportMetadataField(serializers.JSONField):
     def to_internal_value(self, v):
         v = super().to_internal_value(v)
@@ -486,6 +502,7 @@ class UploadForOwnerSerializer(UploadSerializer):
         r = super().to_representation(obj)
         if "audio_file" in r:
             del r["audio_file"]
+        r["privacy_level"] = obj.library.privacy_level
         return r
 
     def validate(self, validated_data):
@@ -540,10 +557,14 @@ class UploadBulkUpdateSerializer(serializers.Serializer):
             raise serializers.ValidationError(
                 f"Upload with uuid {data['uuid']} does not exist"
             )
+        lib = upload.library.actor.libraries.filter(
+            privacy_level=data["privacy_level"], name=data["privacy_level"]
+        ).exclude(playlist__isnull=False)
 
-        upload.library = upload.library.actor.libraries.get(
-            privacy_level=data["privacy_level"]
-        )
+        if len(lib) == 1:
+            upload.library = lib[0]
+        else:
+            raise serializers.ValidationError("Built-in library not found or too many")
         return upload
 
 
@@ -884,13 +905,17 @@ class FSImportSerializer(serializers.Serializer):
     prune = serializers.BooleanField(required=False, default=True)
     outbox = serializers.BooleanField(required=False, default=False)
     broadcast = serializers.BooleanField(required=False, default=False)
+    replace = serializers.BooleanField(required=False, default=False)
     batch_size = serializers.IntegerField(required=False, default=1000)
     verbosity = serializers.IntegerField(required=False, default=1)
 
     def validate_path(self, value):
         try:
             utils.browse_dir(settings.MUSIC_DIRECTORY_PATH, value)
-        except (NotADirectoryError, FileNotFoundError, ValueError):
+        except NotADirectoryError:
+            if not os.path.isfile(pathlib.Path(settings.MUSIC_DIRECTORY_PATH) / value):
+                raise serializers.ValidationError("Invalid path")
+        except (FileNotFoundError, ValueError):
             raise serializers.ValidationError("Invalid path")
 
         return value

api/funkwhale_api/music/tasks.py

@@ -1209,6 +1209,7 @@ def fs_import(
     prune=True,
     outbox=False,
     broadcast=False,
+    replace=False,
     batch_size=1000,
     verbosity=1,
 ):
@@ -1229,7 +1230,7 @@ def fs_import(
         "batch_size": batch_size,
         "async_": False,
         "prune": prune,
-        "replace": False,
+        "replace": replace,
         "verbosity": verbosity,
         "exit_on_failure": False,
         "outbox": outbox,

api/funkwhale_api/music/views.py

@@ -386,6 +386,7 @@ class LibraryViewSet(
                 prune=serializer.validated_data["prune"],
                 outbox=serializer.validated_data["outbox"],
                 broadcast=serializer.validated_data["broadcast"],
+                replace=serializer.validated_data["replace"],
                 batch_size=serializer.validated_data["batch_size"],
                 verbosity=serializer.validated_data["verbosity"],
             )
@@ -798,6 +799,9 @@ class UploadViewSet(
             cover_data["content"] = base64.b64encode(cover_data["content"])
         return Response(payload, status=200)
 
+    @extend_schema(
+        request=serializers.UploadBulkUpdateSerializer(many=True),
+    )
     @action(detail=False, methods=["patch"])
     def bulk_update(self, request, *args, **kwargs):
         """
@@ -811,7 +815,9 @@ class UploadViewSet(
         models.Upload.objects.bulk_update(serializer.validated_data, ["library"])
 
         return Response(
-            serializers.UploadForOwnerSerializer(serializer.validated_data).data,
+            serializers.UploadForOwnerSerializer(
+                serializer.validated_data, many=True
+            ).data,
             status=200,
         )
 

api/funkwhale_api/playlists/admin.py

@@ -15,3 +15,22 @@ class PlaylistTrackAdmin(admin.ModelAdmin):
     list_display = ["playlist", "track", "index"]
     search_fields = ["track__name", "playlist__name"]
     list_select_related = True
+
+
+@admin.register(models.PlaylistScan)
+class LibraryScanAdmin(admin.ModelAdmin):
+    list_display = [
+        "id",
+        "playlist",
+        "actor",
+        "status",
+        "creation_date",
+        "modification_date",
+        "status",
+        "total_files",
+        "processed_files",
+        "errored_files",
+    ]
+    list_select_related = True
+    search_fields = ["actor__username", "playlist__name"]
+    list_filter = ["status"]

api/funkwhale_api/playlists/factories.py

@@ -3,7 +3,7 @@ from django.conf import settings
 
 from funkwhale_api.factories import NoUpdateOnCreate, registry
 from funkwhale_api.federation import models
-from funkwhale_api.federation.factories import ActorFactory
+from funkwhale_api.federation.factories import ActorFactory, MusicLibraryFactory
 from funkwhale_api.music.factories import TrackFactory
 
 
@@ -13,6 +13,7 @@ class PlaylistFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     actor = factory.SubFactory(ActorFactory)
     fid = factory.Faker("federation_url")
     uuid = factory.Faker("uuid4")
+    library = factory.SubFactory(MusicLibraryFactory)
 
     class Meta:
         model = "playlists.Playlist"

api/funkwhale_api/playlists/migrations/0005_remove_playlist_user_playlist_actor.py

@@ -25,7 +25,7 @@ def gen_uuid(apps, schema_editor):
             unique_uuid = uuid.uuid4()
 
         fid = utils.full_url(
-            reverse("federation:music:playlist-detail", kwargs={"uuid": unique_uuid})
+            reverse("federation:music:playlists-detail", kwargs={"uuid": unique_uuid})
         )
         row.uuid = unique_uuid
         row.fid = fid
@@ -42,7 +42,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name="playlist",
             name="fid",
-            field=models.URLField(max_length=500            ),
+            field=models.URLField(max_length=500, null=True),
         ),
         migrations.AddField(
             model_name="playlist",
@@ -63,8 +63,13 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name="playlist",
             name="fid",
-            field=models.URLField(max_length=500, unique=True, db_index=True,
-            ),),
+            field=models.URLField(
+                max_length=500,
+                unique=True,
+                db_index=True,
+                null=False,
+            ),
+        ),
         migrations.AddField(
             model_name="playlist",
             name="actor",

api/funkwhale_api/playlists/migrations/0006_playlisttrack_fid_playlisttrack_url_and_more.py

@@ -9,14 +9,14 @@ from funkwhale_api.federation import utils
 from django.urls import reverse
 
 def gen_uuid(apps, schema_editor):
-    MyModel = apps.get_model("playlists", "Playlist")
+    MyModel = apps.get_model("playlists", "PlaylistTrack")
     for row in MyModel.objects.all():
         unique_uuid = uuid.uuid4()
         while MyModel.objects.filter(uuid=unique_uuid).exists():
             unique_uuid = uuid.uuid4()
 
         fid = utils.full_url(
-            reverse("federation:music:playlist-detail", kwargs={"uuid": unique_uuid})
+            reverse("federation:music:playlists-detail", kwargs={"uuid": unique_uuid})
         )
         row.uuid = unique_uuid
         row.fid = fid
@@ -38,8 +38,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name="playlisttrack",
             name="fid",
-            field=models.URLField(max_length=500
-            ),
+            field=models.URLField(max_length=500, null=True),
         ),
         migrations.AddField(
             model_name="playlisttrack",
@@ -48,7 +47,7 @@ class Migration(migrations.Migration):
         ),
         migrations.RunPython(gen_uuid, reverse_code=migrations.RunPython.noop),
         migrations.AlterField(
-            model_name="playlist",
+            model_name="playlisttrack",
             name="uuid",
             field=models.UUIDField(default=uuid.uuid4, null=False, unique=True),
         ),
@@ -56,6 +55,7 @@ class Migration(migrations.Migration):
             model_name="playlisttrack",
             name="fid",
             field=models.URLField(
-                db_index=True, max_length=500, unique=True
-            ),),
+                db_index=True, max_length=500, unique=True, null=False
+            ),
+        ),
     ]

api/funkwhale_api/playlists/migrations/0009_playlist_library.py

+import django.db.models.deletion
+from django.db import migrations, models, transaction
+from funkwhale_api.federation import utils as federation_utils
+from django.urls import reverse
+import uuid
+from django.conf import settings
+
+
+def add_uploads_to_pl_library(playlist, library):
+    for plt in playlist.playlist_tracks.all():
+        for upload in plt.track.uploads.filter(library__actor=playlist.actor):
+            library.uploads.add(upload)
+
+
+@transaction.atomic
+def create_playlist_libraries(apps, schema_editor):
+    Playlist = apps.get_model("playlists", "Playlist")
+    Library = apps.get_model("music", "Library")
+    Actor = apps.get_model("federation", "Actor")
+    playlist_with_lib_count = 0
+
+    for playlist in Playlist.objects.all():
+        if not federation_utils.is_local(playlist.actor.fid):
+            continue
+        library = playlist.library
+        if not library:
+            try:
+                # we don't want to get_or_create in case it's a channel lib
+                library = Library.objects.create(
+                    name="playlist_" + playlist.name,
+                    privacy_level="me",
+                    actor=playlist.actor,
+                    uuid=(new_uuid := uuid.uuid4()),
+                    fid=federation_utils.full_url(
+                        reverse(
+                            "federation:music:libraries-detail",
+                            kwargs={"uuid": new_uuid},
+                        )
+                    ),
+                )
+                library.save()
+                playlist.library = library
+                playlist.save()
+                with transaction.atomic():
+                    add_uploads_to_pl_library(playlist, library)
+            except Exception as e:
+                print(
+                    f"An error occurred during playlist.library creation, raising since we want\
+                      to enforce one lib per playlist"
+                )
+                raise e
+        playlist_with_lib_count = playlist_with_lib_count + 1
+        local_actors = Actor.objects.filter(domain_id=settings.FEDERATION_HOSTNAME)
+
+        if (
+            Library.objects.filter(
+                playlist__isnull=False, actor__in=local_actors
+            ).count()
+            != playlist_with_lib_count
+        ):
+            raise Exception(
+                "Should have the same amount of local playlist and libraries with playlist"
+            )
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("playlists", "0008_playlist_library_drop"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="playlist",
+            name="library",
+            field=models.OneToOneField(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="playlist",
+                to="music.library",
+            ),
+        ),
+        migrations.RunPython(
+            create_playlist_libraries, reverse_code=migrations.RunPython.noop
+        ),
+    ]

api/funkwhale_api/playlists/models.py

@@ -88,12 +88,19 @@ class Playlist(federation_models.FederationMixin):
     description = models.TextField(max_length=5000, null=True, blank=True)
     objects = PlaylistQuerySet.as_manager()
     federation_namespace = "playlists"
+    library = models.OneToOneField(
+        "music.Library",
+        null=True,
+        blank=True,
+        on_delete=models.CASCADE,
+        related_name="playlist",
+    )
 
     def __str__(self):
         return self.name
 
     def get_absolute_url(self):
-        return f"/library/playlists/{self.pk}"
+        return f"/library/playlists/{self.uuid}"
 
     def get_federation_id(self):
         if self.fid:
@@ -109,6 +116,19 @@ class Playlist(federation_models.FederationMixin):
         if not self.pk and not self.fid:
             self.fid = self.get_federation_id()
 
+        if not self.pk and not self.library_id:
+            self.library = music_models.Library.objects.create(
+                actor=self.actor,
+                name="playlist_" + self.name,
+                privacy_level="me",
+                uuid=(new_uuid := uuid.uuid4()),
+                fid=federation_utils.full_url(
+                    reverse(
+                        "federation:music:libraries-detail", kwargs={"uuid": new_uuid}
+                    ),
+                ),
+            )
+
         return super().save(**kwargs)
 
     @transaction.atomic
@@ -232,7 +252,7 @@ class Playlist(federation_models.FederationMixin):
         latest_scan = (
             self.scans.exclude(status="errored").order_by("-creation_date").first()
         )
-        delay_between_scans = datetime.timedelta(seconds=3600 * 24)
+        delay_between_scans = datetime.timedelta(seconds=1)
         now = timezone.now()
         if (
             not force
@@ -287,7 +307,7 @@ class PlaylistTrackQuerySet(models.QuerySet, common_models.LocalFromFidQuerySet)
         return self.annotate(is_playable_by_actor=subquery)
 
     def playable_by(self, actor, include=True):
-        tracks = music_models.Track.objects.playable_by(actor, include)
+        tracks = music_models.Track.objects.playable_by(actor)
         if include:
             return self.filter(track__pk__in=tracks).distinct()
         else:
@@ -345,6 +365,9 @@ class PlaylistTrack(federation_models.FederationMixin):
 
         return super().save(**kwargs)
 
+    def get_absolute_url(self):
+        return f"/library/tracks/{self.track.pk}"
+
 
 class PlaylistScan(models.Model):
     actor = models.ForeignKey(

api/funkwhale_api/playlists/renderers.py

@@ -15,7 +15,7 @@ class PlaylistXspfRenderer(renderers.BaseRenderer):
         if isinstance(data, bytes):
             return data
 
-        fw_playlist = Playlist.objects.get(id=data["id"])
+        fw_playlist = Playlist.objects.get(uuid=data["uuid"])
         plt_tracks = fw_playlist.playlist_tracks.prefetch_related("track")
         top = Element("playlist", version="1", xmlns="http://xspf.org/ns/0/")
         title_xspf = SubElement(top, "title")

api/funkwhale_api/playlists/serializers.py

@@ -34,11 +34,14 @@ class PlaylistSerializer(serializers.ModelSerializer):
     album_covers = serializers.SerializerMethodField(read_only=True)
     is_playable = serializers.SerializerMethodField()
     actor = APIActorSerializer(read_only=True)
+    library = serializers.SerializerMethodField()
+    library_followed = serializers.SerializerMethodField()
 
     class Meta:
         model = models.Playlist
         fields = (
-            "id",
+            "uuid",
+            "fid",
             "name",
             "actor",
             "modification_date",
@@ -49,8 +52,37 @@ class PlaylistSerializer(serializers.ModelSerializer):
             "duration",
             "is_playable",
             "actor",
+            "description",
+            "library",
+            "library_followed",
         )
-        read_only_fields = ["id", "modification_date", "creation_date"]
+        read_only_fields = ["uuid", "fid", "modification_date", "creation_date"]
+
+    @extend_schema_field(OpenApiTypes.URI)
+    def get_library(self, obj):
+        if obj.library:
+            return obj.library.fid
+        else:
+            return None
+
+    @extend_schema_field(OpenApiTypes.BOOL)
+    def get_library_followed(self, obj):
+        if (
+            self.context.get("request", False)
+            and hasattr(self.context["request"], "user")
+            and hasattr(self.context["request"].user, "actor")
+        ):
+            actor = self.context["request"].user.actor
+            lib_qs = obj.library.received_follows.filter(actor=actor)
+
+            if lib_qs.exists():
+                if lib_qs[0].approved is None:
+                    return False
+                else:
+                    return lib_qs[0].approved
+            else:
+                return None
+        return None
 
     @extend_schema_field(OpenApiTypes.BOOL)
     def get_is_playable(self, obj):

api/funkwhale_api/playlists/tasks.py

+import logging
+
 import requests
 from django.db.models import F
 from django.utils import timezone
@@ -9,6 +11,8 @@ from funkwhale_api.taskapp import celery
 
 from . import models
 
+logger = logging.getLogger(__name__)
+
 
 def get_playlist_data(playlist_url, actor):
     auth = signing.get_auth(actor.private_key, actor.private_key_id)
@@ -24,7 +28,11 @@ def get_playlist_data(playlist_url, actor):
     if scode == 401:
         return {"errors": ["This playlist requires authentication"]}
     elif scode == 403:
-        return {"errors": ["Permission denied while scanning playlist"]}
+        return {
+            "errors": [
+                f"Permission denied while scanning playlist. Error : {scode}. PLaylist url = {playlist_url}"
+            ]
+        }
     elif scode >= 400:
         return {"errors": [f"Error {scode} while fetching the playlist"]}
     serializer = serializers.PlaylistCollectionSerializer(data=response.json())
@@ -47,6 +55,7 @@ def get_playlist_page(playlist, page_url, actor):
         context={
             "playlist": playlist,
             "item_serializer": serializers.PlaylistTrackSerializer,
+            "conf": {"library": playlist.library},
         },
     )
     serializer.is_valid(raise_exception=True)
@@ -60,6 +69,7 @@ def get_playlist_page(playlist, page_url, actor):
 )
 def start_playlist_scan(playlist_scan):
     playlist_scan.playlist.playlist_tracks.all().delete()
+
     try:
         data = get_playlist_data(playlist_scan.playlist.fid, actor=playlist_scan.actor)
     except Exception:
@@ -90,13 +100,30 @@ def start_playlist_scan(playlist_scan):
 )
 def scan_playlist_page(playlist_scan, page_url):
     data = get_playlist_page(playlist_scan.playlist, page_url, playlist_scan.actor)
-    tracks = []
+    plts = []
     for item_serializer in data["items"]:
-        print(" item_serializer is " + str(item_serializer))
-        track = item_serializer.save(playlist=playlist_scan.playlist.fid)
-        tracks.append(track)
+        try:
+            plt = item_serializer.save(playlist=playlist_scan.playlist.fid)
+            # we get any upload owned by the playlist.actor and add a m2m with playlist_libraries
+            upload_qs = plt.track.uploads.filter(
+                library__actor=playlist_scan.playlist.actor
+            )
+            if not upload_qs:
+                logger.debug(
+                    f"Could not find a upload for the playlist track {plt.track.title}. Probably the \
+                    playlist.library library_scan failed or was not launched by inbox_update_playlist ?"
+                )
+            else:
+                upload_qs[0].playlist_libraries.add(playlist_scan.playlist.library)
+                logger.debug(f"Added {plt.track.title} to playlist library")
+            plts.append(plt)
+        except Exception as e:
+            logger.info(
+                f"Error while saving track to playlist {playlist_scan.playlist}: {e}"
+            )
+            continue
 
-    playlist_scan.processed_files = F("processed_files") + len(tracks)
+    playlist_scan.processed_files = F("processed_files") + len(plts)
     playlist_scan.modification_date = timezone.now()
     update_fields = ["modification_date", "processed_files"]
 

api/funkwhale_api/playlists/views.py

 import logging
+from itertools import chain
 
+from django.conf import settings
 from django.db import transaction
 from django.db.models import Count
 from drf_spectacular.utils import extend_schema
 from rest_framework import exceptions, mixins, status, viewsets
 from rest_framework.decorators import action
+from rest_framework.pagination import PageNumberPagination
 from rest_framework.parsers import FormParser, JSONParser, MultiPartParser
 from rest_framework.renderers import JSONRenderer
 from rest_framework.response import Response
 
+from config import plugins
 from funkwhale_api.common import fields, permissions
 from funkwhale_api.federation import routes
 from funkwhale_api.music import models as music_models
@@ -29,6 +33,7 @@ class PlaylistViewSet(
     mixins.ListModelMixin,
     viewsets.GenericViewSet,
 ):
+    lookup_field = "uuid"
     serializer_class = serializers.PlaylistSerializer
     queryset = (
         models.Playlist.objects.all()
@@ -125,12 +130,25 @@ class PlaylistViewSet(
     @action(methods=["get"], detail=True)
     def tracks(self, request, *args, **kwargs):
         playlist = self.get_object()
-        plts = playlist.playlist_tracks.all().for_nested_serialization(
-            music_utils.get_actor_from_request(request)
+        actor = music_utils.get_actor_from_request(request)
+        plts = playlist.playlist_tracks.all().for_nested_serialization(actor)
+        for plt in plts.playable_by(actor, include=False)[
+            : settings.THIRD_PARTY_UPLOAD_MAX_UPLOADS
+        ]:
+            plugins.trigger_hook(
+                plugins.TRIGGER_THIRD_PARTY_UPLOAD,
+                track=plt.track,
+                actor=actor,
             )
-        serializer = serializers.PlaylistTrackSerializer(plts, many=True)
-        data = {"count": len(plts), "results": serializer.data}
-        return Response(data, status=200)
+
+        # Apply pagination
+        paginator = PageNumberPagination()
+        paginator.page_size = 50  # Set the page size (number of items per page)
+        paginated_plts = paginator.paginate_queryset(plts, request)
+
+        # Serialize the paginated data
+        serializer = serializers.PlaylistTrackSerializer(paginated_plts, many=True)
+        return paginator.get_paginated_response(serializer.data)
 
     @extend_schema(
         operation_id="add_to_playlist", request=serializers.PlaylistAddManySerializer
@@ -157,6 +175,7 @@ class PlaylistViewSet(
         )
         serializer = serializers.PlaylistTrackSerializer(plts, many=True)
         data = {"count": len(plts), "results": serializer.data}
+        update_playlist_library_uploads(playlist, plts)
         playlist.schedule_scan(playlist.actor, force=True)
         return Response(data, status=201)
 
@@ -167,7 +186,8 @@ class PlaylistViewSet(
         playlist = self.get_object()
         playlist.playlist_tracks.all().delete()
         playlist.save(update_fields=["modification_date"])
-        playlist.schedule_scan(playlist.actor)
+        playlist.library.uploads.filter().delete()
+        playlist.schedule_scan(playlist.actor, force=True)
         return Response(status=204)
 
     def get_queryset(self):
@@ -200,6 +220,8 @@ class PlaylistViewSet(
             plt = playlist.playlist_tracks.by_index(index)
         except models.PlaylistTrack.DoesNotExist:
             return Response(status=404)
+        for upload in plt.track.uploads.filter(playlist_libraries=playlist.library):
+            upload.playlist_libraries.remove(playlist.library)
         plt.delete(update_indexes=True)
         plt.playlist.schedule_scan(playlist.actor)
         return Response(status=204)
@@ -244,7 +266,7 @@ class PlaylistViewSet(
         serializer = music_serializers.AlbumSerializer(releases, many=True)
         return Response(serializer.data, status=200)
 
-    @extend_schema(operation_id="get_playlist_artits")
+    @extend_schema(operation_id="get_playlist_artists")
     @action(methods=["get"], detail=True)
     @transaction.atomic
     def artists(self, request, *args, **kwargs):
@@ -256,5 +278,15 @@ class PlaylistViewSet(
         except models.PlaylistTrack.DoesNotExist:
             return Response(status=404)
         artists = music_models.Artist.objects.filter(pk__in=artists_pks)
-        serializer = music_serializers.SimpleArtistSerializer(artists, many=True)
+        serializer = music_serializers.ArtistSerializer(artists, many=True)
         return Response(serializer.data, status=200)
+
+
+def update_playlist_library_uploads(playlist, plts):
+    uploads = list(
+        chain(
+            *[plt.track.uploads.filter(library__actor=playlist.actor) for plt in plts]
+        )
+    )
+    for upload in uploads:
+        upload.playlist_libraries.add(playlist.library)

api/funkwhale_api/radios/radios_v2.py

@@ -109,7 +109,7 @@ class SessionRadio(SimpleRadio):
         queryset = self.filter_queryset(queryset)
 
         # select a random batch of the qs
-        sliced_queryset = queryset.order_by("?")[:BATCH_SIZE]
+        sliced_queryset = queryset.random(BATCH_SIZE)
         if len(sliced_queryset) <= 0 and not cached_evaluated_radio_tracks:
             raise ValueError("No more radio candidates")
 
@@ -166,7 +166,7 @@ class SessionRadio(SimpleRadio):
 class RandomRadio(SessionRadio):
     def get_queryset(self, **kwargs):
         qs = super().get_queryset(**kwargs)
-        return qs.filter(artist_credit__artist__content_category="music").order_by("?")
+        return qs.filter(artist_credit__artist__content_category="music").random(100)
 
 
 @registry.register(name="random_library")
@@ -179,7 +179,7 @@ class RandomLibraryRadio(SessionRadio):
         query = Q(artist_credit__artist__content_category="music") & Q(
             pk__in=tracks_ids
         )
-        return qs.filter(query).order_by("?")
+        return qs.filter(query).random(100)
 
 
 @registry.register(name="favorites")
@@ -390,7 +390,7 @@ class LessListenedRadio(SessionRadio):
         return (
             qs.filter(artist_credit__artist__content_category="music")
             .exclude(pk__in=listened)
-            .order_by("?")
+            .random(100)
         )
 
 
@@ -411,7 +411,7 @@ class LessListenedLibraryRadio(SessionRadio):
         query = Q(artist_credit__artist__content_category="music") & Q(
             pk__in=tracks_ids
         )
-        return qs.filter(query).exclude(pk__in=listened).order_by("?")
+        return qs.filter(query).exclude(pk__in=listened).random(100)
 
 
 @registry.register(name="actor-content")