api/funkwhale_api/music/tasks.py

@@ -2,6 +2,7 @@ import collections
 import datetime
 import logging
 import os
+import re
 
 from django.conf import settings
 from django.core.cache import cache
@@ -9,25 +10,21 @@ from django.db import transaction
 from django.db.models import F, Q
 from django.dispatch import receiver
 from django.utils import timezone
-
-from musicbrainzngs import ResponseError
+from musicbrainzngs import NetworkError, ResponseError
 from requests.exceptions import RequestException
 
 from funkwhale_api import musicbrainz
 from funkwhale_api.common import channels, preferences
 from funkwhale_api.common import utils as common_utils
-from funkwhale_api.federation import routes
 from funkwhale_api.federation import library as lb
+from funkwhale_api.federation import routes
 from funkwhale_api.federation import utils as federation_utils
 from funkwhale_api.music.management.commands import import_files
 from funkwhale_api.tags import models as tags_models
 from funkwhale_api.tags import tasks as tags_tasks
 from funkwhale_api.taskapp import celery
 
-from . import licenses
-from . import models
-from . import metadata
-from . import signals
+from . import licenses, metadata, models, signals
 
 logger = logging.getLogger(__name__)
 
@@ -70,7 +67,7 @@ def get_cover_from_fs(dir_path):
     if os.path.exists(dir_path):
         for name in FOLDER_IMAGE_NAMES:
             for e, m in IMAGE_TYPES:
-                cover_path = os.path.join(dir_path, "{}.{}".format(name, e))
+                cover_path = os.path.join(dir_path, f"{name}.{e}")
                 if not os.path.exists(cover_path):
                     logger.debug("Cover %s does not exists", cover_path)
                     continue
@@ -79,6 +76,19 @@ def get_cover_from_fs(dir_path):
                     return {"mimetype": m, "content": c.read()}
 
 
+@celery.app.task(name="music.library.schedule_remote_scan")
+def schedule_scan_for_all_remote_libraries():
+    from funkwhale_api.federation import actors
+
+    libraries = models.Library.objects.all().prefetch_related()
+    actor = actors.get_service_actor()
+
+    for library in libraries:
+        if library.actor.is_local:
+            continue
+        library.schedule_scan(actor)
+
+
 @celery.app.task(name="music.start_library_scan")
 @celery.require_instance(
     models.LibraryScan.objects.select_related().filter(status="pending"), "library_scan"
@@ -90,6 +100,10 @@ def start_library_scan(library_scan):
         library_scan.status = "errored"
         library_scan.save(update_fields=["status", "modification_date"])
         raise
+    if "errors" in data.keys():
+        library_scan.status = "errored"
+        library_scan.save(update_fields=["status", "modification_date"])
+        raise Exception("Error from remote server : " + str(data))
     library_scan.modification_date = timezone.now()
     library_scan.status = "scanning"
     library_scan.total_files = data["totalItems"]
@@ -162,7 +176,7 @@ def fail_import(upload, error_code, detail=None, **fields):
         upload.import_metadata, "funkwhale", "config", "broadcast", default=True
     )
     if broadcast:
-        signals.upload_import_status_updated.send(
+        signals.upload_import_status_updated.send_robust(
             old_status=old_status,
             new_status=upload.import_status,
             upload=upload,
@@ -173,7 +187,8 @@ def fail_import(upload, error_code, detail=None, **fields):
 @celery.app.task(name="music.process_upload")
 @celery.require_instance(
     models.Upload.objects.filter(import_status="pending").select_related(
-        "library__actor__user", "library__channel__artist",
+        "library__actor__user",
+        "library__channel__artist",
     ),
     "upload",
 )
@@ -212,7 +227,7 @@ def process_upload(upload, update_denormalization=True):
         forced_values["artist"] = upload.library.channel.artist
 
     old_status = upload.import_status
-    additional_data = {"upload_source": upload.source}
+    upload_source = {"upload_source": upload.source}
 
     if use_file_metadata:
         audio_file = upload.get_audio_file()
@@ -233,13 +248,22 @@ def process_upload(upload, update_denormalization=True):
             return fail_import(
                 upload, "invalid_metadata", detail=detail, file_metadata=metadata_dump
             )
+        check_mbid = preferences.get("music__only_allow_musicbrainz_tagged_files")
+        if check_mbid and not serializer.validated_data.get("mbid"):
+            return fail_import(
+                upload,
+                "Only content tagged with a MusicBrainz ID is permitted on this pod.",
+                detail="You can tag your files with MusicBrainz Picard",
+            )
 
         final_metadata = collections.ChainMap(
-            additional_data, serializer.validated_data, internal_config
+            upload_source, serializer.validated_data, internal_config
         )
     else:
         final_metadata = collections.ChainMap(
-            additional_data, forced_values, internal_config,
+            upload_source,
+            forced_values,
+            internal_config,
         )
     try:
         track = get_track_from_import_metadata(
@@ -247,8 +271,8 @@ def process_upload(upload, update_denormalization=True):
         )
     except UploadImportError as e:
         return fail_import(upload, e.code)
-    except Exception:
-        fail_import(upload, "unknown_error")
+    except Exception as e:
+        fail_import(upload, "unknown_error", e)
         raise
 
     broadcast = getter(
@@ -264,14 +288,16 @@ def process_upload(upload, update_denormalization=True):
         upload.import_status = "skipped"
         upload.import_details = {
             "code": "already_imported_in_owned_libraries",
-            "duplicates": list(owned_duplicates),
+            # In order to avoid exponential growth of the database, we only
+            # reference the first known upload which gets duplicated
+            "duplicates": owned_duplicates[0],
         }
         upload.import_date = timezone.now()
         upload.save(
             update_fields=["import_details", "import_status", "import_date", "track"]
         )
         if broadcast:
-            signals.upload_import_status_updated.send(
+            signals.upload_import_status_updated.send_robust(
                 old_status=old_status,
                 new_status=upload.import_status,
                 upload=upload,
@@ -310,19 +336,24 @@ def process_upload(upload, update_denormalization=True):
     # update album cover, if needed
     if track.album and not track.album.attachment_cover:
         populate_album_cover(
-            track.album, source=final_metadata.get("upload_source"),
+            track.album,
+            source=final_metadata.get("upload_source"),
         )
 
     if broadcast:
-        signals.upload_import_status_updated.send(
+        signals.upload_import_status_updated.send_robust(
             old_status=old_status,
             new_status=upload.import_status,
             upload=upload,
             sender=None,
         )
-    dispatch_outbox = getter(
+    dispatch_outbox = (
+        False
+        if upload.library.privacy_level in ["me", "instance"]
+        else getter(
             internal_config, "funkwhale", "config", "dispatch_outbox", default=True
         )
+    )
     if dispatch_outbox:
         routes.outbox.dispatch(
             {"type": "Create", "object": {"type": "Audio"}}, context={"upload": upload}
@@ -353,9 +384,9 @@ def federation_audio_track_to_metadata(payload, references):
         "copyright": payload.get("copyright"),
         "description": payload.get("description"),
         "attributed_to": references.get(payload.get("attributedTo")),
-        "mbid": str(payload.get("musicbrainzId"))
-        if payload.get("musicbrainzId")
-        else None,
+        "mbid": (
+            str(payload.get("musicbrainzId")) if payload.get("musicbrainzId") else None
+        ),
         "cover_data": get_cover(payload, "image"),
         "album": {
             "title": payload["album"]["name"],
@@ -363,44 +394,65 @@ def federation_audio_track_to_metadata(payload, references):
             "fid": payload["album"]["id"],
             "description": payload["album"].get("description"),
             "attributed_to": references.get(payload["album"].get("attributedTo")),
-            "mbid": str(payload["album"]["musicbrainzId"])
+            "mbid": (
+                str(payload["album"]["musicbrainzId"])
                 if payload["album"].get("musicbrainzId")
-            else None,
+                else None
+            ),
             "cover_data": get_cover(payload["album"], "image"),
             "release_date": payload["album"].get("released"),
             "tags": [t["name"] for t in payload["album"].get("tags", []) or []],
-            "artists": [
+            "artist_credit": [
                 {
-                    "fid": a["id"],
-                    "name": a["name"],
-                    "fdate": a["published"],
-                    "cover_data": get_cover(a, "image"),
-                    "description": a.get("description"),
-                    "attributed_to": references.get(a.get("attributedTo")),
-                    "mbid": str(a["musicbrainzId"]) if a.get("musicbrainzId") else None,
-                    "tags": [t["name"] for t in a.get("tags", []) or []],
+                    "artist": {
+                        "fid": a["artist"]["id"],
+                        "name": a["artist"]["name"],
+                        "fdate": a["artist"]["published"],
+                        "cover_data": get_cover(a["artist"], "image"),
+                        "description": a["artist"].get("description"),
+                        "attributed_to": references.get(
+                            a["artist"].get("attributedTo")
+                        ),
+                        "mbid": (
+                            str(a["artist"]["musicbrainzId"])
+                            if a["artist"].get("musicbrainzId")
+                            else None
+                        ),
+                        "tags": [t["name"] for t in a["artist"].get("tags", []) or []],
+                    },
+                    "joinphrase": (a["joinphrase"] if "joinphrase" in a else ""),
+                    "credit": a["credit"],
                 }
-                for a in payload["album"]["artists"]
+                for a in payload["album"]["artist_credit"]
             ],
         },
-        "artists": [
+        "artist_credit": [
             {
-                "fid": a["id"],
-                "name": a["name"],
-                "fdate": a["published"],
-                "description": a.get("description"),
-                "attributed_to": references.get(a.get("attributedTo")),
-                "mbid": str(a["musicbrainzId"]) if a.get("musicbrainzId") else None,
-                "tags": [t["name"] for t in a.get("tags", []) or []],
-                "cover_data": get_cover(a, "image"),
+                "artist": {
+                    "fid": a["artist"]["id"],
+                    "name": a["artist"]["name"],
+                    "fdate": a["artist"]["published"],
+                    "description": a["artist"].get("description"),
+                    "attributed_to": references.get(a["artist"].get("attributedTo")),
+                    "mbid": (
+                        str(a["artist"]["musicbrainzId"])
+                        if a["artist"].get("musicbrainzId")
+                        else None
+                    ),
+                    "tags": [t["name"] for t in a["artist"].get("tags", []) or []],
+                    "cover_data": get_cover(a["artist"], "image"),
+                },
+                "joinphrase": (a["joinphrase"] if "joinphrase" in a else ""),
+                "credit": a["credit"],
             }
-            for a in payload["artists"]
+            for a in payload["artist_credit"]
         ],
         # federation
         "fid": payload["id"],
         "fdate": payload["published"],
         "tags": [t["name"] for t in payload.get("tags", []) or []],
     }
+
     return new_data
 
 
@@ -408,6 +460,7 @@ def get_owned_duplicates(upload, track):
     """
     Ensure we skip duplicate tracks to avoid wasting user/instance storage
     """
+
     owned_libraries = upload.library.actor.libraries.all()
     return (
         models.Upload.objects.filter(
@@ -415,6 +468,7 @@ def get_owned_duplicates(upload, track):
         )
         .exclude(pk=upload.pk)
         .values_list("uuid", flat=True)
+        .order_by("creation_date")
     )
 
 
@@ -425,7 +479,6 @@ def get_best_candidate_or_create(model, query, defaults, sort_fields):
     """
     candidates = model.objects.filter(query)
     if candidates:
-
         return sort_candidates(candidates, sort_fields)[0], False
 
     return model.objects.create(**defaults), True
@@ -465,9 +518,11 @@ def sort_candidates(candidates, important_fields):
 
 @transaction.atomic
 def get_track_from_import_metadata(
-    data, update_cover=False, attributed_to=None, **forced_values
+    data, update_cover=False, attributed_to=None, query_mb=True, **forced_values
 ):
-    track = _get_track(data, attributed_to=attributed_to, **forced_values)
+    track = _get_track(
+        data, attributed_to=attributed_to, query_mb=query_mb, **forced_values
+    )
     if update_cover and track and not track.album.attachment_cover:
         populate_album_cover(track.album, source=data.get("upload_source"))
     return track
@@ -479,9 +534,11 @@ def truncate(v, length):
     return v[:length]
 
 
-def _get_track(data, attributed_to=None, **forced_values):
+def _get_track(data, attributed_to=None, query_mb=True, **forced_values):
+    sync_mb_tag = preferences.get("music__sync_musicbrainz_tags")
     track_uuid = getter(data, "funkwhale", "track", "uuid")
 
+    logger.debug(f"Getting track from import metadata: {data}")
     if track_uuid:
         # easy case, we have a reference to a uuid of a track that
         # already exists in our database
@@ -521,85 +578,84 @@ def _get_track(data, attributed_to=None, **forced_values):
         except IndexError:
             pass
 
-    # get / create artist and album artist
-    artists = getter(data, "artists", default=[])
+    # get / create artist, artist_credit
+    album_artists_credits = None
+    artist_credit_data = getter(data, "artist_credit", default=[])
     if "artist" in forced_values:
         artist = forced_values["artist"]
+        query = Q(artist=artist)
+        defaults = {
+            "artist": artist,
+            "joinphrase": "",
+            "credit": artist.name,
+        }
+        track_artist_credit, created = get_best_candidate_or_create(
+            models.ArtistCredit, query, defaults=defaults, sort_fields=["mbid", "fid"]
+        )
+        track_artists_credits = [track_artist_credit]
     else:
-        artist_data = artists[0]
-        artist = get_artist(
-            artist_data, attributed_to=attributed_to, from_activity_id=from_activity_id
+        mbid = query_mb and (data.get("musicbrainz_id", None) or data.get("mbid", None))
+        try:
+            track_artists_credits = get_or_create_artists_credits_from_musicbrainz(
+                "recording",
+                mbid,
+                attributed_to=attributed_to,
+                from_activity_id=from_activity_id,
             )
-        artist_name = artist.name
+        except (NoMbid, ResponseError, NetworkError):
+            track_artists_credits = (
+                get_or_create_artists_credits_from_artist_credit_metadata(
+                    artist_credit_data,
+                    attributed_to=attributed_to,
+                    from_activity_id=from_activity_id,
+                )
+            )
+
+    # get / create album artist, album artist_credit
     if "album" in forced_values:
         album = forced_values["album"]
+        album_artists_credits = track_artists_credits
     else:
-        if "artist" in forced_values:
-            album_artist = forced_values["artist"]
-        else:
-            album_artists = getter(data, "album", "artists", default=artists) or artists
-            album_artist_data = album_artists[0]
-            album_artist_name = truncate(
-                album_artist_data.get("name"), models.MAX_LENGTHS["ARTIST_NAME"]
-            )
-            if album_artist_name == artist_name:
-                album_artist = artist
-            else:
-                query = Q(name__iexact=album_artist_name)
-                album_artist_mbid = album_artist_data.get("mbid", None)
-                album_artist_fid = album_artist_data.get("fid", None)
-                if album_artist_mbid:
-                    query |= Q(mbid=album_artist_mbid)
-                if album_artist_fid:
-                    query |= Q(fid=album_artist_fid)
-                defaults = {
-                    "name": album_artist_name,
-                    "mbid": album_artist_mbid,
-                    "fid": album_artist_fid,
-                    "from_activity_id": from_activity_id,
-                    "attributed_to": album_artist_data.get(
-                        "attributed_to", attributed_to
-                    ),
-                }
-                if album_artist_data.get("fdate"):
-                    defaults["creation_date"] = album_artist_data.get("fdate")
-
-                album_artist, created = get_best_candidate_or_create(
-                    models.Artist, query, defaults=defaults, sort_fields=["mbid", "fid"]
+        if album_artists_credits:
+            pass
+        mbid = query_mb and (data.get("musicbrainz_albumid", None) or album_mbid)
+        try:
+            album_artists_credits = get_or_create_artists_credits_from_musicbrainz(
+                "release",
+                mbid,
+                attributed_to=attributed_to,
+                from_activity_id=from_activity_id,
             )
-                if created:
-                    tags_models.add_tags(
-                        album_artist, *album_artist_data.get("tags", [])
+        except (NoMbid, ResponseError, NetworkError):
+            if album_artists := getter(data, "album", "artist_credit", default=None):
+                album_artists_credits = (
+                    get_or_create_artists_credits_from_artist_credit_metadata(
+                        album_artists,
+                        attributed_to=attributed_to,
+                        from_activity_id=from_activity_id,
                     )
-                    common_utils.attach_content(
-                        album_artist,
-                        "description",
-                        album_artist_data.get("description"),
-                    )
-                    common_utils.attach_file(
-                        album_artist,
-                        "attachment_cover",
-                        album_artist_data.get("cover_data"),
                 )
+            else:
+                album_artists_credits = track_artists_credits
 
         # get / create album
         if "album" in data:
             album_data = data["album"]
-            album_title = truncate(
-                album_data["title"], models.MAX_LENGTHS["ALBUM_TITLE"]
-            )
+            album_title = album_data["title"]
             album_fid = album_data.get("fid", None)
 
             if album_mbid:
                 query = Q(mbid=album_mbid)
             else:
-                query = Q(title__iexact=album_title, artist=album_artist)
+                query = Q(
+                    title__iexact=album_title, artist_credit__in=album_artists_credits
+                )
 
             if album_fid:
                 query |= Q(fid=album_fid)
+
             defaults = {
                 "title": album_title,
-                "artist": album_artist,
                 "mbid": album_mbid,
                 "release_date": album_data.get("release_date"),
                 "fid": album_fid,
@@ -612,6 +668,8 @@ def _get_track(data, attributed_to=None, **forced_values):
             album, created = get_best_candidate_or_create(
                 models.Album, query, defaults=defaults, sort_fields=["mbid", "fid"]
             )
+            album.artist_credit.set(album_artists_credits)
+
             if created:
                 tags_models.add_tags(album, *album_data.get("tags", []))
                 common_utils.attach_content(
@@ -620,14 +678,14 @@ def _get_track(data, attributed_to=None, **forced_values):
                 common_utils.attach_file(
                     album, "attachment_cover", album_data.get("cover_data")
                 )
+
+            if sync_mb_tag and album_mbid:
+                tags_tasks.sync_fw_item_tag_with_musicbrainz_tags(album)
+
         else:
             album = None
     # get / create track
-    track_title = (
-        forced_values["title"]
-        if "title" in forced_values
-        else truncate(data["title"], models.MAX_LENGTHS["TRACK_TITLE"])
-    )
+    track_title = forced_values["title"] if "title" in forced_values else data["title"]
     position = (
         forced_values["position"]
         if "position" in forced_values
@@ -646,7 +704,7 @@ def _get_track(data, attributed_to=None, **forced_values):
     copyright = (
         forced_values["copyright"]
         if "copyright" in forced_values
-        else truncate(data.get("copyright"), models.MAX_LENGTHS["COPYRIGHT"])
+        else data.get("copyright")
     )
     description = (
         {"text": forced_values["description"], "content_type": "text/markdown"}
@@ -659,7 +717,7 @@ def _get_track(data, attributed_to=None, **forced_values):
 
     query = Q(
         title__iexact=track_title,
-        artist=artist,
+        artist_credit__in=track_artists_credits,
         album=album,
         position=position,
         disc_number=disc_number,
@@ -672,17 +730,10 @@ def _get_track(data, attributed_to=None, **forced_values):
     if track_fid:
         query |= Q(fid=track_fid)
 
-    if album and len(artists) > 1:
-        # we use the second artist to preserve featuring information
-        artist = artist = get_artist(
-            artists[1], attributed_to=attributed_to, from_activity_id=from_activity_id
-        )
-
     defaults = {
         "title": track_title,
         "album": album,
         "mbid": track_mbid,
-        "artist": artist,
         "position": position,
         "disc_number": disc_number,
         "fid": track_fid,
@@ -706,42 +757,193 @@ def _get_track(data, attributed_to=None, **forced_values):
         common_utils.attach_content(track, "description", description)
         common_utils.attach_file(track, "attachment_cover", cover_data)
 
+    if sync_mb_tag and track_mbid:
+        tags_tasks.sync_fw_item_tag_with_musicbrainz_tags(track)
+
+    track.artist_credit.set(track_artists_credits)
     return track
 
 
-def get_artist(artist_data, attributed_to, from_activity_id):
-    artist_mbid = artist_data.get("mbid", None)
-    artist_fid = artist_data.get("fid", None)
-    artist_name = truncate(artist_data["name"], models.MAX_LENGTHS["ARTIST_NAME"])
+def get_or_create_artist_from_ac(ac_data, attributed_to, from_activity_id):
+    sync_mb_tag = preferences.get("music__sync_musicbrainz_tags")
+    mbid = ac_data.get("artist", {}).get("mbid", None)
+    fid = ac_data.get("artist", {}).get("fid", None)
+    name = ac_data.get("artist", {}).get("name", ac_data.get("credit", None))
+    creation_date = ac_data.get("artist", {}).get("fdate", timezone.now())
+    description = ac_data.get("artist", {}).get("description", None)
+    attributed_to = ac_data.get("artist", {}).get("attributed_to", attributed_to)
+    tags = ac_data.get("artist", {}).get("tags", [])
+    cover = ac_data.get("artist", {}).get("cover_data", None)
 
-    if artist_mbid:
-        query = Q(mbid=artist_mbid)
+    if mbid:
+        query = Q(mbid=mbid)
     else:
-        query = Q(name__iexact=artist_name)
-    if artist_fid:
-        query |= Q(fid=artist_fid)
+        query = Q(name__iexact=name)
+
+    if fid:
+        query |= Q(fid=fid)
+
     defaults = {
-        "name": artist_name,
-        "mbid": artist_mbid,
-        "fid": artist_fid,
+        "name": name,
+        "mbid": mbid,
+        "fid": fid,
         "from_activity_id": from_activity_id,
-        "attributed_to": artist_data.get("attributed_to", attributed_to),
+        "attributed_to": attributed_to,
+        "creation_date": creation_date,
     }
-    if artist_data.get("fdate"):
-        defaults["creation_date"] = artist_data.get("fdate")
+    if ac_data.get("fdate"):
+        defaults["creation_date"] = ac_data.get("fdate")
 
     artist, created = get_best_candidate_or_create(
         models.Artist, query, defaults=defaults, sort_fields=["mbid", "fid"]
     )
     if created:
-        tags_models.add_tags(artist, *artist_data.get("tags", []))
-        common_utils.attach_content(
-            artist, "description", artist_data.get("description")
+        tags_models.add_tags(artist, *tags)
+        common_utils.attach_content(artist, "description", description)
+        common_utils.attach_file(artist, "attachment_cover", cover)
+    if sync_mb_tag and mbid:
+        tags_tasks.sync_fw_item_tag_with_musicbrainz_tags(artist)
+
+    return artist
+
+
+class NoMbid(Exception):
+    pass
+
+
+def get_or_create_artists_credits_from_musicbrainz(
+    mb_obj_type, mbid, attributed_to, from_activity_id
+):
+    if not mbid:
+        raise NoMbid
+
+    try:
+        if mb_obj_type == "release":
+            mb_obj = musicbrainz.api.releases.get(mbid, includes=["artists"])
+        elif mb_obj_type == "recording":
+            mb_obj = musicbrainz.api.recordings.get(mbid, includes=["artists"])
+    except (ResponseError, NetworkError) as e:
+        logger.warning(
+            f"Couldn't get Musicbrainz information for {mb_obj_type} with {mbid} mbid  \
+            because of the following exception : {e}"
         )
-        common_utils.attach_file(
-            artist, "attachment_cover", artist_data.get("cover_data")
+        raise e
+
+    artists_credits = []
+    acs = mb_obj.get("recording", mb_obj)["artist-credit"]
+    logger.debug(f"MusicBrainz responded with : {mb_obj}")
+    for i, ac in enumerate(acs):
+        if isinstance(ac, str):
+            continue
+        artist_name = ac["artist"]["name"]
+        joinphrase = ac["joinphrase"]
+
+        # mb use "name" instead of "credit" and id instead of mbdi
+        credit = ac.get("name", ac.get("credit", artist_name))
+        ac["credit"] = credit
+        ac["artist"]["mbid"] = ac["artist"]["id"]
+
+        artist = get_or_create_artist_from_ac(ac, attributed_to, from_activity_id)
+
+        # artist_credit creation
+        defaults = {
+            "artist": artist,
+            "joinphrase": joinphrase,
+            "credit": credit,
+            "index": i,
+        }
+        query = (
+            Q(artist=artist.pk)
+            & Q(joinphrase=joinphrase)
+            & Q(credit=credit)
+            & Q(index=i)
         )
-    return artist
+
+        artist_credit, created = get_best_candidate_or_create(
+            models.ArtistCredit, query, defaults=defaults, sort_fields=["mbid", "fid"]
+        )
+        artists_credits.append(artist_credit)
+    return artists_credits
+
+
+def parse_credits(artist_string, forced_joinphrase, forced_index, forced_artist=None):
+    """
+    Return a list of parsed artist_credit information from a string like :
+    LoveDiversity featuring Hatingprisons
+    """
+    if not artist_string:
+        return []
+    join_phrase = preferences.get("music__join_phrases")
+    join_phrase_regex = re.compile(rf"({join_phrase})", re.IGNORECASE)
+    split = re.split(join_phrase_regex, artist_string)
+    raw_artists_credits = tuple(zip(split[0::2], split[1::2]))
+
+    artists_credits_tuple = []
+    for index, raw_artist_credit in enumerate(raw_artists_credits):
+        credit = raw_artist_credit[0].strip()
+        if forced_joinphrase:
+            join_phrase = forced_joinphrase
+        else:
+            join_phrase = raw_artist_credit[1]
+            if join_phrase == "( " or join_phrase == ") ":
+                join_phrase = join_phrase.strip()
+
+        artists_credits_tuple.append(
+            (
+                credit,
+                join_phrase,
+                (index if not forced_index else forced_index),
+                forced_artist,
+            )
+        )
+
+    # impar split :
+    if len(split) % 2 != 0 and split[len(split) - 1] != "" and len(split) > 1:
+        artists_credits_tuple.append(
+            (
+                str(split[len(split) - 1]).rstrip(),
+                ("" if not forced_joinphrase else forced_joinphrase),
+                (len(artists_credits_tuple) if not forced_index else forced_index),
+                forced_artist,
+            )
+        )
+
+    # if "name" is empty or didn't split
+    if not raw_artists_credits:
+        credit = forced_artist.name if forced_artist else artist_string
+        artists_credits_tuple.append(
+            (
+                credit,
+                ("" if not forced_joinphrase else forced_joinphrase),
+                (0 if not forced_index else forced_index),
+                forced_artist,
+            )
+        )
+    return artists_credits_tuple
+
+
+def get_or_create_artists_credits_from_artist_credit_metadata(
+    artists_credits_data, attributed_to, from_activity_id
+):
+    artists_credits = []
+    for i, ac in enumerate(artists_credits_data):
+        ac["artist"] = get_or_create_artist_from_ac(ac, attributed_to, from_activity_id)
+        ac["index"] = ac.get("index", i)
+
+        credit = ac.get("credit", ac["artist"].name)
+        query = (
+            Q(artist=ac["artist"])
+            & Q(credit=credit)
+            & Q(joinphrase=ac["joinphrase"])
+            & Q(index=ac.get("index", i))
+        )
+
+        artist_credit, created = get_best_candidate_or_create(
+            models.ArtistCredit, query, ac, ["artist", "credit", "joinphrase"]
+        )
+        artists_credits.append(artist_credit)
+
+    return artists_credits
 
 
 @receiver(signals.upload_import_status_updated)
@@ -752,7 +954,7 @@ def broadcast_import_status_update_to_owner(old_status, new_status, upload, **kw
 
     from . import serializers
 
-    group = "user.{}.imports".format(user.pk)
+    group = f"user.{user.pk}.imports"
     channels.group_send(
         group,
         {
@@ -776,7 +978,7 @@ def clean_transcoding_cache():
     limit = timezone.now() - datetime.timedelta(minutes=delay)
     candidates = (
         models.UploadVersion.objects.filter(
-            (Q(accessed_date__lt=limit) | Q(accessed_date=None))
+            Q(accessed_date__lt=limit) | Q(accessed_date=None)
         )
         .only("audio_file", "id")
         .order_by("id")
@@ -793,15 +995,18 @@ def albums_set_tags_from_tracks(ids=None, dry_run=False):
     if ids is not None:
         qs = qs.filter(pk__in=ids)
     data = tags_tasks.get_tags_from_foreign_key(
-        ids=qs, foreign_key_model=models.Track, foreign_key_attr="album",
+        ids=qs,
+        foreign_key_model=models.Track,
+        foreign_key_attr="albums",
     )
     logger.info("Found automatic tags for %s albums…", len(data))
     if dry_run:
-        logger.info("Running in dry-run mode, not commiting")
+        logger.info("Running in dry-run mode, not committing")
         return
 
     tags_tasks.add_tags_batch(
-        data, model=models.Album,
+        data,
+        model=models.Album,
     )
     return data
 
@@ -815,15 +1020,18 @@ def artists_set_tags_from_tracks(ids=None, dry_run=False):
     if ids is not None:
         qs = qs.filter(pk__in=ids)
     data = tags_tasks.get_tags_from_foreign_key(
-        ids=qs, foreign_key_model=models.Track, foreign_key_attr="artist",
+        ids=qs,
+        foreign_key_model=models.Track,
+        foreign_key_attr="artist",
     )
     logger.info("Found automatic tags for %s artists…", len(data))
     if dry_run:
-        logger.info("Running in dry-run mode, not commiting")
+        logger.info("Running in dry-run mode, not committing")
         return
 
     tags_tasks.add_tags_batch(
-        data, model=models.Artist,
+        data,
+        model=models.Artist,
     )
     return data
 
@@ -859,7 +1067,7 @@ def get_prunable_albums():
 
 
 def get_prunable_artists():
-    return models.Artist.objects.filter(tracks__isnull=True, albums__isnull=True)
+    return models.Artist.objects.filter(artist_credit__isnull=True)
 
 
 def update_library_entity(obj, data):
@@ -890,8 +1098,8 @@ UPDATE_CONFIG = {
             )
         },
     },
+    "artists": {},
     "album": {"title": {}, "mbid": {}, "release_date": {}},
-    "artist": {"name": {}, "mbid": {}},
     "album_artist": {"name": {}, "mbid": {}},
 }
 
@@ -905,11 +1113,15 @@ def update_track_metadata(audio_metadata, track):
     to_update = [
         ("track", track, lambda data: data),
         ("album", track.album, lambda data: data["album"]),
-        ("artist", track.artist, lambda data: data["artists"][0]),
+        (
+            "artist_credit",
+            track.artist_credit.all(),
+            lambda data: data["artist_credit"],
+        ),
         (
             "album_artist",
-            track.album.artist if track.album else None,
-            lambda data: data["album"]["artists"][0],
+            track.album.artist_credit.all() if track.album else None,
+            lambda data: data["album"]["artist_credit"],
         ),
     ]
     for id, obj, data_getter in to_update:
@@ -920,6 +1132,54 @@ def update_track_metadata(audio_metadata, track):
             obj_data = data_getter(new_data)
         except IndexError:
             continue
+
+        if id == "artist_credit":
+            if new_data.get("mbid", False):
+                logger.warning(
+                    "If a track mbid is provided, it will be use to generate artist_credit \
+                    information. If you want to set a custom artist_credit you nee to remove the track mbid"
+                )
+                track_artists_credits = get_or_create_artists_credits_from_musicbrainz(
+                    "recording", new_data.get("mbid"), None, None
+                )
+            else:
+                track_artists_credits = (
+                    get_or_create_artists_credits_from_artist_credit_metadata(
+                        obj_data,
+                        None,
+                        None,
+                    )
+                )
+            if track_artists_credits == obj:
+                continue
+
+            track.artist_credit.set(track_artists_credits)
+            continue
+
+        if id == "album_artist":
+            if new_data["album"].get("mbid", False):
+                logger.warning(
+                    "If a album mbid is provided, it will be use to generate album artist_credit \
+                    information. If you want to set a custom artist_credit you nee to remove the track mbid"
+                )
+                album_artists_credits = get_or_create_artists_credits_from_musicbrainz(
+                    "release", new_data["album"].get("mbid"), None, None
+                )
+            else:
+                album_artists_credits = (
+                    get_or_create_artists_credits_from_artist_credit_metadata(
+                        obj_data,
+                        None,
+                        None,
+                    )
+                )
+
+            if album_artists_credits == obj:
+                continue
+
+            track.album.artist_credit.set(album_artists_credits)
+            continue
+
         for field, config in UPDATE_CONFIG[id].items():
             getter = config.get(
                 "getter", lambda data, field: data[config.get("field", field)]
@@ -936,7 +1196,6 @@ def update_track_metadata(audio_metadata, track):
 
         if obj_updated_fields:
             obj.save(update_fields=obj_updated_fields)
-
     tags_models.set_tags(track, *new_data.get("tags", []))
 
     if track.album and "album" in new_data and new_data["album"].get("cover_data"):
@@ -947,7 +1206,17 @@ def update_track_metadata(audio_metadata, track):
 
 @celery.app.task(name="music.fs_import")
 @celery.require_instance(models.Library.objects.all(), "library")
-def fs_import(library, path, import_reference):
+def fs_import(
+    library,
+    path,
+    import_reference,
+    prune=True,
+    outbox=False,
+    broadcast=False,
+    replace=False,
+    batch_size=1000,
+    verbosity=1,
+):
     if cache.get("fs-import:status") != "pending":
         raise ValueError("Invalid import status")
 
@@ -962,13 +1231,13 @@ def fs_import(library, path, import_reference):
         "reference": import_reference,
         "watch": False,
         "interactive": False,
-        "batch_size": 1000,
+        "batch_size": batch_size,
         "async_": False,
-        "prune": True,
-        "replace": False,
-        "verbosity": 1,
+        "prune": prune,
+        "replace": replace,
+        "verbosity": verbosity,
         "exit_on_failure": False,
-        "outbox": False,
-        "broadcast": False,
+        "outbox": outbox,
+        "broadcast": broadcast,
     }
     command.handle(**options)

api/funkwhale_api/music/utils.py

+import mimetypes
 import os
 import pathlib
-import mimetypes
 
 import magic
 import mutagen
-import pydub
-
 from django.conf import settings
 from django.core.cache import cache
 from django.db.models import F
+from ffmpeg import FFmpeg
 
 from funkwhale_api.common import throttling
 from funkwhale_api.common.search import get_fts_query  # noqa
@@ -57,6 +56,7 @@ AUDIO_EXTENSIONS_AND_MIMETYPE = [
     ("opus", "audio/opus"),
     ("aac", "audio/x-m4a"),
     ("m4a", "audio/x-m4a"),
+    ("m4a", "audio/m4a"),
     ("flac", "audio/x-flac"),
     ("flac", "audio/flac"),
     ("aif", "audio/aiff"),
@@ -68,9 +68,18 @@ AUDIO_EXTENSIONS_AND_MIMETYPE = [
 EXTENSION_TO_MIMETYPE = {ext: mt for ext, mt in AUDIO_EXTENSIONS_AND_MIMETYPE}
 MIMETYPE_TO_EXTENSION = {mt: ext for ext, mt in AUDIO_EXTENSIONS_AND_MIMETYPE}
 
-SUPPORTED_EXTENSIONS = list(
-    sorted(set([ext for ext, _ in AUDIO_EXTENSIONS_AND_MIMETYPE]))
-)
+SUPPORTED_EXTENSIONS = list(sorted({ext for ext, _ in AUDIO_EXTENSIONS_AND_MIMETYPE}))
+
+
+def get_extension_to_mimetype_dict():
+    extension_dict = {}
+
+    for ext, mimetype in AUDIO_EXTENSIONS_AND_MIMETYPE:
+        if ext not in extension_dict:
+            extension_dict[ext] = []
+        extension_dict[ext].append(mimetype)
+
+    return extension_dict
 
 
 def get_ext_from_type(mimetype):
@@ -105,15 +114,10 @@ def get_actor_from_request(request):
     return actor
 
 
-def transcode_file(input, output, input_format=None, output_format="mp3", **kwargs):
-    with input.open("rb"):
-        audio = pydub.AudioSegment.from_file(input, format=input_format)
-    return transcode_audio(audio, output, output_format, **kwargs)
-
-
-def transcode_audio(audio, output, output_format, **kwargs):
-    with output.open("wb"):
-        return audio.export(output, format=output_format, **kwargs)
+def transcode_audio(audio_file_path, output_path, output_format="mp3", **kwargs):
+    FFmpeg().input(audio_file_path).output(
+        output_path, format=output_format, **kwargs
+    ).option("y").execute()
 
 
 def increment_downloads_count(upload, user, wsgi_request):
@@ -154,3 +158,10 @@ def browse_dir(root, path):
             files.append({"name": el, "dir": False})
 
     return dirs + files
+
+
+def get_artist_credit_string(obj):
+    final_credit = ""
+    for ac in obj.artist_credit.all():
+        final_credit = final_credit + ac.credit + ac.joinphrase
+    return final_credit

api/funkwhale_api/music/views.py

@@ -2,38 +2,37 @@ import base64
 import datetime
 import logging
 import urllib.parse
+
+import django.db.utils
+import requests.exceptions
 from django.conf import settings
 from django.core.cache import cache
 from django.db import transaction
-from django.db.models import Count, Prefetch, Sum, F, Q
-import django.db.utils
+from django.db.models import BooleanField, Case, Count, F, Prefetch, Q, Sum, Value, When
+from django.db.models.functions import Collate
 from django.utils import timezone
-
-from rest_framework import mixins
-from rest_framework import renderers
+from drf_spectacular.utils import OpenApiParameter, extend_schema, extend_schema_view
+from rest_framework import mixins, renderers
 from rest_framework import settings as rest_settings
 from rest_framework import views, viewsets
 from rest_framework.decorators import action
 from rest_framework.response import Response
 
-import requests.exceptions
-
 from funkwhale_api.common import decorators as common_decorators
 from funkwhale_api.common import permissions as common_permissions
 from funkwhale_api.common import preferences
 from funkwhale_api.common import utils as common_utils
 from funkwhale_api.common import views as common_views
-from funkwhale_api.federation.authentication import SignatureAuthentication
 from funkwhale_api.federation import actors
 from funkwhale_api.federation import api_serializers as federation_api_serializers
 from funkwhale_api.federation import decorators as federation_decorators
 from funkwhale_api.federation import models as federation_models
 from funkwhale_api.federation import routes
 from funkwhale_api.federation import tasks as federation_tasks
+from funkwhale_api.federation.authentication import SignatureAuthentication
 from funkwhale_api.tags.models import Tag, TaggedItem
-from funkwhale_api.tags.serializers import TagSerializer
-from funkwhale_api.users.oauth import permissions as oauth_permissions
 from funkwhale_api.users.authentication import ScopedTokenAuthentication
+from funkwhale_api.users.oauth import permissions as oauth_permissions
 
 from . import filters, licenses, models, serializers, tasks, utils
 
@@ -54,7 +53,8 @@ def get_libraries(filter_uploads):
         uploads = filter_uploads(obj, uploads)
         uploads = uploads.playable_by(actor)
         qs = models.Library.objects.filter(
-            pk__in=uploads.values_list("library", flat=True), channel=None,
+            pk__in=uploads.values_list("library", flat=True),
+            channel=None,
         ).annotate(_uploads_count=Count("uploads"))
         qs = qs.prefetch_related("actor")
         page = self.paginate_queryset(qs)
@@ -65,7 +65,10 @@ def get_libraries(filter_uploads):
         serializer = federation_api_serializers.LibrarySerializer(qs, many=True)
         return Response(serializer.data)
 
-    return libraries
+    return extend_schema(
+        responses=federation_api_serializers.LibrarySerializer(many=True),
+        parameters=[OpenApiParameter("id", location="query", exclude=True)],
+    )(action(methods=["get"], detail=True)(libraries))
 
 
 def refetch_obj(obj, queryset):
@@ -99,7 +102,7 @@ def refetch_obj(obj, queryset):
     return obj
 
 
-class HandleInvalidSearch(object):
+class HandleInvalidSearch:
     def list(self, *args, **kwargs):
         try:
             return super().list(*args, **kwargs)
@@ -117,16 +120,12 @@ class ArtistViewSet(
 ):
     queryset = (
         models.Artist.objects.all()
-        .prefetch_related("attributed_to", "attachment_cover")
+        .select_related("attributed_to", "attachment_cover")
+        .order_by("-id")
         .prefetch_related(
             "channel__actor",
-            Prefetch(
-                "tracks",
-                queryset=models.Track.objects.all(),
-                to_attr="_prefetched_tracks",
-            ),
         )
-        .order_by("-id")
+        .annotate(_tracks_count=Count("artist_credit__tracks"))
     )
     serializer_class = serializers.ArtistWithAlbumsSerializer
     permission_classes = [oauth_permissions.ScopePermission]
@@ -163,14 +162,12 @@ class ArtistViewSet(
             utils.get_actor_from_request(self.request)
         )
         return queryset.prefetch_related(
-            Prefetch("albums", queryset=albums), TAG_PREFETCH
+            Prefetch("artist_credit__albums", queryset=albums), TAG_PREFETCH
         )
 
-    libraries = action(methods=["get"], detail=True)(
-        get_libraries(
-            filter_uploads=lambda o, uploads: uploads.filter(
-                Q(track__artist=o) | Q(track__album__artist=o)
-            )
+    libraries = get_libraries(
+        lambda o, uploads: uploads.filter(
+            Q(track__artist_credit__artist=o) | Q(track__album__artist_credit__artist=o)
         )
     )
 
@@ -185,10 +182,9 @@ class AlbumViewSet(
     queryset = (
         models.Album.objects.all()
         .order_by("-creation_date")
-        .prefetch_related(
-            "artist__channel", "attributed_to", "attachment_cover", "tracks"
-        )
-    )
+        .select_related("attributed_to", "attachment_cover")
+        .prefetch_related("artist_credit__artist__channel")
+    ).distinct()
     serializer_class = serializers.AlbumSerializer
     permission_classes = [oauth_permissions.ScopePermission]
     required_scope = "libraries"
@@ -220,15 +216,19 @@ class AlbumViewSet(
     def get_queryset(self):
         queryset = super().get_queryset()
         if self.action in ["destroy"]:
-            queryset = queryset.exclude(artist__channel=None).filter(
-                artist__attributed_to=self.request.user.actor
+            queryset = queryset.exclude(artist_credit__artist__channel=None).filter(
+                artist_credit__artist__attributed_to=self.request.user.actor
             )
-        qs = queryset.prefetch_related(TAG_PREFETCH)
-        return qs
 
-    libraries = action(methods=["get"], detail=True)(
-        get_libraries(filter_uploads=lambda o, uploads: uploads.filter(track__album=o))
+        tracks = models.Track.objects.all().prefetch_related("album")
+        tracks = tracks.annotate_playable_by_actor(
+            utils.get_actor_from_request(self.request)
         )
+        return queryset.prefetch_related(
+            Prefetch("tracks", queryset=tracks), TAG_PREFETCH
+        )
+
+    libraries = get_libraries(lambda o, uploads: uploads.filter(track__album=o))
 
     def get_serializer_class(self):
         if self.action in ["create"]:
@@ -243,13 +243,58 @@ class AlbumViewSet(
         )
         models.Album.objects.filter(pk=instance.pk).delete()
 
+    @transaction.atomic
+    def create(self, request, *args, **kwargs):
+        request_data = request.data.copy()
+
+        if mbid := request_data.get("musicbrainz_albumid", None) or request_data.get(
+            "mbid", None
+        ):
+            artist_credit = tasks.get_or_create_artists_credits_from_musicbrainz(
+                "release",
+                mbid,
+                attributed_to=request.user.actor,
+                from_activity_id=None,
+            )
+        else:
+            artist_data = request_data.pop("artist", False)
+            artist_credit_data = request_data.pop("artist_credit", False)
+            if not artist_data and not artist_credit_data:
+                return Response({}, status=400)
+            if artist_data:
+                try:
+                    artist = models.Artist.objects.get(pk=artist_data)
+                except models.Artist.DoesNotExist:
+                    return Response({"détail": "artist id not found"}, status=400)
+
+                artist_credit, created = models.ArtistCredit.objects.get_or_create(
+                    **{
+                        "artist": artist,
+                        "credit": artist.name,
+                        "joinphrase": "",
+                        "index": 0,
+                    }
+                )
+            elif artist_credit_data:
+                try:
+                    artist_credit = models.ArtistCredit.objects.get(
+                        pk=artist_credit_data
+                    )
+                except models.ArtistCredit.DoesNotExist:
+                    return Response(
+                        {"détail": "artist_credit id not found"}, status=400
+                    )
+
+        request_data["artist_credit"] = [artist_credit.pk]
+        serializer = self.get_serializer(data=request_data)
+        serializer.is_valid(raise_exception=True)
+        self.perform_create(serializer)
+        return Response(serializer.data, status=204)
+
 
 class LibraryViewSet(
-    mixins.CreateModelMixin,
     mixins.ListModelMixin,
     mixins.RetrieveModelMixin,
-    mixins.UpdateModelMixin,
-    mixins.DestroyModelMixin,
     viewsets.GenericViewSet,
 ):
     lookup_field = "uuid"
@@ -284,38 +329,8 @@ class LibraryViewSet(
 
         return qs
 
-    def perform_create(self, serializer):
-        serializer.save(actor=self.request.user.actor)
-
-    @transaction.atomic
-    def perform_destroy(self, instance):
-        routes.outbox.dispatch(
-            {"type": "Delete", "object": {"type": "Library"}},
-            context={"library": instance},
-        )
-        instance.delete()
-
-    follows = action
-
-    @action(methods=["get"], detail=True)
-    @transaction.non_atomic_requests
-    def follows(self, request, *args, **kwargs):
-        library = self.get_object()
-        queryset = (
-            library.received_follows.filter(target__actor=self.request.user.actor)
-            .prefetch_related("actor", "target__actor")
-            .order_by("-creation_date")
-        )
-        page = self.paginate_queryset(queryset)
-        if page is not None:
-            serializer = federation_api_serializers.LibraryFollowSerializer(
-                page, many=True
-            )
-            return self.get_paginated_response(serializer.data)
-
-        serializer = self.get_serializer(queryset, many=True)
-        return Response(serializer.data)
-
+    # TODO quickfix, basically specifying the response would be None
+    @extend_schema(responses=None)
     @action(
         methods=["get", "post", "delete"],
         detail=False,
@@ -368,6 +383,12 @@ class LibraryViewSet(
                 library_id=serializer.validated_data["library"].pk,
                 path=serializer.validated_data["path"],
                 import_reference=serializer.validated_data["import_reference"],
+                prune=serializer.validated_data["prune"],
+                outbox=serializer.validated_data["outbox"],
+                broadcast=serializer.validated_data["broadcast"],
+                replace=serializer.validated_data["replace"],
+                batch_size=serializer.validated_data["batch_size"],
+                verbosity=serializer.validated_data["verbosity"],
             )
             return Response(status=201)
         if request.method == "DELETE":
@@ -390,7 +411,7 @@ class TrackViewSet(
         .for_nested_serialization()
         .prefetch_related("attributed_to", "attachment_cover")
         .order_by("-creation_date")
-    )
+    ).distinct()
     serializer_class = serializers.TrackSerializer
     permission_classes = [oauth_permissions.ScopePermission]
     required_scope = "libraries"
@@ -412,8 +433,8 @@ class TrackViewSet(
     def get_queryset(self):
         queryset = super().get_queryset()
         if self.action in ["destroy"]:
-            queryset = queryset.exclude(artist__channel=None).filter(
-                artist__attributed_to=self.request.user.actor
+            queryset = queryset.exclude(artist_credit__artist__channel=None).filter(
+                artist_credit__artist__attributed_to=self.request.user.actor
             )
         filter_favorites = self.request.GET.get("favorites", None)
         user = self.request.user
@@ -425,9 +446,7 @@ class TrackViewSet(
         )
         return queryset.prefetch_related(TAG_PREFETCH)
 
-    libraries = action(methods=["get"], detail=True)(
-        get_libraries(filter_uploads=lambda o, uploads: uploads.filter(track=o))
-    )
+    libraries = get_libraries(lambda o, uploads: uploads.filter(track=o))
 
     def get_serializer_context(self):
         context = super().get_serializer_context()
@@ -509,7 +528,7 @@ def should_transcode(upload, format, max_bitrate=None):
         # upload should have a mimetype, otherwise we cannot transcode
         format_need_transcoding = False
     elif upload.mimetype == utils.EXTENSION_TO_MIMETYPE[format]:
-        # requested format sould be different than upload mimetype, otherwise
+        # requested format should be different than upload mimetype, otherwise
         # there is no need to transcode
         format_need_transcoding = False
 
@@ -524,8 +543,8 @@ def should_transcode(upload, format, max_bitrate=None):
 
 
 def get_content_disposition(filename):
-    filename = "filename*=UTF-8''{}".format(urllib.parse.quote(filename))
-    return "attachment; {}".format(filename)
+    filename = f"filename*=UTF-8''{urllib.parse.quote(filename)}"
+    return f"attachment; {filename}"
 
 
 def record_downloads(f):
@@ -575,7 +594,7 @@ def handle_serve(
             try:
                 f.download_audio_from_remote(actor=actor)
             except requests.exceptions.RequestException:
-                return Response({"detail": "Remove track is unavailable"}, status=503)
+                return Response({"detail": "Remote track is unavailable"}, status=503)
         data = f.get_audio_data()
         if data:
             f.duration = data["duration"]
@@ -627,6 +646,7 @@ class ListenMixin(mixins.RetrieveModelMixin, viewsets.GenericViewSet):
     anonymous_policy = "setting"
     lookup_field = "uuid"
 
+    @extend_schema(responses=bytes)
     def retrieve(self, request, *args, **kwargs):
         config = {
             "explicit_file": request.GET.get("upload"),
@@ -640,11 +660,21 @@ class ListenMixin(mixins.RetrieveModelMixin, viewsets.GenericViewSet):
 
 def handle_stream(track, request, download, explicit_file, format, max_bitrate):
     actor = utils.get_actor_from_request(request)
-    queryset = track.uploads.prefetch_related("track__album__artist", "track__artist")
+    queryset = track.uploads.prefetch_related(
+        "track__album__artist_credit__artist", "track__artist_credit"
+    )
     if explicit_file:
         queryset = queryset.filter(uuid=explicit_file)
     queryset = queryset.playable_by(actor)
-    queryset = queryset.order_by(F("audio_file").desc(nulls_last=True))
+    # third_party uploads are displayed before manual upload only if no audio file is found in manual upload
+    queryset = queryset.order_by(
+        Case(
+            When(third_party_provider__isnull=False, then=Value(1)),
+            default=Value(0),
+            output_field=BooleanField(),
+        ),
+        F("audio_file").desc(nulls_last=True),
+    )
     upload = queryset.first()
     if not upload:
         return Response(status=404)
@@ -667,8 +697,13 @@ def handle_stream(track, request, download, explicit_file, format, max_bitrate):
     )
 
 
+class AudioRenderer(renderers.JSONRenderer):
+    media_type = "audio/*"
+
+
+@extend_schema_view(get=extend_schema(operation_id="get_track_file"))
 class ListenViewSet(ListenMixin):
-    pass
+    renderer_classes = [AudioRenderer]
 
 
 class MP3Renderer(renderers.JSONRenderer):
@@ -679,6 +714,7 @@ class MP3Renderer(renderers.JSONRenderer):
 class StreamViewSet(ListenMixin):
     renderer_classes = [MP3Renderer]
 
+    @extend_schema(operation_id="get_track_stream", responses=bytes)
     def retrieve(self, request, *args, **kwargs):
         config = {
             "explicit_file": None,
@@ -704,8 +740,8 @@ class UploadViewSet(
         .order_by("-creation_date")
         .prefetch_related(
             "library__actor",
-            "track__artist",
-            "track__album__artist",
+            "track__artist_credit",
+            "track__album__artist_credit",
             "track__attachment_cover",
         )
     )
@@ -724,7 +760,7 @@ class UploadViewSet(
         "import_date",
         "bitrate",
         "size",
-        "artist__name",
+        "artist_credit__artist__name",
     )
 
     def get_queryset(self):
@@ -739,6 +775,10 @@ class UploadViewSet(
             qs = qs.playable_by(actor)
         return qs
 
+    @extend_schema(
+        responses=tasks.metadata.TrackMetadataSerializer(),
+        operation_id="get_upload_metadata",
+    )
     @action(methods=["get"], detail=True, url_path="audio-file-metadata")
     def audio_file_metadata(self, request, *args, **kwargs):
         upload = self.get_object()
@@ -759,6 +799,31 @@ class UploadViewSet(
             cover_data["content"] = base64.b64encode(cover_data["content"])
         return Response(payload, status=200)
 
+    @extend_schema(
+        request=serializers.UploadBulkUpdateSerializer(many=True),
+    )
+    @action(detail=False, methods=["patch"])
+    def bulk_update(self, request, *args, **kwargs):
+        """
+        Used to move an upload from one library to another. Receive a upload uuid and a privacy_level
+        """
+        if not self.request.user.is_authenticated:
+            return Response({}, status=403)
+
+        serializer = serializers.UploadBulkUpdateSerializer(
+            data=request.data, many=True, context={"actor": self.request.user.actor}
+        )
+        serializer.is_valid(raise_exception=True)
+        objs = serializer.save()
+        routes.outbox.dispatch(
+            {"type": "Update", "object": {"type": "AudioCollection"}},
+            context={"audios": objs},
+        )
+        return Response(
+            serializers.UploadForOwnerSerializer(objs, many=True).data,
+            status=200,
+        )
+
     @action(methods=["post"], detail=False)
     def action(self, request, *args, **kwargs):
         queryset = self.get_queryset()
@@ -797,6 +862,9 @@ class Search(views.APIView):
     required_scope = "libraries"
     anonymous_policy = "setting"
 
+    @extend_schema(
+        operation_id="get_search_results", responses=serializers.SearchResultSerializer
+    )
     def get(self, request, *args, **kwargs):
         query = request.GET.get("query", request.GET.get("q", "")) or ""
         query = query.strip()
@@ -804,17 +872,10 @@ class Search(views.APIView):
             return Response({"detail": "empty query"}, status=400)
         try:
             results = {
-                # 'tags': serializers.TagSerializer(self.get_tags(query), many=True).data,
-                "artists": serializers.ArtistWithAlbumsSerializer(
-                    self.get_artists(query), many=True
-                ).data,
-                "tracks": serializers.TrackSerializer(
-                    self.get_tracks(query), many=True
-                ).data,
-                "albums": serializers.AlbumSerializer(
-                    self.get_albums(query), many=True
-                ).data,
-                "tags": TagSerializer(self.get_tags(query), many=True).data,
+                "artists": self.get_artists(query),
+                "tracks": self.get_tracks(query),
+                "albums": self.get_albums(query),
+                "tags": self.get_tags(query),
             }
         except django.db.utils.ProgrammingError as e:
             if "in tsquery:" in str(e):
@@ -822,25 +883,29 @@ class Search(views.APIView):
             else:
                 raise
 
-        return Response(results, status=200)
+        return Response(serializers.SearchResultSerializer(results).data, status=200)
 
     def get_tracks(self, query):
         query_obj = utils.get_fts_query(
             query,
-            fts_fields=["body_text", "album__body_text", "artist__body_text"],
+            fts_fields=[
+                "body_text",
+                "album__body_text",
+                "artist_credit__artist__body_text",
+            ],
             model=models.Track,
         )
         qs = (
             models.Track.objects.all()
             .filter(query_obj)
             .prefetch_related(
-                "artist",
+                "artist_credit",
                 "attributed_to",
                 Prefetch(
                     "album",
                     queryset=models.Album.objects.select_related(
-                        "artist", "attachment_cover", "attributed_to"
-                    ).prefetch_related("tracks"),
+                        "attachment_cover", "attributed_to"
+                    ).prefetch_related("tracks", "artist_credit"),
                 ),
             )
         )
@@ -848,13 +913,15 @@ class Search(views.APIView):
 
     def get_albums(self, query):
         query_obj = utils.get_fts_query(
-            query, fts_fields=["body_text", "artist__body_text"], model=models.Album
+            query,
+            fts_fields=["body_text", "artist_credit__artist__body_text"],
+            model=models.Album,
         )
         qs = (
             models.Album.objects.all()
             .filter(query_obj)
-            .select_related("artist", "attachment_cover", "attributed_to")
-            .prefetch_related("tracks__artist")
+            .select_related("attachment_cover", "attributed_to")
+            .prefetch_related("tracks__artist_credit", "artist_credit")
         )
         return common_utils.order_for_search(qs, "title")[: self.max_results]
 
@@ -870,9 +937,13 @@ class Search(views.APIView):
         return common_utils.order_for_search(qs, "name")[: self.max_results]
 
     def get_tags(self, query):
-        search_fields = ["name__unaccent"]
+        search_fields = ["tag_deterministic"]
         query_obj = utils.get_query(query, search_fields)
-        qs = Tag.objects.all().filter(query_obj)
+        qs = (
+            Tag.objects.all()
+            .annotate(tag_deterministic=Collate("name", "und-x-icu"))
+            .filter(query_obj)
+        )
         return common_utils.order_for_search(qs, "name")[: self.max_results]
 
 
@@ -908,6 +979,7 @@ class OembedView(views.APIView):
     permission_classes = [oauth_permissions.ScopePermission]
     required_scope = "libraries"
     anonymous_policy = "setting"
+    serializer_class = serializers.OembedSerializer
 
     def get(self, request, *args, **kwargs):
         serializer = serializers.OembedSerializer(data=request.GET)

api/funkwhale_api/musicbrainz/client.py

-import memoize.djangocache
 import musicbrainzngs
+from cache_memoize import cache_memoize
 from django.conf import settings
 
 from funkwhale_api import __version__
@@ -7,10 +7,7 @@ from funkwhale_api import __version__
 _api = musicbrainzngs
 _api.set_useragent("funkwhale", str(__version__), settings.FUNKWHALE_URL)
 _api.set_hostname(settings.MUSICBRAINZ_HOSTNAME)
-
-
-store = memoize.djangocache.Cache("default")
-memo = memoize.Memoizer(store, namespace="memoize:musicbrainz")
+_api.set_format(fmt="json")
 
 
 def clean_artist_search(query, **kwargs):
@@ -20,42 +17,63 @@ def clean_artist_search(query, **kwargs):
     return _api.search_artists(query, **cleaned_kwargs)
 
 
-class API(object):
+class API:
     _api = _api
 
-    class artists(object):
-        search = memo(clean_artist_search, max_age=settings.MUSICBRAINZ_CACHE_DURATION)
-        get = memo(_api.get_artist_by_id, max_age=settings.MUSICBRAINZ_CACHE_DURATION)
-
-    class images(object):
-        get_front = memo(
-            _api.get_image_front, max_age=settings.MUSICBRAINZ_CACHE_DURATION
-        )
-
-    class recordings(object):
-        search = memo(
-            _api.search_recordings, max_age=settings.MUSICBRAINZ_CACHE_DURATION
-        )
-        get = memo(
-            _api.get_recording_by_id, max_age=settings.MUSICBRAINZ_CACHE_DURATION
-        )
-
-    class releases(object):
-        search = memo(_api.search_releases, max_age=settings.MUSICBRAINZ_CACHE_DURATION)
-        get = memo(_api.get_release_by_id, max_age=settings.MUSICBRAINZ_CACHE_DURATION)
-        browse = memo(_api.browse_releases, max_age=settings.MUSICBRAINZ_CACHE_DURATION)
+    class artists:
+        search = cache_memoize(
+            settings.MUSICBRAINZ_CACHE_DURATION,
+            prefix="memoize:musicbrainz:clean_artist_search",
+        )(clean_artist_search)
+        get = cache_memoize(
+            settings.MUSICBRAINZ_CACHE_DURATION,
+            prefix="memoize:musicbrainz:get_artist_by_id",
+        )(_api.get_artist_by_id)
+
+    class images:
+        get_front = cache_memoize(
+            settings.MUSICBRAINZ_CACHE_DURATION,
+            prefix="memoize:musicbrainz:get_image_front",
+        )(_api.get_image_front)
+
+    class recordings:
+        search = cache_memoize(
+            settings.MUSICBRAINZ_CACHE_DURATION,
+            prefix="memoize:musicbrainz:search_recordings",
+        )(_api.search_recordings)
+        get = cache_memoize(
+            settings.MUSICBRAINZ_CACHE_DURATION,
+            prefix="memoize:musicbrainz:get_recording_by_id",
+        )(_api.get_recording_by_id)
+
+    class releases:
+        search = cache_memoize(
+            settings.MUSICBRAINZ_CACHE_DURATION,
+            prefix="memoize:musicbrainz:search_releases",
+        )(_api.search_releases)
+        get = cache_memoize(
+            settings.MUSICBRAINZ_CACHE_DURATION,
+            prefix="memoize:musicbrainz:get_release_by_id",
+        )(_api.get_release_by_id)
+        browse = cache_memoize(
+            settings.MUSICBRAINZ_CACHE_DURATION,
+            prefix="memoize:musicbrainz:browse_releases",
+        )(_api.browse_releases)
         # get_image_front = _api.get_image_front
 
-    class release_groups(object):
-        search = memo(
-            _api.search_release_groups, max_age=settings.MUSICBRAINZ_CACHE_DURATION
-        )
-        get = memo(
-            _api.get_release_group_by_id, max_age=settings.MUSICBRAINZ_CACHE_DURATION
-        )
-        browse = memo(
-            _api.browse_release_groups, max_age=settings.MUSICBRAINZ_CACHE_DURATION
-        )
+    class release_groups:
+        search = cache_memoize(
+            settings.MUSICBRAINZ_CACHE_DURATION,
+            prefix="memoize:musicbrainz:search_release_groups",
+        )(_api.search_release_groups)
+        get = cache_memoize(
+            settings.MUSICBRAINZ_CACHE_DURATION,
+            prefix="memoize:musicbrainz:get_release_group_by_id",
+        )(_api.get_release_group_by_id)
+        browse = cache_memoize(
+            settings.MUSICBRAINZ_CACHE_DURATION,
+            prefix="memoize:musicbrainz:browse_release_groups",
+        )(_api.browse_release_groups)
         # get_image_front = _api.get_image_front
 
 

api/funkwhale_api/musicbrainz/urls.py

-from django.conf.urls import url
+from django.urls import re_path
+
 from funkwhale_api.common import routers
 
 from . import views
@@ -6,22 +7,22 @@ from . import views
 router = routers.OptionalSlashRouter()
 router.register(r"search", views.SearchViewSet, "search")
 urlpatterns = [
-    url(
+    re_path(
         "releases/(?P<uuid>[0-9a-z-]+)/$",
         views.ReleaseDetail.as_view(),
         name="release-detail",
     ),
-    url(
+    re_path(
         "artists/(?P<uuid>[0-9a-z-]+)/$",
         views.ArtistDetail.as_view(),
         name="artist-detail",
     ),
-    url(
+    re_path(
         "release-groups/browse/(?P<artist_uuid>[0-9a-z-]+)/$",
         views.ReleaseGroupBrowse.as_view(),
         name="release-group-browse",
     ),
-    url(
+    re_path(
         "releases/browse/(?P<release_group_uuid>[0-9a-z-]+)/$",
         views.ReleaseBrowse.as_view(),
         name="release-browse",

api/funkwhale_api/playlists/admin.py

@@ -5,7 +5,7 @@ from . import models
 
 @admin.register(models.Playlist)
 class PlaylistAdmin(admin.ModelAdmin):
-    list_display = ["name", "user", "privacy_level", "creation_date"]
+    list_display = ["name", "actor", "privacy_level", "creation_date"]
     search_fields = ["name"]
     list_select_related = True
 
@@ -15,3 +15,22 @@ class PlaylistTrackAdmin(admin.ModelAdmin):
     list_display = ["playlist", "track", "index"]
     search_fields = ["track__name", "playlist__name"]
     list_select_related = True
+
+
+@admin.register(models.PlaylistScan)
+class LibraryScanAdmin(admin.ModelAdmin):
+    list_display = [
+        "id",
+        "playlist",
+        "actor",
+        "status",
+        "creation_date",
+        "modification_date",
+        "status",
+        "total_files",
+        "processed_files",
+        "errored_files",
+    ]
+    list_select_related = True
+    search_fields = ["actor__username", "playlist__name"]
+    list_filter = ["status"]

api/funkwhale_api/playlists/factories.py

 import factory
+from django.conf import settings
 
-from funkwhale_api.factories import registry, NoUpdateOnCreate
+from funkwhale_api.factories import NoUpdateOnCreate, registry
+from funkwhale_api.federation import models
+from funkwhale_api.federation.factories import ActorFactory, MusicLibraryFactory
 from funkwhale_api.music.factories import TrackFactory
-from funkwhale_api.users.factories import UserFactory
 
 
 @registry.register
 class PlaylistFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     name = factory.Faker("name")
-    user = factory.SubFactory(UserFactory)
+    actor = factory.SubFactory(ActorFactory)
+    fid = factory.Faker("federation_url")
+    uuid = factory.Faker("uuid4")
+    library = factory.SubFactory(MusicLibraryFactory)
 
     class Meta:
         model = "playlists.Playlist"
 
+    @factory.post_generation
+    def local(self, create, extracted, **kwargs):
+        if not extracted and not kwargs:
+            return
+        domain = models.Domain.objects.get_or_create(name=settings.FEDERATION_HOSTNAME)[
+            0
+        ]
+        self.fid = f"https://{domain}/federation/music/playlists/{self.uuid}"
+        self.save(update_fields=["fid"])
+
 
 @registry.register
 class PlaylistTrackFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     playlist = factory.SubFactory(PlaylistFactory)
     track = factory.SubFactory(TrackFactory)
+    fid = factory.Faker("federation_url")
+    uuid = factory.Faker("uuid4")
 
     class Meta:
         model = "playlists.PlaylistTrack"
+
+    @factory.post_generation
+    def local(self, create, extracted, **kwargs):
+        if not extracted and not kwargs:
+            return
+        domain = models.Domain.objects.get_or_create(name=settings.FEDERATION_HOSTNAME)[
+            0
+        ]
+        self.fid = f"https://{domain}/federation/music/playlists-tracks/{self.uuid}"
+        self.save(update_fields=["fid"])
+
+
+@registry.register
+class PlaylistScanFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
+    playlist = factory.SubFactory(PlaylistFactory)
+    actor = factory.SubFactory(ActorFactory)
+
+    class Meta:
+        model = "playlists.PlaylistScan"

api/funkwhale_api/playlists/filters.py

@@ -22,11 +22,11 @@ class PlaylistFilter(filters.FilterSet):
         distinct=True,
     )
     artist = filters.ModelChoiceFilter(
-        "playlist_tracks__track__artist",
+        "playlist_tracks__track__artist_credit__artist",
         queryset=music_models.Artist.objects.all(),
         distinct=True,
     )
-    scope = common_filters.ActorScopeFilter(actor_field="user__actor", distinct=True)
+    scope = common_filters.ActorScopeFilter(actor_field="actor", distinct=True)
 
     class Meta:
         model = models.Playlist
@@ -42,5 +42,5 @@ class PlaylistFilter(filters.FilterSet):
             return queryset.filter(plts_count=0)
 
     def filter_q(self, queryset, name, value):
-        query = utils.get_query(value, ["name", "user__username"])
+        query = utils.get_query(value, ["name", "actor__user__username"])
         return queryset.filter(query)

api/funkwhale_api/playlists/migrations/0005_remove_playlist_user_playlist_actor.py

+# Generated by Django 4.2.9 on 2024-11-25 12:03
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import uuid
+
+from funkwhale_api.federation import utils
+from django.urls import reverse
+
+
+def get_user_actor(apps, schema_editor):
+    MyModel = apps.get_model("playlists", "Playlist")
+    for row in MyModel.objects.all():
+        actor = row.user.actor
+        row.actor = actor
+        row.save(update_fields=["actor"])
+
+
+def gen_uuid(apps, schema_editor):
+    MyModel = apps.get_model("playlists", "Playlist")
+    for row in MyModel.objects.all():
+        unique_uuid = uuid.uuid4()
+        while MyModel.objects.filter(uuid=unique_uuid).exists():
+            unique_uuid = uuid.uuid4()
+
+        fid = utils.full_url(
+            reverse("federation:music:playlists-detail", kwargs={"uuid": unique_uuid})
+        )
+        row.uuid = unique_uuid
+        row.fid = fid
+        row.save(update_fields=["uuid", "fid"])
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("federation", "0028_auto_20221027_1141"),
+        ("playlists", "0004_auto_20180320_1713"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="playlist",
+            name="fid",
+            field=models.URLField(max_length=500, null=True),
+        ),
+        migrations.AddField(
+            model_name="playlist",
+            name="url",
+            field=models.URLField(blank=True, max_length=500, null=True),
+        ),
+        migrations.AddField(
+            model_name="playlist",
+            name="uuid",
+            field=models.UUIDField(default=uuid.uuid4, null=True),
+        ),
+        migrations.RunPython(gen_uuid, reverse_code=migrations.RunPython.noop),
+        migrations.AlterField(
+            model_name="playlist",
+            name="uuid",
+            field=models.UUIDField(default=uuid.uuid4, null=False, unique=True),
+        ),
+        migrations.AlterField(
+            model_name="playlist",
+            name="fid",
+            field=models.URLField(
+                max_length=500,
+                unique=True,
+                db_index=True,
+                null=False,
+            ),
+        ),
+        migrations.AddField(
+            model_name="playlist",
+            name="actor",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="playlists",
+                to="federation.actor",
+                null=True,
+                blank=True,
+            ),
+        ),
+        migrations.RunPython(get_user_actor, reverse_code=migrations.RunPython.noop),
+        migrations.AlterField(
+            model_name="playlist",
+            name="actor",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="track_favorites",
+                to="federation.actor",
+            ),
+        ),
+        migrations.RemoveField(
+            model_name="playlist",
+            name="user",
+        ),
+    ]

api/funkwhale_api/playlists/migrations/0006_playlisttrack_fid_playlisttrack_url_and_more.py

+# Generated by Django 4.2.9 on 2024-11-28 17:49
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import uuid
+
+from funkwhale_api.federation import utils
+from django.urls import reverse
+
+
+def gen_uuid(apps, schema_editor):
+    MyModel = apps.get_model("playlists", "PlaylistTrack")
+    rows = []
+    for row in MyModel.objects.all():
+        unique_uuid = uuid.uuid4()
+        while MyModel.objects.filter(uuid=unique_uuid).exists():
+            unique_uuid = uuid.uuid4()
+
+        fid = utils.full_url(
+            reverse("federation:music:playlists-detail", kwargs={"uuid": unique_uuid})
+        )
+        row.uuid = unique_uuid
+        row.fid = fid
+        rows.append(row)
+
+    MyModel.objects.bulk_update(rows, fields=["uuid", "fid"], batch_size=5000)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("federation", "0028_auto_20221027_1141"),
+        ("playlists", "0005_remove_playlist_user_playlist_actor"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="playlisttrack",
+            name="uuid",
+            field=models.UUIDField(default=uuid.uuid4, null=True),
+        ),
+        migrations.AddField(
+            model_name="playlisttrack",
+            name="fid",
+            field=models.URLField(max_length=500, null=True),
+        ),
+        migrations.AddField(
+            model_name="playlisttrack",
+            name="url",
+            field=models.URLField(blank=True, max_length=500, null=True),
+        ),
+        migrations.RunPython(gen_uuid, reverse_code=migrations.RunPython.noop),
+        migrations.AlterField(
+            model_name="playlisttrack",
+            name="uuid",
+            field=models.UUIDField(default=uuid.uuid4, null=False, unique=True),
+        ),
+        migrations.AlterField(
+            model_name="playlisttrack",
+            name="fid",
+            field=models.URLField(
+                db_index=True, max_length=500, unique=True, null=False
+            ),
+        ),
+    ]

api/funkwhale_api/playlists/migrations/0007_alter_playlist_actor_alter_playlisttrack_uuid_and_more.py

+# Generated by Django 4.2.9 on 2024-12-03 11:28
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("federation", "0028_auto_20221027_1141"),
+        ("playlists", "0006_playlisttrack_fid_playlisttrack_url_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="playlist",
+            name="actor",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="playlists",
+                to="federation.actor",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="playlisttrack",
+            name="uuid",
+            field=models.UUIDField(default=uuid.uuid4, unique=True),
+        ),
+        migrations.CreateModel(
+            name="PlaylistScan",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("total_files", models.PositiveIntegerField(default=0)),
+                ("processed_files", models.PositiveIntegerField(default=0)),
+                ("errored_files", models.PositiveIntegerField(default=0)),
+                ("status", models.CharField(default="pending", max_length=25)),
+                (
+                    "creation_date",
+                    models.DateTimeField(default=django.utils.timezone.now),
+                ),
+                ("modification_date", models.DateTimeField(blank=True, null=True)),
+                (
+                    "actor",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="federation.actor",
+                    ),
+                ),
+                (
+                    "playlist",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="scans",
+                        to="playlists.playlist",
+                    ),
+                ),
+            ],
+        ),
+    ]

api/funkwhale_api/playlists/migrations/0008_playlist_library_drop.py

+# Generated by Django 4.2.9 on 2025-01-03 16:12
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("playlists", "0007_alter_playlist_actor_alter_playlisttrack_uuid_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="playlist",
+            name="name",
+            field=models.CharField(max_length=100),
+        ),
+        migrations.AddField(
+            model_name="playlist",
+            name="description",
+            field=models.TextField(blank=True, max_length=5000, null=True),
+        ),
+    ]

api/funkwhale_api/playlists/migrations/0009_playlist_library.py

+import django.db.models.deletion
+from django.db import migrations, models, transaction
+from funkwhale_api.federation import utils as federation_utils
+from django.urls import reverse
+import uuid
+from django.conf import settings
+
+
+def add_uploads_to_pl_library(playlist, library):
+    for plt in playlist.playlist_tracks.all():
+        for upload in plt.track.uploads.filter(library__actor=playlist.actor):
+            library.uploads.add(upload)
+
+
+@transaction.atomic
+def create_playlist_libraries(apps, schema_editor):
+    Playlist = apps.get_model("playlists", "Playlist")
+    Library = apps.get_model("music", "Library")
+    Actor = apps.get_model("federation", "Actor")
+    playlist_with_lib_count = 0
+    playlists = []
+    for playlist in Playlist.objects.all():
+        if not federation_utils.is_local(playlist.actor.fid):
+            continue
+        library = playlist.library
+        if not library:
+            try:
+                # we don't want to get_or_create in case it's a channel lib
+                library = Library.objects.create(
+                    name="playlist_" + playlist.name,
+                    privacy_level="me",
+                    actor=playlist.actor,
+                    uuid=(new_uuid := uuid.uuid4()),
+                    fid=federation_utils.full_url(
+                        reverse(
+                            "federation:music:libraries-detail",
+                            kwargs={"uuid": new_uuid},
+                        )
+                    ),
+                )
+                library.save()
+                playlist.library = library
+                playlists.append(playlist)
+                with transaction.atomic():
+                    add_uploads_to_pl_library(playlist, library)
+            except Exception as e:
+                print(
+                    f"An error occurred during playlist.library creation, raising since we want\
+                      to enforce one lib per playlist"
+                )
+                raise e
+        Playlist.objects.bulk_update(playlists, fields=["library"], batch_size=5000)
+        playlist_with_lib_count = playlist_with_lib_count + 1
+        local_actors = Actor.objects.filter(domain_id=settings.FEDERATION_HOSTNAME)
+
+        if (
+            Library.objects.filter(
+                playlist__isnull=False, actor__in=local_actors
+            ).count()
+            != playlist_with_lib_count
+        ):
+            raise Exception(
+                "Should have the same amount of local playlist and libraries with playlist"
+            )
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("playlists", "0008_playlist_library_drop"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="playlist",
+            name="library",
+            field=models.OneToOneField(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="playlist",
+                to="music.library",
+            ),
+        ),
+        migrations.RunPython(
+            create_playlist_libraries, reverse_code=migrations.RunPython.noop
+        ),
+    ]

api/funkwhale_api/playlists/migrations/0010_alter_playlist_privacy_level.py

+# Generated by Django 5.1.6 on 2025-09-12 08:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("playlists", "0009_playlist_library"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="playlist",
+            name="privacy_level",
+            field=models.CharField(
+                choices=[
+                    ("me", "Only me"),
+                    ("followers", "Me and my followers"),
+                    ("instance", "Everyone on my instance, and my followers"),
+                    ("everyone", "Everyone, including people on other instances"),
+                ],
+                max_length=30,
+                default="me",
+            ),
+        ),
+    ]

api/funkwhale_api/playlists/models.py

+import datetime
+import uuid
+
 from django.db import models, transaction
+from django.db.models import Q
+from django.db.models.expressions import OuterRef, Subquery
+from django.db.models.signals import post_delete
+from django.dispatch import receiver
+from django.urls import reverse
 from django.utils import timezone
 from rest_framework import exceptions
 
-from funkwhale_api.common import fields, preferences
+from funkwhale_api.common import fields
+from funkwhale_api.common import models as common_models
+from funkwhale_api.common import preferences
+from funkwhale_api.common import utils as common_utils
+from funkwhale_api.federation import models as federation_models
+from funkwhale_api.federation import utils as federation_utils
 from funkwhale_api.music import models as music_models
 
 
-class PlaylistQuerySet(models.QuerySet):
+class PlaylistQuerySet(models.QuerySet, common_models.LocalFromFidQuerySet):
     def with_tracks_count(self):
         return self.annotate(_tracks_count=models.Count("playlist_tracks"))
 
     def with_duration(self):
+        subquery = Subquery(
+            music_models.Upload.objects.filter(
+                track_id=OuterRef("playlist_tracks__track__id")
+            )
+            .order_by("id")
+            .values("id")[:1]
+        )
         return self.annotate(
-            duration=models.Sum("playlist_tracks__track__uploads__duration")
+            duration=models.Sum(
+                "playlist_tracks__track__uploads__duration",
+                filter=Q(playlist_tracks__track__uploads=subquery),
+            )
         )
 
     def with_covers(self):
@@ -55,22 +78,62 @@ class PlaylistQuerySet(models.QuerySet):
             return self.exclude(playlist_tracks__in=plts).distinct()
 
 
-class Playlist(models.Model):
-    name = models.CharField(max_length=50)
-    user = models.ForeignKey(
-        "users.User", related_name="playlists", on_delete=models.CASCADE
+class Playlist(federation_models.FederationMixin):
+    uuid = models.UUIDField(default=uuid.uuid4, unique=True)
+    name = models.CharField(max_length=100)
+    actor = models.ForeignKey(
+        "federation.Actor", related_name="playlists", on_delete=models.CASCADE
     )
     creation_date = models.DateTimeField(default=timezone.now)
     modification_date = models.DateTimeField(auto_now=True)
     privacy_level = fields.get_privacy_field()
-
+    description = models.TextField(max_length=5000, null=True, blank=True)
     objects = PlaylistQuerySet.as_manager()
+    federation_namespace = "playlists"
+    library = models.OneToOneField(
+        "music.Library",
+        null=True,
+        blank=True,
+        on_delete=models.CASCADE,
+        related_name="playlist",
+    )
 
     def __str__(self):
         return self.name
 
     def get_absolute_url(self):
-        return "/library/playlists/{}".format(self.pk)
+        return f"/library/playlists/{self.uuid}"
+
+    def get_federation_id(self):
+        if self.fid:
+            return self.fid
+        return federation_utils.full_url(
+            reverse(
+                f"federation:music:{self.federation_namespace}-detail",
+                kwargs={"uuid": self.uuid},
+            )
+        )
+
+    def save(self, **kwargs):
+        if not self.pk and not self.fid:
+            self.fid = self.get_federation_id()
+
+        if not self.pk and not self.library_id:
+            self.library = music_models.Library.objects.create(
+                actor=self.actor,
+                name="playlist_" + self.name,
+                privacy_level="me",
+                uuid=(new_uuid := uuid.uuid4()),
+                fid=federation_utils.full_url(
+                    reverse(
+                        "federation:music:libraries-detail", kwargs={"uuid": new_uuid}
+                    ),
+                ),
+            )
+        if not self.privacy_level:
+            self.privacy_level = self.actor.user.privacy_level
+
+        return super().save(**kwargs)
 
     @transaction.atomic
     def insert(self, plt, index=None, allow_duplicates=True):
@@ -139,7 +202,7 @@ class Playlist(models.Model):
         max_tracks = preferences.get("playlists__max_tracks")
         if existing.count() + len(tracks) > max_tracks:
             raise exceptions.ValidationError(
-                "Playlist would reach the maximum of {} tracks".format(max_tracks)
+                f"Playlist would reach the maximum of {max_tracks} tracks"
             )
 
         if not allow_duplicates:
@@ -147,9 +210,20 @@ class Playlist(models.Model):
 
         self.save(update_fields=["modification_date"])
         start = total
+
         plts = [
             PlaylistTrack(
-                creation_date=now, playlist=self, track=track, index=start + i
+                creation_date=now,
+                playlist=self,
+                track=track,
+                index=start + i,
+                uuid=(new_uuid := uuid.uuid4()),
+                fid=federation_utils.full_url(
+                    reverse(
+                        f"federation:music:{self.federation_namespace}-detail",
+                        kwargs={"uuid": new_uuid},  # Use the newly generated UUID
+                    )
+                ),
             )
             for i, track in enumerate(tracks)
         ]
@@ -175,12 +249,58 @@ class Playlist(models.Model):
                 }
             )
 
+    def schedule_scan(self, actor, force=False):
+        """Update playlist tracks if playlist is a remote one. If it's a local playlist it send an update activity
+        on the remote server which will trigger a scan"""
 
-class PlaylistTrackQuerySet(models.QuerySet):
+        latest_scan = (
+            self.scans.exclude(status="errored").order_by("-creation_date").first()
+        )
+        delay_between_scans = datetime.timedelta(seconds=1)
+        now = timezone.now()
+        if (
+            not force
+            and latest_scan
+            and latest_scan.creation_date + delay_between_scans > now
+        ):
+            return
+
+        from . import tasks
+
+        scan = self.scans.create(
+            total_files=len(self.playlist_tracks.all()), actor=actor
+        )
+
+        if self.actor.is_local:
+            from funkwhale_api.federation import routes
+
+            routes.outbox.dispatch(
+                {"type": "Update", "object": {"type": "Playlist"}},
+                context={"playlist": self, "actor": self.actor},
+            )
+            scan.status = "finished"
+            return scan
+        else:
+            common_utils.on_commit(
+                tasks.start_playlist_scan.delay, playlist_scan_id=scan.pk
+            )
+            return scan
+
+
+@receiver(post_delete, sender=Playlist)
+def delete_playlist_library(sender, instance, **kwargs):
+    if instance.library:
+        instance.library.delete()
+
+
+class PlaylistTrackQuerySet(models.QuerySet, common_models.LocalFromFidQuerySet):
     def for_nested_serialization(self, actor=None):
         tracks = music_models.Track.objects.with_playable_uploads(actor)
-        tracks = tracks.select_related(
-            "artist", "album__artist", "album__attachment_cover", "attributed_to"
+        tracks = tracks.prefetch_related(
+            "artist_credit__artist",
+            "album__artist_credit__artist",
+            "album__attachment_cover",
+            "attributed_to",
         )
         return self.prefetch_related(
             models.Prefetch("track", queryset=tracks, to_attr="_prefetched_track")
@@ -197,7 +317,7 @@ class PlaylistTrackQuerySet(models.QuerySet):
         return self.annotate(is_playable_by_actor=subquery)
 
     def playable_by(self, actor, include=True):
-        tracks = music_models.Track.objects.playable_by(actor, include)
+        tracks = music_models.Track.objects.playable_by(actor)
         if include:
             return self.filter(track__pk__in=tracks).distinct()
         else:
@@ -213,7 +333,8 @@ class PlaylistTrackQuerySet(models.QuerySet):
         return PlaylistTrack.objects.get(pk=plt_id)
 
 
-class PlaylistTrack(models.Model):
+class PlaylistTrack(federation_models.FederationMixin):
+    uuid = models.UUIDField(default=uuid.uuid4, unique=True)
     track = models.ForeignKey(
         "music.Track", related_name="playlist_tracks", on_delete=models.CASCADE
     )
@@ -224,6 +345,7 @@ class PlaylistTrack(models.Model):
     creation_date = models.DateTimeField(default=timezone.now)
 
     objects = PlaylistTrackQuerySet.as_manager()
+    federation_namespace = "playlist-tracks"
 
     class Meta:
         ordering = ("-playlist", "index")
@@ -236,3 +358,37 @@ class PlaylistTrack(models.Model):
         if index is not None and update_indexes:
             playlist.remove(index)
         return r
+
+    def get_federation_id(self):
+        if self.fid:
+            return self.fid
+        return federation_utils.full_url(
+            reverse(
+                f"federation:music:{self.federation_namespace}-detail",
+                kwargs={"uuid": self.uuid},
+            )
+        )
+
+    def save(self, **kwargs):
+        if not self.pk and not self.fid:
+            self.fid = self.get_federation_id()
+
+        return super().save(**kwargs)
+
+    def get_absolute_url(self):
+        return f"/library/tracks/{self.track.pk}"
+
+
+class PlaylistScan(models.Model):
+    actor = models.ForeignKey(
+        "federation.Actor", null=True, blank=True, on_delete=models.CASCADE
+    )
+    playlist = models.ForeignKey(
+        Playlist, related_name="scans", on_delete=models.CASCADE
+    )
+    total_files = models.PositiveIntegerField(default=0)
+    processed_files = models.PositiveIntegerField(default=0)
+    errored_files = models.PositiveIntegerField(default=0)
+    status = models.CharField(default="pending", max_length=25)
+    creation_date = models.DateTimeField(default=timezone.now)
+    modification_date = models.DateTimeField(null=True, blank=True)

api/funkwhale_api/playlists/parsers.py

+from defusedxml.ElementTree import parse
+from rest_framework.parsers import BaseParser
+
+
+# from https://github.com/jpadilla/django-rest-framework-xml/blob/master/rest_framework_xml/parsers.py
+class XspfParser(BaseParser):
+    """
+    Takes a xspf stream, validate it, and return an xspf json
+    """
+
+    media_type = "application/octet-stream"
+
+    def parse(self, stream, media_type=None, parser_context=None):
+        playlist = {"tracks": []}
+
+        tree = parse(stream, forbid_dtd=True)
+        root = tree.getroot()
+
+        # Extract playlist information
+        playlist_info = root.find(".")
+        if playlist_info is not None:
+            playlist["title"] = playlist_info.findtext(
+                "{http://xspf.org/ns/0/}title", default=""
+            )
+            playlist["creator"] = playlist_info.findtext(
+                "{http://xspf.org/ns/0/}creator", default=""
+            )
+            playlist["creation_date"] = playlist_info.findtext(
+                "{http://xspf.org/ns/0/}date", default=""
+            )
+            playlist["version"] = playlist_info.attrib.get("version", "")
+
+        # Extract track information
+        for track in root.findall(".//{http://xspf.org/ns/0/}track"):
+            track_info = {
+                "location": track.findtext(
+                    "{http://xspf.org/ns/0/}location", default=""
+                ),
+                "title": track.findtext("{http://xspf.org/ns/0/}title", default=""),
+                "creator": track.findtext("{http://xspf.org/ns/0/}creator", default=""),
+                "album": track.findtext("{http://xspf.org/ns/0/}album", default=""),
+                "duration": track.findtext(
+                    "{http://xspf.org/ns/0/}duration", default=""
+                ),
+            }
+            playlist["tracks"].append(track_info)
+        return playlist

api/funkwhale_api/playlists/renderers.py

+import xml.etree.ElementTree as etree
+from xml.etree.ElementTree import Element, SubElement
+
+from defusedxml import minidom
+from rest_framework import renderers
+
+from funkwhale_api.playlists.models import Playlist
+
+
+class PlaylistXspfRenderer(renderers.BaseRenderer):
+    media_type = "application/octet-stream"
+    format = "xspf"
+
+    def render(self, data, accepted_media_type=None, renderer_context=None):
+        if isinstance(data, bytes):
+            return data
+
+        fw_playlist = Playlist.objects.get(uuid=data["uuid"])
+        plt_tracks = fw_playlist.playlist_tracks.prefetch_related("track")
+        top = Element("playlist", version="1", xmlns="http://xspf.org/ns/0/")
+        title_xspf = SubElement(top, "title")
+        title_xspf.text = fw_playlist.name
+        date_xspf = SubElement(top, "date")
+        date_xspf.text = fw_playlist.creation_date.isoformat()
+        trackList_xspf = SubElement(top, "trackList")
+
+        for plt_track in plt_tracks:
+            track = plt_track.track
+            write_xspf_track_data(track, trackList_xspf)
+        return prettify(top)
+
+
+def write_xspf_track_data(track, trackList_xspf):
+    """
+    Insert a track into the trackList subelement of a xspf file
+    """
+    track_xspf = SubElement(trackList_xspf, "track")
+    location_xspf = SubElement(track_xspf, "location")
+    location_xspf.text = "https://" + track.domain_name + track.listen_url
+    title_xspf = SubElement(track_xspf, "title")
+    title_xspf.text = str(track.title)
+    creator_xspf = SubElement(track_xspf, "creator")
+    creator_xspf.text = str(track.get_artist_credit_string)
+    if str(track.album) == "[non-album tracks]":
+        return
+    else:
+        album_xspf = SubElement(track_xspf, "album")
+        album_xspf.text = str(track.album)
+
+
+def prettify(elem):
+    """
+    Return a pretty-printed XML string for the Element.
+    """
+    rough_string = etree.tostring(elem, "utf-8")
+    reparsed = minidom.parseString(rough_string)
+    return reparsed.toprettyxml(indent="  ")

api/funkwhale_api/playlists/serializers.py

+import logging
+
+from django.core.exceptions import ObjectDoesNotExist, ValidationError
+from drf_spectacular.types import OpenApiTypes
+from drf_spectacular.utils import extend_schema_field
 from rest_framework import serializers
 
-from funkwhale_api.federation import serializers as federation_serializers
-from funkwhale_api.music.models import Track
+from funkwhale_api.common.fields import PRIVACY_LEVEL_CHOICES
+from funkwhale_api.federation.api_serializers import FullActorSerializer
+from funkwhale_api.music import tasks
+from funkwhale_api.music.models import Album, Artist, Track
 from funkwhale_api.music.serializers import TrackSerializer
-from funkwhale_api.users.serializers import UserBasicSerializer
 
 from . import models
 
+logger = logging.getLogger(__name__)
+
 
 class PlaylistTrackSerializer(serializers.ModelSerializer):
     # track = TrackSerializer()
@@ -25,16 +33,21 @@ class PlaylistSerializer(serializers.ModelSerializer):
     tracks_count = serializers.SerializerMethodField(read_only=True)
     duration = serializers.SerializerMethodField(read_only=True)
     album_covers = serializers.SerializerMethodField(read_only=True)
-    user = UserBasicSerializer(read_only=True)
     is_playable = serializers.SerializerMethodField()
-    actor = serializers.SerializerMethodField()
+    actor = FullActorSerializer(read_only=True)
+    library = serializers.SerializerMethodField()
+    library_followed = serializers.SerializerMethodField()
+    privacy_level = serializers.ChoiceField(
+        choices=PRIVACY_LEVEL_CHOICES, required=False
+    )
 
     class Meta:
         model = models.Playlist
         fields = (
-            "id",
+            "uuid",
+            "fid",
             "name",
-            "user",
+            "actor",
             "modification_date",
             "creation_date",
             "privacy_level",
@@ -43,34 +56,53 @@ class PlaylistSerializer(serializers.ModelSerializer):
             "duration",
             "is_playable",
             "actor",
+            "description",
+            "library",
+            "library_followed",
         )
-        read_only_fields = ["id", "modification_date", "creation_date"]
+        read_only_fields = ["uuid", "fid", "modification_date", "creation_date"]
 
-    def get_actor(self, obj):
-        actor = obj.user.actor
-        if actor:
-            return federation_serializers.APIActorSerializer(actor).data
+    @extend_schema_field(OpenApiTypes.URI)
+    def get_library(self, obj):
+        if obj.library:
+            return obj.library.fid
+        else:
+            return None
 
-    def get_is_playable(self, obj):
-        try:
-            return bool(obj.playable_plts)
-        except AttributeError:
+    @extend_schema_field(OpenApiTypes.BOOL)
+    def get_library_followed(self, obj):
+        if (
+            self.context.get("request", False)
+            and hasattr(self.context["request"], "user")
+            and hasattr(self.context["request"].user, "actor")
+        ):
+            actor = self.context["request"].user.actor
+            lib_qs = obj.library.received_follows.filter(actor=actor)
+
+            if lib_qs.exists():
+                if lib_qs[0].approved is None:
+                    return False
+                else:
+                    return lib_qs[0].approved
+            else:
+                return None
         return None
 
-    def get_tracks_count(self, obj):
-        try:
-            return obj.tracks_count
-        except AttributeError:
-            # no annotation?
-            return obj.playlist_tracks.count()
+    @extend_schema_field(OpenApiTypes.BOOL)
+    def get_is_playable(self, obj):
+        return getattr(obj, "is_playable_by_actor", False)
+
+    def get_tracks_count(self, obj) -> int:
+        return getattr(obj, "tracks_count", obj.playlist_tracks.count())
 
-    def get_duration(self, obj):
+    def get_duration(self, obj) -> int:
         try:
             return obj.duration
         except AttributeError:
             # no annotation?
             return 0
 
+    @extend_schema_field({"type": "array", "items": {"type": "string"}})
     def get_album_covers(self, obj):
         try:
             plts = obj.plts_for_cover
@@ -90,7 +122,9 @@ class PlaylistSerializer(serializers.ModelSerializer):
         covers = []
         max_covers = 5
         for plt in plts:
-            if plt.track.album.artist_id in excluded_artists:
+            if [
+                ac.artist.pk for ac in plt.track.album.artist_credit.all()
+            ] in excluded_artists:
                 continue
             url = plt.track.album.attachment_cover.download_url_medium_square_crop
             if url in covers:
@@ -115,3 +149,60 @@ class PlaylistAddManySerializer(serializers.Serializer):
 
     class Meta:
         fields = "allow_duplicates"
+
+
+class XspfTrackSerializer(serializers.Serializer):
+    location = serializers.CharField(allow_blank=True, required=False)
+    title = serializers.CharField()
+    creator = serializers.CharField()
+    album = serializers.CharField(allow_blank=True, required=False)
+    duration = serializers.CharField(allow_blank=True, required=False)
+
+    def validate(self, data):
+        title = data["title"]
+        album = data.get("album", None)
+        acs_tuples = tasks.parse_credits(data["creator"], "", 0)
+        try:
+            artist_id = Artist.objects.get(name=acs_tuples[0][0])
+        except ObjectDoesNotExist:
+            raise ValidationError("Couldn't find artist in the database")
+        if album:
+            try:
+                album_id = Album.objects.get(title=album)
+                fw_track = Track.objects.get(
+                    title=title, artist_credit__artist=artist_id, album=album_id
+                )
+            except ObjectDoesNotExist:
+                pass
+        try:
+            fw_track = Track.objects.get(title=title, artist_credit__artist=artist_id)
+        except ObjectDoesNotExist as e:
+            raise ValidationError(f"Couldn't find track in the database : {e!r}")
+
+        super().validate(data)
+        return fw_track
+
+
+class XspfSerializer(serializers.Serializer):
+    title = serializers.CharField()
+    creator = serializers.CharField(allow_blank=True, required=False)
+    creation_date = serializers.DateTimeField(required=False)
+    version = serializers.IntegerField(required=False)
+    tracks = XspfTrackSerializer(many=True, required=False)
+
+    def create(self, validated_data):
+        pl = models.Playlist.objects.create(
+            name=validated_data["title"],
+            privacy_level="private",
+            actor=validated_data["request"].user.actor,
+        )
+        pl.insert_many(validated_data["tracks"])
+
+        return pl
+
+    def update(self, instance, validated_data):
+        instance.name = validated_data["title"]
+        instance.playlist_tracks.all().delete()
+        instance.insert_many(validated_data["tracks"])
+        instance.save()
+        return instance

api/funkwhale_api/playlists/tasks.py

+import logging
+
+import requests
+from django.db.models import F
+from django.utils import timezone
+from requests.exceptions import RequestException
+
+from funkwhale_api.common import session
+from funkwhale_api.federation import serializers, signing
+from funkwhale_api.taskapp import celery
+
+from . import models
+
+logger = logging.getLogger(__name__)
+
+
+def get_playlist_data(playlist_url, actor):
+    auth = signing.get_auth(actor.private_key, actor.private_key_id)
+    try:
+        response = session.get_session().get(
+            playlist_url,
+            auth=auth,
+            headers={"Accept": "application/activity+json"},
+        )
+    except requests.ConnectionError:
+        return {"errors": ["This playlist is not reachable"]}
+    scode = response.status_code
+    if scode == 401:
+        return {"errors": ["This playlist requires authentication"]}
+    elif scode == 403:
+        return {
+            "errors": [
+                f"Permission denied while scanning playlist. Error : {scode}. PLaylist url = {playlist_url}"
+            ]
+        }
+    elif scode >= 400:
+        return {"errors": [f"Error {scode} while fetching the playlist"]}
+    serializer = serializers.PlaylistCollectionSerializer(data=response.json())
+
+    if not serializer.is_valid():
+        return {"errors": ["Invalid ActivityPub response from remote playlist"]}
+
+    return serializer.validated_data
+
+
+def get_playlist_page(playlist, page_url, actor):
+    auth = signing.get_auth(actor.private_key, actor.private_key_id)
+    response = session.get_session().get(
+        page_url,
+        auth=auth,
+        headers={"Accept": "application/activity+json"},
+    )
+    serializer = serializers.CollectionPageSerializer(
+        data=response.json(),
+        context={
+            "playlist": playlist,
+            "item_serializer": serializers.PlaylistTrackSerializer,
+            "conf": {"library": playlist.library},
+        },
+    )
+    serializer.is_valid(raise_exception=True)
+    return serializer.validated_data
+
+
+@celery.app.task(name="playlist.start_playlist_scan")
+@celery.require_instance(
+    models.PlaylistScan.objects.select_related().filter(status="pending"),
+    "playlist_scan",
+)
+def start_playlist_scan(playlist_scan):
+    playlist_scan.playlist.playlist_tracks.all().delete()
+    playlist_scan.playlist.library.uploads.all().delete()
+    try:
+        data = get_playlist_data(playlist_scan.playlist.fid, actor=playlist_scan.actor)
+    except Exception:
+        playlist_scan.status = "errored"
+        playlist_scan.save(update_fields=["status", "modification_date"])
+        raise
+    if "errors" in data.keys():
+        playlist_scan.status = "errored"
+        playlist_scan.save(update_fields=["status", "modification_date"])
+        raise Exception("Error from remote server : " + str(data))
+    playlist_scan.modification_date = timezone.now()
+    playlist_scan.status = "scanning"
+    playlist_scan.total_files = data["totalItems"]
+
+    playlist_scan.save(update_fields=["status", "modification_date", "total_files"])
+    scan_playlist_page.delay(playlist_scan_id=playlist_scan.pk, page_url=data["first"])
+
+
+@celery.app.task(
+    name="playlist.scan_playlist_page",
+    retry_backoff=60,
+    max_retries=5,
+    autoretry_for=[RequestException],
+)
+@celery.require_instance(
+    models.PlaylistScan.objects.select_related().filter(status="scanning"),
+    "playlist_scan",
+)
+def scan_playlist_page(playlist_scan, page_url):
+    data = get_playlist_page(playlist_scan.playlist, page_url, playlist_scan.actor)
+    plts = []
+    for item_serializer in data["items"]:
+        try:
+            plt = item_serializer.save(playlist=playlist_scan.playlist.fid)
+            # we get any upload owned by the playlist.actor and add a m2m with playlist_libraries
+            upload_qs = plt.track.uploads.filter(
+                library__actor=playlist_scan.playlist.actor
+            )
+            if not upload_qs:
+                logger.debug(
+                    f"Could not find a upload for the playlist track {plt.track.title}. Probably the \
+                    playlist.library library_scan failed or was not launched by inbox_update_playlist ?"
+                )
+            else:
+                upload_qs[0].playlist_libraries.add(playlist_scan.playlist.library)
+                logger.debug(f"Added {plt.track.title} to playlist library")
+            plts.append(plt)
+        except Exception as e:
+            logger.info(
+                f"Error while saving track to playlist {playlist_scan.playlist}: {e}"
+            )
+            continue
+
+    playlist_scan.processed_files = F("processed_files") + len(plts)
+    playlist_scan.modification_date = timezone.now()
+    update_fields = ["modification_date", "processed_files"]
+
+    next_page = data.get("next")
+    fetch_next = next_page and next_page != page_url
+
+    if not fetch_next:
+        update_fields.append("status")
+        playlist_scan.status = "finished"
+    playlist_scan.save(update_fields=update_fields)
+
+    if fetch_next:
+        scan_playlist_page.delay(playlist_scan_id=playlist_scan.pk, page_url=next_page)

api/funkwhale_api/playlists/views.py

+import logging
+from itertools import chain
+
+from django.conf import settings
 from django.db import transaction
 from django.db.models import Count
-from rest_framework import exceptions, mixins, viewsets
+from drf_spectacular.utils import OpenApiParameter, OpenApiTypes, extend_schema
+from rest_framework import exceptions, mixins, status, viewsets
 from rest_framework.decorators import action
+from rest_framework.pagination import PageNumberPagination
+from rest_framework.parsers import FormParser, JSONParser, MultiPartParser
+from rest_framework.renderers import JSONRenderer
 from rest_framework.response import Response
 
+from config import plugins
 from funkwhale_api.common import fields, permissions
+from funkwhale_api.federation import routes
+from funkwhale_api.music import models as music_models
+from funkwhale_api.music import serializers as music_serializers
 from funkwhale_api.music import utils as music_utils
 from funkwhale_api.users.oauth import permissions as oauth_permissions
 
-from . import filters, models, serializers
+from . import filters, models, parsers, renderers, serializers
+
+logger = logging.getLogger(__name__)
 
 
 class PlaylistViewSet(
@@ -19,11 +33,11 @@ class PlaylistViewSet(
     mixins.ListModelMixin,
     viewsets.GenericViewSet,
 ):
-
+    lookup_field = "uuid"
     serializer_class = serializers.PlaylistSerializer
     queryset = (
         models.Playlist.objects.all()
-        .select_related("user__actor__attachment_icon")
+        .select_related("actor__attachment_icon")
         .annotate(tracks_count=Count("playlist_tracks", distinct=True))
         .with_covers()
         .with_duration()
@@ -35,19 +49,110 @@ class PlaylistViewSet(
     required_scope = "playlists"
     anonymous_policy = "setting"
     owner_checks = ["write"]
+    owner_field = "actor.user"
     filterset_class = filters.PlaylistFilter
     ordering_fields = ("id", "name", "creation_date", "modification_date")
+    parser_classes = [parsers.XspfParser, JSONParser, FormParser, MultiPartParser]
+    renderer_classes = [JSONRenderer, renderers.PlaylistXspfRenderer]
 
+    def update(self, request, *args, **kwargs):
+        playlist = self.get_object()
+        content_type = request.headers.get("Content-Type")
+        if content_type and "application/octet-stream" in content_type:
+            tracks = []
+            for track_data in request.data.get("tracks", []):
+                track_serializer = serializers.XspfTrackSerializer(data=track_data)
+                if track_serializer.is_valid():
+                    tracks.append(track_serializer.validated_data)
+                else:
+                    request.data["tracks"].remove(track_data)
+                    logger.info(
+                        f"Removing track {track_data} because we didn't find a match in db"
+                    )
+
+            serializer = serializers.XspfSerializer(
+                playlist, data=request.data, partial=True
+            )
+            serializer.is_valid(raise_exception=True)
+            pl = serializer.save()
+            routes.outbox.dispatch(
+                {"type": "Update", "object": {"type": "Playlist"}},
+                context={"playlist": pl, "actor": playlist.actor},
+            )
+            return Response(serializers.PlaylistSerializer(pl).data, status=201)
+
+        response = super().update(request, *args, **kwargs)
+        routes.outbox.dispatch(
+            {"type": "Update", "object": {"type": "Playlist"}},
+            context={"playlist": self.get_object(), "actor": playlist.actor},
+        )
+        return response
+
+    def create(self, request, *args, **kwargs):
+        content_type = request.headers.get("Content-Type")
+        if content_type and "application/octet-stream" in content_type:
+            #  We check if tracks are in the db, and exclude the ones we don't find
+            for track_data in list(request.data.get("tracks", [])):
+                track_serializer = serializers.XspfTrackSerializer(data=track_data)
+                if not track_serializer.is_valid():
+                    request.data["tracks"].remove(track_data)
+                    logger.info(
+                        f"Removing track {track_data} because we didn't find a match in db"
+                    )
+
+            serializer = serializers.XspfSerializer(data=request.data)
+            serializer.is_valid(raise_exception=True)
+            pl = serializer.save(request=request)
+            return Response(serializers.PlaylistSerializer(pl).data, status=201)
+
+        serializer = self.get_serializer(data=request.data)
+        serializer.is_valid(raise_exception=True)
+        playlist = self.perform_create(serializer)
+        headers = self.get_success_headers(serializer.data)
+        routes.outbox.dispatch(
+            {"type": "Create", "object": {"type": "Playlist"}},
+            context={"playlist": playlist, "actor": playlist.actor},
+        )
+        return Response(
+            serializer.data, status=status.HTTP_201_CREATED, headers=headers
+        )
+
+    def destroy(self, request, *args, **kwargs):
+        playlist = self.get_object()
+        self.perform_destroy(playlist)
+        routes.outbox.dispatch(
+            {"type": "Delete", "object": {"type": "Playlist"}},
+            context={"playlist": playlist, "actor": playlist.actor},
+        )
+        return Response(status=status.HTTP_204_NO_CONTENT)
+
+    @extend_schema(responses=serializers.PlaylistTrackSerializer(many=True))
     @action(methods=["get"], detail=True)
     def tracks(self, request, *args, **kwargs):
         playlist = self.get_object()
-        plts = playlist.playlist_tracks.all().for_nested_serialization(
-            music_utils.get_actor_from_request(request)
+        actor = music_utils.get_actor_from_request(request)
+        plts = playlist.playlist_tracks.all().for_nested_serialization(actor)
+        for plt in plts.playable_by(actor, include=False)[
+            : settings.THIRD_PARTY_UPLOAD_MAX_UPLOADS
+        ]:
+            plugins.trigger_hook(
+                plugins.TRIGGER_THIRD_PARTY_UPLOAD,
+                track=plt.track,
+                actor=actor,
             )
-        serializer = serializers.PlaylistTrackSerializer(plts, many=True)
-        data = {"count": len(plts), "results": serializer.data}
-        return Response(data, status=200)
 
+        # Apply pagination
+        paginator = PageNumberPagination()
+        paginator.page_size = 50  # Set the page size (number of items per page)
+        paginated_plts = paginator.paginate_queryset(plts, request)
+
+        # Serialize the paginated data
+        serializer = serializers.PlaylistTrackSerializer(paginated_plts, many=True)
+        return paginator.get_paginated_response(serializer.data)
+
+    @extend_schema(
+        operation_id="add_to_playlist", request=serializers.PlaylistAddManySerializer
+    )
     @action(methods=["post"], detail=True)
     @transaction.atomic
     def add(self, request, *args, **kwargs):
@@ -70,29 +175,40 @@ class PlaylistViewSet(
         )
         serializer = serializers.PlaylistTrackSerializer(plts, many=True)
         data = {"count": len(plts), "results": serializer.data}
+        update_playlist_library_uploads(playlist, plts)
+        playlist.schedule_scan(playlist.actor, force=True)
         return Response(data, status=201)
 
+    @extend_schema(operation_id="clear_playlist")
     @action(methods=["delete"], detail=True)
     @transaction.atomic
     def clear(self, request, *args, **kwargs):
         playlist = self.get_object()
         playlist.playlist_tracks.all().delete()
         playlist.save(update_fields=["modification_date"])
+        playlist.library.uploads.filter().delete()
+        playlist.schedule_scan(playlist.actor, force=True)
         return Response(status=204)
 
     def get_queryset(self):
         return self.queryset.filter(
-            fields.privacy_level_query(self.request.user)
+            fields.privacy_level_query(
+                self.request.user, "privacy_level", "actor__user"
+            )
         ).with_playable_plts(music_utils.get_actor_from_request(self.request))
 
     def perform_create(self, serializer):
         return serializer.save(
-            user=self.request.user,
+            actor=self.request.user.actor,
             privacy_level=serializer.validated_data.get(
                 "privacy_level", self.request.user.privacy_level
             ),
         )
 
+    @extend_schema(
+        operation_id="remove_from_playlist",
+        parameters=[OpenApiParameter("index", OpenApiTypes.INT, location="query")],
+    )
     @action(methods=["post", "delete"], detail=True)
     @transaction.atomic
     def remove(self, request, *args, **kwargs):
@@ -107,10 +223,13 @@ class PlaylistViewSet(
             plt = playlist.playlist_tracks.by_index(index)
         except models.PlaylistTrack.DoesNotExist:
             return Response(status=404)
+        for upload in plt.track.uploads.filter(playlist_libraries=playlist.library):
+            upload.playlist_libraries.remove(playlist.library)
         plt.delete(update_indexes=True)
-
+        plt.playlist.schedule_scan(playlist.actor)
         return Response(status=204)
 
+    @extend_schema(operation_id="reorder_track_in_playlist")
     @action(methods=["post"], detail=True)
     @transaction.atomic
     def move(self, request, *args, **kwargs):
@@ -132,4 +251,45 @@ class PlaylistViewSet(
         except models.PlaylistTrack.DoesNotExist:
             return Response(status=404)
         playlist.insert(plt, to_index)
+        plt.playlist.schedule_scan(playlist.actor)
         return Response(status=204)
+
+    @extend_schema(operation_id="get_playlist_albums")
+    @action(methods=["get"], detail=True)
+    @transaction.atomic
+    def albums(self, request, *args, **kwargs):
+        playlist = self.get_object()
+        try:
+            albums_pks = playlist.playlist_tracks.values_list(
+                "track__album__pk", flat=True
+            ).distinct()
+        except models.PlaylistTrack.DoesNotExist:
+            return Response(status=404)
+        releases = music_models.Album.objects.filter(pk__in=albums_pks)
+        serializer = music_serializers.AlbumSerializer(releases, many=True)
+        return Response(serializer.data, status=200)
+
+    @extend_schema(operation_id="get_playlist_artists")
+    @action(methods=["get"], detail=True)
+    @transaction.atomic
+    def artists(self, request, *args, **kwargs):
+        playlist = self.get_object()
+        try:
+            artists_pks = playlist.playlist_tracks.values_list(
+                "track__artist_credit__artist__pk", flat=True
+            ).distinct()
+        except models.PlaylistTrack.DoesNotExist:
+            return Response(status=404)
+        artists = music_models.Artist.objects.filter(pk__in=artists_pks)
+        serializer = music_serializers.ArtistSerializer(artists, many=True)
+        return Response(serializer.data, status=200)
+
+
+def update_playlist_library_uploads(playlist, plts):
+    uploads = list(
+        chain(
+            *[plt.track.uploads.filter(library__actor=playlist.actor) for plt in plts]
+        )
+    )
+    for upload in uploads:
+        upload.playlist_libraries.add(playlist.library)