Merge branch 'develop'

CHANGELOG

@@ -10,6 +10,168 @@ This changelog is viewable on the web at https://docs.funkwhale.audio/changelog.
 
 .. towncrier
 
+1.1 (unreleased)
+----------------
+
+Upgrade instructions are available at
+https://docs.funkwhale.audio/admin/upgrading.html
+
+Enhancements:
+
+- Add number of tracks and discs of an album to API (#1238)
+- Add spacing after "Play all" button in playlist view (!1271)
+- Added a ListenBrainz plugin to submit listenings
+- Added ability to choose fediverse addresses from channel subscription page/podcast screen (#1294)
+- Added new search functions to allow users to more easily search for podcasts in the UI.
+- Added padding to volume slider to ease mouse control (#1241)
+- Logarithmic scale for volume slider (#1222)
+- More user-friendly subsonic tokens (#1269)
+- Remove manual entry of Import Reference on front-end import (#1284)
+- Support AIFF file format (#1243)
+
+
+Bugfixes:
+
+- "Add check for empty/null covers (#1281)"
+- Added an album filter to fix problem where channel entries would show up in the wrong series (#1282)
+- Avoid broken Faker version (#1323)
+- Changed audio format detection to happen via sniffing and not file extensions (#1274)
+- Changed default behaviour of channel entries to use channel artwork if no entry artwork available (#1289)
+- Fix delete library modal closing immediately (#1272)
+- Fix public shared remote library radio button being disabled (#1292)
+- Fixed an issue that prevented disabling plugins
+- Fixed an issue where channel albums don't show up in the album search (#1300)
+- Fixed an issue where modals would prevent users being able to interact with channels (#1295)
+- Update MediaSession metadata for initially loaded track (#1252)
+- Update playback position slider also when track is paused (#1266)
+- Fixed follows from Pleroma with custom Emoji as Tag by ignoring not supported tag types #1342
+- Update pleroma JSON-LD Schema (#1341)
+- Pin twisted version to 20.3.0
+
+Contributors to this release (development, documentation, reviews):
+
+Adam Novak
+Agate
+alemairebe
+Alicia Blasco Leon
+anonymous
+Amaranthe
+appzer0
+Arne
+Asier Iturralde Sarasola
+Christian Paul
+Ciarán Ainsworth
+Daniel
+David
+Dominik Danelski
+Eorn le goéland
+Eleos
+Erik Duxstad
+Esteban
+Fred Uggla
+Freyja Wildes
+Georg Krause
+ghose
+hellekin
+heyarne
+interfect
+Jess Jing
+Johannes H.
+jovuit
+marzzzello
+Meliurwen
+Mehdi
+Nitai Bezerra da Silva
+Philipp Wolfer
+Pierre Couy
+Porrumentzio
+Reg
+Robert Kaye
+Rubén Cabrera
+Silver Fox
+Snack Capt
+SpcCw
+Strom Lin
+vicdorke
+x
+
+
+1.1-rc2 (2021-03-01)
+--------------------
+
+Upgrade instructions are available at
+https://docs.funkwhale.audio/admin/upgrading.html
+
+Bugfixes:
+
+- Fixed follows from Pleroma with custom Emoji as Tag by ignoring not supported tag types #1342
+- Update pleroma JSON-LD Schema (#1341)
+- Revert fork replacement of http-signature since official package breaks federation
+- Pin twisted version to 20.3.0
+
+
+1.1-rc1 (2021-02-24)
+--------------------
+
+Upgrade instructions are available at
+https://docs.funkwhale.audio/admin/upgrading.html
+
+Enhancements:
+
+- Add number of tracks and discs of an album to API (#1238)
+- Add spacing after "Play all" button in playlist view (!1271)
+- Added a ListenBrainz plugin to submit listenings
+- Added ability to choose fediverse addresses from channel subscription page/podcast screen (#1294)
+- Added new search functions to allow users to more easily search for podcasts in the UI.
+- Added padding to volume slider to ease mouse control (#1241)
+- Logarithmic scale for volume slider (#1222)
+- More user-friendly subsonic tokens (#1269)
+- Remove manual entry of Import Reference on front-end import (#1284)
+- Replaced forked http-signature dependency with official package (#876)
+- Support AIFF file format (#1243)
+
+
+Bugfixes:
+
+- "Add check for empty/null covers (#1281)"
+- Added an album filter to fix problem where channel entries would show up in the wrong series (#1282)
+- Avoid broken Faker version (#1323)
+- Changed audio format detection to happen via sniffing and not file extensions (#1274)
+- Changed default behaviour of channel entries to use channel artwork if no entry artwork available (#1289)
+- Fix delete library modal closing immediately (#1272)
+- Fix public shared remote library radio button being disabled (#1292)
+- Fixed an issue that prevented disabling plugins
+- Fixed an issue where channel albums don't show up in the album search (#1300)
+- Fixed an issue where modals would prevent users being able to interact with channels (#1295)
+- Update MediaSession metadata for initially loaded track (#1252)
+- Update playback position slider also when track is paused (#1266)
+
+Contributors to this release (development, documentation, reviews):
+
+- Reg
+- hellekin
+- Esteban
+- Freyja Wildes
+- Amaranthe
+- Eleos
+- Johannes H.
+- Mehdi
+- Adam Novak
+- Agate
+- Christian Paul
+- Ciarán Ainsworth
+- Erik Duxstad
+- Fred Uggla
+- Georg Krause
+- heyarne
+- interfect
+- jovuit
+- Nitai Bezerra da Silva
+- Philipp Wolfer
+- Pierre Couy
+- Robert Kaye
+- Strom Lin
+
 1.0.1 (2020-10-31)
 ------------------
 

CONTRIBUTING.rst

@@ -199,7 +199,7 @@ Once everything is up, you can access the various funkwhale's components:
 
 - The Vue webapp, on http://localhost:8000
 - The API, on http://localhost:8000/api/v1/
-- The django admin, on http://localhost:800/api/admin/
+- The django admin, on http://localhost:8000/api/admin/
 
 Stopping everything
 ^^^^^^^^^^^^^^^^^^^
@@ -687,7 +687,7 @@ useful when testing components that depend on each other:
 
     def test_downgrade_not_superuser_skips_email(factories, mocker):
         mocked_notify = mocker.patch('funkwhale_api.myapp.notifications.notify')
-        user = factories['users.User'](is_superuser=True)
+        user = factories['users.User'](is_superuser=False)
         users.downgrade_user(user)
 
         # here, we ensure no email was sent

README.rst

@@ -12,15 +12,7 @@ LICENSE: AGPL3
 Getting help
 ------------
 
-We offer various Matrix.org rooms to discuss about Funkwhale:
-
-- `#funkwhale:matrix.org <https://matrix.to/#/#funkwhale:matrix.org>`_ for general questions about funkwhale
-- `#funkwhale-dev:matrix.org <https://matrix.to/#/#funkwhale-dev:matrix.org>`_ for development-focused discussion
-
-Please join those rooms if you have any questions!
-
-You can also contact `@funkwhale@mastodon.eliotberriot.com <https://mastodon.eliotberriot.com/@funkwhale>`_ on the fediverse.
-
+There are several places to get help or get in touch with other members of the community: https://funkwhale.audio/community/
 
 Contribute
 ----------
@@ -33,7 +25,7 @@ Security issues and vulnerabilities
 
 If you found a vulnerability in Funkwhale, please report it on our Gitlab instance at `https://dev.funkwhale.audio/funkwhale/funkwhale/-/issues`_, ensuring
 you have checked the ``This issue is confidential and should only be visible to team members with at least Reporter access.
-`` box. 
+`` box.
 
 This will ensure only maintainers and developpers have access to the vulnerability. Thank you for your help!
 

api/config/settings/common.py

@@ -94,6 +94,7 @@ Path to a directory containing Funkwhale plugins. These will be imported at runt
 sys.path.append(FUNKWHALE_PLUGINS_PATH)
 CORE_PLUGINS = [
     "funkwhale_api.contrib.scrobbler",
+    "funkwhale_api.contrib.listenbrainz",
 ]
 
 LOAD_CORE_PLUGINS = env.bool("FUNKWHALE_LOAD_CORE_PLUGINS", default=True)

api/funkwhale_api/__init__.py

 # -*- coding: utf-8 -*-
-__version__ = "1.0.1"
+__version__ = "1.1"
 __version_info__ = tuple(
     [
         int(num) if num.isdigit() else num

api/funkwhale_api/common/dynamic_preferences_registry.py

@@ -11,7 +11,7 @@ class APIAutenticationRequired(types.BooleanPreference):
     verbose_name = "API Requires authentication"
     default = True
     help_text = (
-        "If disabled, anonymous users will be able to query the API"
+        "If disabled, anonymous users will be able to query the API "
         "and access music data (as well as other data exposed in the API "
         "without specific permissions)."
     )

api/funkwhale_api/contrib/listenbrainz/client.py

+# Copyright (c) 2018 Philipp Wolfer <ph.wolfer@gmail.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+import json
+import logging
+import ssl
+import time
+from http.client import HTTPSConnection
+
+HOST_NAME = "api.listenbrainz.org"
+PATH_SUBMIT = "/1/submit-listens"
+SSL_CONTEXT = ssl.create_default_context()
+
+
+class Track:
+    """
+    Represents a single track to submit.
+
+    See https://listenbrainz.readthedocs.io/en/latest/dev/json.html
+    """
+
+    def __init__(self, artist_name, track_name, release_name=None, additional_info={}):
+        """
+        Create a new Track instance
+        @param artist_name as str
+        @param track_name as str
+        @param release_name as str
+        @param additional_info as dict
+        """
+        self.artist_name = artist_name
+        self.track_name = track_name
+        self.release_name = release_name
+        self.additional_info = additional_info
+
+    @staticmethod
+    def from_dict(data):
+        return Track(
+            data["artist_name"],
+            data["track_name"],
+            data.get("release_name", None),
+            data.get("additional_info", {}),
+        )
+
+    def to_dict(self):
+        return {
+            "artist_name": self.artist_name,
+            "track_name": self.track_name,
+            "release_name": self.release_name,
+            "additional_info": self.additional_info,
+        }
+
+    def __repr__(self):
+        return "Track(%s, %s)" % (self.artist_name, self.track_name)
+
+
+class ListenBrainzClient:
+    """
+    Submit listens to ListenBrainz.org.
+
+    See https://listenbrainz.readthedocs.io/en/latest/dev/api.html
+    """
+
+    def __init__(self, user_token, logger=logging.getLogger(__name__)):
+        self.__next_request_time = 0
+        self.user_token = user_token
+        self.logger = logger
+
+    def listen(self, listened_at, track):
+        """
+        Submit a listen for a track
+        @param listened_at as int
+        @param entry as Track
+        """
+        payload = _get_payload(track, listened_at)
+        return self._submit("single", [payload])
+
+    def playing_now(self, track):
+        """
+        Submit a playing now notification for a track
+        @param track as Track
+        """
+        payload = _get_payload(track)
+        return self._submit("playing_now", [payload])
+
+    def import_tracks(self, tracks):
+        """
+        Import a list of tracks as (listened_at, Track) pairs
+        @param track as [(int, Track)]
+        """
+        payload = _get_payload_many(tracks)
+        return self._submit("import", payload)
+
+    def _submit(self, listen_type, payload, retry=0):
+        self._wait_for_ratelimit()
+        self.logger.debug("ListenBrainz %s: %r", listen_type, payload)
+        data = {"listen_type": listen_type, "payload": payload}
+        headers = {
+            "Authorization": "Token %s" % self.user_token,
+            "Content-Type": "application/json",
+        }
+        body = json.dumps(data)
+        conn = HTTPSConnection(HOST_NAME, context=SSL_CONTEXT)
+        conn.request("POST", PATH_SUBMIT, body, headers)
+        response = conn.getresponse()
+        response_text = response.read()
+        try:
+            response_data = json.loads(response_text)
+        except json.decoder.JSONDecodeError:
+            response_data = response_text
+
+        self._handle_ratelimit(response)
+        log_msg = "Response %s: %r" % (response.status, response_data)
+        if response.status == 429 and retry < 5:  # Too Many Requests
+            self.logger.warning(log_msg)
+            return self._submit(listen_type, payload, retry + 1)
+        elif response.status == 200:
+            self.logger.debug(log_msg)
+        else:
+            self.logger.error(log_msg)
+        return response
+
+    def _wait_for_ratelimit(self):
+        now = time.time()
+        if self.__next_request_time > now:
+            delay = self.__next_request_time - now
+            self.logger.debug("Rate limit applies, delay %d", delay)
+            time.sleep(delay)
+
+    def _handle_ratelimit(self, response):
+        remaining = int(response.getheader("X-RateLimit-Remaining", 0))
+        reset_in = int(response.getheader("X-RateLimit-Reset-In", 0))
+        self.logger.debug("X-RateLimit-Remaining: %i", remaining)
+        self.logger.debug("X-RateLimit-Reset-In: %i", reset_in)
+        if remaining == 0:
+            self.__next_request_time = time.time() + reset_in
+
+
+def _get_payload_many(tracks):
+    payload = []
+    for (listened_at, track) in tracks:
+        data = _get_payload(track, listened_at)
+        payload.append(data)
+    return payload
+
+
+def _get_payload(track, listened_at=None):
+    data = {"track_metadata": track.to_dict()}
+    if listened_at is not None:
+        data["listened_at"] = listened_at
+    return data

api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py

+from config import plugins
+from .funkwhale_startup import PLUGIN
+from .client import ListenBrainzClient, Track
+
+
+@plugins.register_hook(plugins.LISTENING_CREATED, PLUGIN)
+def submit_listen(listening, conf, **kwargs):
+    user_token = conf["user_token"]
+    if not user_token:
+        return
+
+    logger = PLUGIN["logger"]
+    logger.info("Submitting listen to ListenBrainz")
+    client = ListenBrainzClient(user_token=user_token, logger=logger)
+    track = get_track(listening.track)
+    client.listen(int(listening.creation_date.timestamp()), track)
+
+
+def get_track(track):
+    artist = track.artist.name
+    title = track.title
+    album = None
+    additional_info = {
+        "listening_from": "Funkwhale",
+        "tracknumber": track.position,
+        "discnumber": track.disc_number,
+    }
+
+    if track.mbid:
+        additional_info["recording_mbid"] = str(track.mbid)
+
+    if track.album:
+        if track.album.title:
+            album = track.album.title
+        if track.album.mbid:
+            additional_info["release_mbid"] = str(track.album.mbid)
+
+    if track.artist.mbid:
+        additional_info["artist_mbids"] = [str(track.artist.mbid)]
+
+    return Track(artist, title, album, additional_info)

api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py

+from config import plugins
+
+
+PLUGIN = plugins.get_plugin_config(
+    name="listenbrainz",
+    label="ListenBrainz",
+    description="A plugin that allows you to submit your listens to ListenBrainz.",
+    version="0.1",
+    user=True,
+    conf=[
+        {
+            "name": "user_token",
+            "type": "text",
+            "label": "Your ListenBrainz user token",
+            "help": "You can find your user token in your ListenBrainz profile at https://listenbrainz.org/profile/",
+        }
+    ],
+)

api/funkwhale_api/federation/contexts.py

@@ -316,8 +316,8 @@ CONTEXTS = [
         "shortId": "LITEPUB",
         "contextUrl": None,
         "documentUrl": "http://litepub.social/ns",
+        # from https://git.pleroma.social/pleroma/pleroma/-/blob/release/2.2.3/priv/static/schemas/litepub-0.1.jsonld
         "document": {
-            # from https://ap.thequietplace.social/schemas/litepub-0.1.jsonld
             "@context": {
                 "Emoji": "toot:Emoji",
                 "Hashtag": "as:Hashtag",
@@ -326,6 +326,7 @@ CONTEXTS = [
                 "conversation": {"@id": "ostatus:conversation", "@type": "@id"},
                 "discoverable": "toot:discoverable",
                 "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
+                "capabilities": "litepub:capabilities",
                 "ostatus": "http://ostatus.org#",
                 "schema": "http://schema.org#",
                 "toot": "http://joinmastodon.org/ns#",
@@ -340,6 +341,7 @@ CONTEXTS = [
                     "@type": "@id",
                 },
                 "EmojiReact": "litepub:EmojiReact",
+                "ChatMessage": "litepub:ChatMessage",
                 "alsoKnownAs": {"@id": "as:alsoKnownAs", "@type": "@id"},
             }
         },

api/funkwhale_api/federation/serializers.py

@@ -258,9 +258,15 @@ class ActorSerializer(jsonld.JsonLdSerializer):
     )
     attributedTo = serializers.URLField(max_length=500, required=False)
 
-    tags = serializers.ListField(
-        child=TagSerializer(), min_length=0, required=False, allow_null=True
-    )
+    tags = serializers.ListField(min_length=0, required=False, allow_null=True)
+
+    def validate_tags(self, tags):
+        valid_tags = []
+        for tag in tags:
+            s = TagSerializer(data=tag)
+            if s.is_valid():
+                valid_tags.append(s.validated_data)
+        return valid_tags
 
     category = serializers.CharField(required=False)
     # languages = serializers.Char(

api/funkwhale_api/music/dynamic_preferences_registry.py

@@ -27,7 +27,7 @@ class MusicCacheDuration(types.IntPreference):
     default = 60 * 24 * 7
     verbose_name = "Transcoding cache duration"
     help_text = (
-        "How much minutes do you want to keep a copy of transcoded tracks "
+        "How many minutes do you want to keep a copy of transcoded tracks "
         "on the server? Transcoded files that were not listened in this interval "
         "will be erased and retranscoded on the next listening."
     )

api/funkwhale_api/music/filters.py

@@ -103,6 +103,7 @@ class ArtistFilter(
     playable = filters.BooleanFilter(field_name="_", method="filter_playable")
     has_albums = filters.BooleanFilter(field_name="_", method="filter_has_albums")
     tag = TAG_FILTER
+    content_category = filters.CharFilter("content_category")
     scope = common_filters.ActorScopeFilter(
         actor_field="tracks__uploads__library__actor",
         distinct=True,
@@ -257,6 +258,7 @@ class AlbumFilter(
         search_fields=["title", "artist__name"],
         fts_search_fields=["body_text", "artist__body_text"],
     )
+    content_category = filters.CharFilter("artist__content_category")
     tag = TAG_FILTER
     scope = common_filters.ActorScopeFilter(
         actor_field="tracks__uploads__library__actor",

api/funkwhale_api/music/management/commands/import_files.py

@@ -17,6 +17,7 @@ from django.core.files import File
 from django.core.management import call_command
 from django.core.management.base import BaseCommand, CommandError
 from django.db.models import Q
+from django.db.utils import IntegrityError
 from django.utils import timezone
 
 from rest_framework import serializers
@@ -832,6 +833,13 @@ def check_upload(stdout, upload):
             except serializers.ValidationError as e:
                 stdout.write("  Invalid metadata: {}".format(e))
                 return
+            except IntegrityError:
+                stdout.write(
+                    "  Duplicate key violation for metadata. Skipping...\n{}".format(
+                        upload.source
+                    )
+                )
+                return
             else:
                 upload.checksum = checksum
                 upload.save(update_fields=["checksum"])

api/funkwhale_api/music/metadata.py

@@ -253,7 +253,7 @@ CONF = {
             "comment": {"field": "comment"},
         },
     },
-    "MP3": {
+    "ID3": {
         "getter": get_id3_tag,
         "clean_pictures": clean_id3_pictures,
         "fields": {
@@ -331,6 +331,9 @@ CONF = {
     },
 }
 
+CONF["MP3"] = CONF["ID3"]
+CONF["AIFF"] = CONF["ID3"]
+
 ALL_FIELDS = [
     "position",
     "disc_number",

api/funkwhale_api/music/models.py

@@ -20,6 +20,7 @@ from django.db.models.signals import post_save, pre_save
 from django.dispatch import receiver
 from django.urls import reverse
 from django.utils import timezone
+from django.db.models import Prefetch, Count
 
 from funkwhale_api import musicbrainz
 from funkwhale_api.common import fields
@@ -420,7 +421,13 @@ def import_album(v):
 class TrackQuerySet(common_models.LocalFromFidQuerySet, models.QuerySet):
     def for_nested_serialization(self):
         return self.prefetch_related(
-            "artist", "album__artist", "album__attachment_cover"
+            "artist",
+            Prefetch(
+                "album",
+                queryset=Album.objects.select_related(
+                    "artist", "attachment_cover"
+                ).annotate(_prefetched_tracks_count=Count("tracks")),
+            ),
         )
 
     def annotate_playable_by_actor(self, actor):
@@ -855,8 +862,7 @@ class Upload(models.Model):
         if not input:
             return
 
-        input_format = utils.MIMETYPE_TO_EXTENSION[self.mimetype]
-        audio = pydub.AudioSegment.from_file(input, format=input_format)
+        audio = pydub.AudioSegment.from_file(input)