Merge branch 'release/1.0'

.gitlab-ci.yml

@@ -97,7 +97,7 @@ black:
   variables:
     GIT_STRATEGY: fetch
   before_script:
-    - pip install black
+    - pip install black==19.10b0
   script:
     - black --check --diff api/
 

CHANGELOG

@@ -10,6 +10,149 @@ This changelog is viewable on the web at https://docs.funkwhale.audio/changelog.
 
 .. towncrier
 
+1.0 (2020-09-09)
+----------------
+
+Upgrade instructions are available at
+https://docs.funkwhale.audio/index.html
+
+
+Dropped python 3.5 support [manual action required, non-docker only]
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+With Funkwhale 1.0, we're dropping support for Python 3.5. Before upgrading,
+ensure ``python3 --version`` returns ``3.6`` or higher.
+
+If it returns ``3.6`` or higher, you have nothing to do.
+
+If it returns ``3.5``, you will need to upgrade your Python version/Host, then recreate your virtual environment::
+
+    rm -rf /srv/funkwhale/virtualenv
+    python3 -m venv /srv/funkwhale/virtualenv
+
+
+Increased quality of JPEG thumbnails [manual action required]
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Default quality for JPEG thumbnails was increased from 70 to 95, as 70 was producing visible artifacts in resized images.
+
+Because of this change, existing thumbnails will not load, and you will need to:
+
+1. delete the ``__sized__`` directory in your ``MEDIA_ROOT`` directory
+2. run ``python manage.py fw media generate-thumbnails`` to regenerate thumbnails with the enhanced quality
+
+If you don't want to regenerate thumbnails, you can keep the old ones by adding ``THUMBNAIL_JPEG_RESIZE_QUALITY=70`` to your .env file.
+
+Small API breaking change in ``/api/v1/libraries``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To allow easier crawling of public libraries on a pod,we had to make a slight breaking change
+to the behaviour of ``GET /api/v1/libraries``.
+
+Before, it returned only libraries owned by the current user.
+
+Now, it returns all the accessible libraries (including ones from other users and pods).
+
+If you are consuming the API via a third-party client and need to retrieve your libraries,
+use the ``scope`` parameter, like this: ``GET /api/v1/libraries?scope=me``
+
+API breaking change in ``/api/v1/albums``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To increase performance, querying ``/api/v1/albums`` doesn't return album tracks anymore. This caused
+some performance issues, especially as some albums and series have dozens or even hundreds of tracks.
+
+If you want to retrieve tracks for an album, you can query ``/api/v1/tracks/?album=<albumid>``.
+
+JWT deprecation
+^^^^^^^^^^^^^^^
+
+API Authentication using JWT is deprecated and will be removed in Funkwhale 1.0. Please use OAuth or application tokens
+and refer to our API documentation at https://docs.funkwhale.audio/swagger/ for guidance.
+
+Full list of changes
+^^^^^^^^^^^^^^^^^^^^
+
+Features:
+
+- Allow users to hide compilation artists on the artist search page (#1053)
+- Can now launch server import from the UI (#1105)
+- Dedicated, advanced search page (#370)
+- Persist theme and language settings accross sessions (#996)
+
+
+Enhancements:
+
+- Add support for unauthenticated users hitting the logout page
+- Added support for Licence Art Libre (#1088)
+- Broadcast/handle rejected follows (#858)
+- Confirm email without requiring the user to validate the form manually (#407)
+- Display channel and track downloads count (#1178)
+- Do not include tracks in album API representation (#1102)
+- Dropped python 3.5 support. Python 3.6 is the minimum required version (#1099)
+- Improved keyboard accessibility (#1125)
+- Improved naming of pages for accessibility (#1127)
+- Improved shuffle behaviour (#1190)
+- Increased quality of JPEG thumbnails
+- Lock focus in modals to improve accessibility (#1128)
+- More consistent search UX on /albums, /artists, /radios and /playlists (#1131)
+- Play button now replace current queue instead of appending to it (#1083)
+- Set proper lang attribute on HTML document (#1130)
+- Use semantic headers for accessibility (#1121)
+- Users can now update their email address (#292)
+- [plugin, scrobbler] Use last.fm API v2 for scrobbling if API key and secret are provided
+- Added a new, large thumbnail size for cover images (#1205
+- Enforce authentication when viewing remote channels, profiles and libraries (#1210)
+
+
+
+Bugfixes:
+
+- Fix broken media support detection (#1180)
+- Fix layout issue with playbar on landscape tablets (#1144)
+- Fix random radio so that podcast content is not picked up (#1140)
+- Fixed an issue with search pages where results would not appear after navigating to another page
+- Fixed crash with negative track position in file tags (#1193)
+- Handle access errors scanning directories when importing files
+- Make channel card updated times more humanly readable, add internationalization (#1089)
+- Ensure search page reloads if another search is submitted in the sidebar (#1197)
+- Fixed "scope=subscribed" on albums, artists, uploads and libraries API (#1217)
+- Fixed broken federation with pods using allow-listing (#1999)
+- Fixed broken search when using (, " or & chars (#1196)
+- Fixed domains table hidden controls when no domains are found (#1198)
+
+
+Documentation:
+
+- Simplify Docker mono-container installation and upgrade documentation
+
+
+Contributors to this release (translation, development, documentation, reviews, design, testing, third-party projects):
+
+- Agate
+- Andy Craze
+- anonymous
+- appzer0
+- Arne
+- Bheesham Persaud
+- Ciarán Ainsworth
+- Creak
+- Daniele Lira Mereb
+- dulz
+- Francesc Galí
+- ghose
+- mekind
+- Puri
+- Quentin PAGÈS
+- Raphaël Ventura
+- Simon Arlott
+- Slimane Selyan Amiri
+- Stefano Pigozzi
+- Sébastien de Melo
+- vicdorke
+- Xosé M
+
+
 0.21.2 (2020-07-27)
 -------------------
 
@@ -223,7 +366,8 @@ All user-related commands are available under the ``python manage.py fw users``
 Please refer to the `Admin documentation <https://docs.funkwhale.audio/admin/commands.html#user-management>`_ for
 more information and instructions.
 
-Progressive web app [Manual action suggested, non-docker only]
+Progressive web app [Manual action sugFull list of changes
+^^^^^^^^^^^^^^^^^^^^gested, non-docker only]
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 We've made Funkwhale's Web UI a Progressive Web Application (PWA), in order to improve the user experience

CONTRIBUTING.rst

@@ -704,6 +704,21 @@ Views: you can find some readable views tests in file: ``api/tests/users/test_vi
 Contributing to the front-end
 -----------------------------
 
+Styles and themes
+^^^^^^^^^^^^^^^^^
+
+Our UI framework is Fomantic UI (https://fomantic-ui.com/), and Funkwhale's custom styles are written in SCSS. All the styles are configured in ``front/src/styles/_main.scss``,
+including imporing of Fomantic UI styles and components.
+
+We're applying several changes on top of the Fomantic CSS files, before they are imported:
+
+1. Many hardcoded color values are replaced by CSS vars: e.g ``color: orange`` is replaced by ``color: var(--vibrant-color)``. This makes theming way easier.
+2. Unused components variations and icons are stripped from the source files, in order to reduce the final size of our CSS files
+
+This changes are applied automatically when running ``yarn install``, through a ``postinstall`` hook. Internally, ``front/scripts/fix-fomantic-css.py`` is called
+and handle both kind of modifications. Please refer to this script if you need to use new icons to the project, or restore some components variations that
+were stripped in order to use them.
+
 Running tests
 ^^^^^^^^^^^^^
 

README.rst

@@ -28,6 +28,16 @@ Contribute
 Contribution guidelines as well as development installation instructions
 are outlined in `CONTRIBUTING <CONTRIBUTING.rst>`_.
 
+Security issues and vulnerabilities
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you found a vulnerability in Funkwhale, please report it on our Gitlab instance at `https://dev.funkwhale.audio/funkwhale/funkwhale/-/issues`_, ensuring
+you have checked the ``This issue is confidential and should only be visible to team members with at least Reporter access.
+`` box. 
+
+This will ensure only maintainers and developpers have access to the vulnerability. Thank you for your help!
+
+
 Translate
 ^^^^^^^^^
 

api/config/api_urls.py

 from django.conf.urls import include, url
-from dynamic_preferences.api.viewsets import GlobalPreferencesViewSet
 from rest_framework import routers
 from rest_framework.urlpatterns import format_suffix_patterns
 
@@ -14,22 +13,20 @@ from funkwhale_api.tags import views as tags_views
 from funkwhale_api.users import jwt_views
 
 router = common_routers.OptionalSlashRouter()
-router.register(r"settings", GlobalPreferencesViewSet, basename="settings")
 router.register(r"activity", activity_views.ActivityViewSet, "activity")
 router.register(r"tags", tags_views.TagViewSet, "tags")
+router.register(r"plugins", common_views.PluginViewSet, "plugins")
 router.register(r"tracks", views.TrackViewSet, "tracks")
 router.register(r"uploads", views.UploadViewSet, "uploads")
 router.register(r"libraries", views.LibraryViewSet, "libraries")
 router.register(r"listen", views.ListenViewSet, "listen")
+router.register(r"stream", views.StreamViewSet, "stream")
 router.register(r"artists", views.ArtistViewSet, "artists")
 router.register(r"channels", audio_views.ChannelViewSet, "channels")
 router.register(r"subscriptions", audio_views.SubscriptionsViewSet, "subscriptions")
 router.register(r"albums", views.AlbumViewSet, "albums")
 router.register(r"licenses", views.LicenseViewSet, "licenses")
 router.register(r"playlists", playlists_views.PlaylistViewSet, "playlists")
-router.register(
-    r"playlist-tracks", playlists_views.PlaylistTrackViewSet, "playlist-tracks"
-)
 router.register(r"mutations", common_views.MutationViewSet, "mutations")
 router.register(r"attachments", common_views.AttachmentViewSet, "attachments")
 v1_patterns = router.urls
@@ -77,9 +74,11 @@ v1_patterns += [
         r"^history/",
         include(("funkwhale_api.history.urls", "history"), namespace="history"),
     ),
+    url(r"^", include(("funkwhale_api.users.api_urls", "users"), namespace="users"),),
+    # XXX: remove if Funkwhale 1.1
     url(
         r"^users/",
-        include(("funkwhale_api.users.api_urls", "users"), namespace="users"),
+        include(("funkwhale_api.users.api_urls", "users"), namespace="users-nested"),
     ),
     url(
         r"^oauth/",

api/config/plugins.py

+import copy
+import logging
+import os
+import subprocess
+import sys
+
+import persisting_theory
+from django.core.cache import cache
+from django.db.models import Q
+
+from rest_framework import serializers
+
+logger = logging.getLogger("plugins")
+
+
+class Startup(persisting_theory.Registry):
+    look_into = "persisting_theory"
+
+
+class Ready(persisting_theory.Registry):
+    look_into = "persisting_theory"
+
+
+startup = Startup()
+ready = Ready()
+
+_plugins = {}
+_filters = {}
+_hooks = {}
+
+
+class PluginCache(object):
+    def __init__(self, prefix):
+        self.prefix = prefix
+
+    def get(self, key, default=None):
+        key = ":".join([self.prefix, key])
+        return cache.get(key, default)
+
+    def set(self, key, value, duration=None):
+        key = ":".join([self.prefix, key])
+        return cache.set(key, value, duration)
+
+
+def get_plugin_config(
+    name,
+    user=False,
+    source=False,
+    registry=_plugins,
+    conf={},
+    settings={},
+    description=None,
+    version=None,
+    label=None,
+    homepage=None,
+):
+    conf = {
+        "name": name,
+        "label": label or name,
+        "logger": logger,
+        # conf is for dynamic settings
+        "conf": conf,
+        # settings is for settings hardcoded in .env
+        "settings": settings,
+        "user": True if source else user,
+        # source plugins are plugins that provide audio content
+        "source": source,
+        "description": description,
+        "version": version,
+        "cache": PluginCache(name),
+        "homepage": homepage,
+    }
+    registry[name] = conf
+    return conf
+
+
+def load_settings(name, settings):
+    from django.conf import settings as django_settings
+
+    mapping = {
+        "boolean": django_settings.ENV.bool,
+        "text": django_settings.ENV,
+    }
+    values = {}
+    prefix = "FUNKWHALE_PLUGIN_{}".format(name.upper())
+    for s in settings:
+        key = "_".join([prefix, s["name"].upper()])
+        value = mapping[s["type"]](key, default=s.get("default", None))
+        values[s["name"]] = value
+
+    logger.debug("Plugin %s running with settings %s", name, values)
+    return values
+
+
+def get_session():
+    from funkwhale_api.common import session
+
+    return session.get_session()
+
+
+def register_filter(name, plugin_config, registry=_filters):
+    def decorator(func):
+        handlers = registry.setdefault(name, [])
+
+        def inner(*args, **kwargs):
+            plugin_config["logger"].debug("Calling filter for %s", name)
+            rval = func(*args, **kwargs)
+            return rval
+
+        handlers.append((plugin_config["name"], inner))
+        return inner
+
+    return decorator
+
+
+def register_hook(name, plugin_config, registry=_hooks):
+    def decorator(func):
+        handlers = registry.setdefault(name, [])
+
+        def inner(*args, **kwargs):
+            plugin_config["logger"].debug("Calling hook for %s", name)
+            func(*args, **kwargs)
+
+        handlers.append((plugin_config["name"], inner))
+        return inner
+
+    return decorator
+
+
+class Skip(Exception):
+    pass
+
+
+def trigger_filter(name, value, enabled=False, **kwargs):
+    """
+    Call filters registered for "name" with the given
+    args and kwargs.
+
+    Return the value (that could be modified by handlers)
+    """
+    logger.debug("Calling handlers for filter %s", name)
+    registry = kwargs.pop("registry", _filters)
+    confs = kwargs.pop("confs", {})
+    for plugin_name, handler in registry.get(name, []):
+        if not enabled and confs.get(plugin_name, {}).get("enabled") is False:
+            continue
+        try:
+            value = handler(value, conf=confs.get(plugin_name, {}), **kwargs)
+        except Skip:
+            pass
+        except Exception as e:
+            logger.warn("Plugin %s errored during filter %s: %s", plugin_name, name, e)
+    return value
+
+
+def trigger_hook(name, enabled=False, **kwargs):
+    """
+    Call hooks registered for "name" with the given
+    args and kwargs.
+
+    Returns nothing
+    """
+    logger.debug("Calling handlers for hook %s", name)
+    registry = kwargs.pop("registry", _hooks)
+    confs = kwargs.pop("confs", {})
+    for plugin_name, handler in registry.get(name, []):
+        if not enabled and confs.get(plugin_name, {}).get("enabled") is False:
+            continue
+        try:
+            handler(conf=confs.get(plugin_name, {}).get("conf"), **kwargs)
+        except Skip:
+            pass
+        except Exception as e:
+            logger.warn("Plugin %s errored during hook %s: %s", plugin_name, name, e)
+
+
+def set_conf(name, conf, user=None, registry=_plugins):
+    from funkwhale_api.common import models
+
+    if not registry[name]["conf"] and not registry[name]["source"]:
+        return
+    conf_serializer = get_serializer_from_conf_template(
+        registry[name]["conf"], user=user, source=registry[name]["source"],
+    )(data=conf)
+    conf_serializer.is_valid(raise_exception=True)
+    if "library" in conf_serializer.validated_data:
+        conf_serializer.validated_data["library"] = str(
+            conf_serializer.validated_data["library"]
+        )
+    conf, _ = models.PluginConfiguration.objects.update_or_create(
+        user=user, code=name, defaults={"conf": conf_serializer.validated_data}
+    )
+
+
+def get_confs(user=None):
+    from funkwhale_api.common import models
+
+    qs = models.PluginConfiguration.objects.filter(code__in=list(_plugins.keys()))
+    if user:
+        qs = qs.filter(Q(user=None) | Q(user=user))
+    else:
+        qs = qs.filter(user=None)
+    confs = {
+        v["code"]: {"conf": v["conf"], "enabled": v["enabled"]}
+        for v in qs.values("code", "conf", "enabled")
+    }
+    for p, v in _plugins.items():
+        if p not in confs:
+            confs[p] = {"conf": None, "enabled": False}
+    return confs
+
+
+def get_conf(plugin, user=None):
+    return get_confs(user=user)[plugin]
+
+
+def enable_conf(code, value, user):
+    from funkwhale_api.common import models
+
+    models.PluginConfiguration.objects.update_or_create(
+        code=code, user=user, defaults={"enabled": value}
+    )
+
+
+class LibraryField(serializers.UUIDField):
+    def __init__(self, *args, **kwargs):
+        self.actor = kwargs.pop("actor")
+        super().__init__(*args, **kwargs)
+
+    def to_internal_value(self, v):
+        v = super().to_internal_value(v)
+        if not self.actor.libraries.filter(uuid=v).first():
+            raise serializers.ValidationError("Invalid library id")
+        return v
+
+
+def get_serializer_from_conf_template(conf, source=False, user=None):
+    conf = copy.deepcopy(conf)
+    validators = {f["name"]: f.pop("validator") for f in conf if "validator" in f}
+    mapping = {
+        "url": serializers.URLField,
+        "boolean": serializers.BooleanField,
+        "text": serializers.CharField,
+        "long_text": serializers.CharField,
+        "password": serializers.CharField,
+        "number": serializers.IntegerField,
+    }
+
+    for attr in ["label", "help"]:
+        for c in conf:
+            c.pop(attr, None)
+
+    class Serializer(serializers.Serializer):
+        def __init__(self, *args, **kwargs):
+            super().__init__(*args, **kwargs)
+            for field_conf in conf:
+                field_kwargs = copy.copy(field_conf)
+                name = field_kwargs.pop("name")
+                self.fields[name] = mapping[field_kwargs.pop("type")](**field_kwargs)
+            if source:
+                self.fields["library"] = LibraryField(actor=user.actor)
+
+    for vname, v in validators.items():
+        setattr(Serializer, "validate_{}".format(vname), v)
+    return Serializer
+
+
+def serialize_plugin(plugin_conf, confs):
+    return {
+        "name": plugin_conf["name"],
+        "label": plugin_conf["label"],
+        "description": plugin_conf.get("description") or None,
+        "user": plugin_conf.get("user", False),
+        "source": plugin_conf.get("source", False),
+        "conf": plugin_conf.get("conf", None),
+        "values": confs.get(plugin_conf["name"], {"conf"}).get("conf"),
+        "enabled": plugin_conf["name"] in confs
+        and confs[plugin_conf["name"]]["enabled"],
+        "homepage": plugin_conf["homepage"],
+    }
+
+
+def install_dependencies(deps):
+    if not deps:
+        return
+    logger.info("Installing plugins dependencies %s", deps)
+    pip_path = os.path.join(os.path.dirname(sys.executable), "pip")
+    subprocess.check_call([pip_path, "install"] + deps)
+
+
+def background_task(name):
+    from funkwhale_api.taskapp import celery
+
+    def decorator(func):
+        return celery.app.task(func, name=name)
+
+    return decorator
+
+
+# HOOKS
+LISTENING_CREATED = "listening_created"
+"""
+Called when a track is being listened
+"""
+SCAN = "scan"
+"""
+
+"""
+# FILTERS
+PLUGINS_DEPENDENCIES = "plugins_dependencies"
+"""
+Called with an empty list, use this filter to append pip dependencies
+to the list for installation.
+"""
+PLUGINS_APPS = "plugins_apps"
+"""
+Called with an empty list, use this filter to append apps to INSTALLED_APPS
+"""
+MIDDLEWARES_BEFORE = "middlewares_before"
+"""
+Called with an empty list, use this filter to prepend middlewares
+to MIDDLEWARE
+"""
+MIDDLEWARES_AFTER = "middlewares_after"
+"""
+Called with an empty list, use this filter to append middlewares
+to MIDDLEWARE
+"""
+URLS = "urls"
+"""
+Called with an empty list, use this filter to register new urls and views
+"""

api/config/routing.py

+from channels.auth import AuthMiddlewareStack
 from channels.routing import ProtocolTypeRouter, URLRouter
-from django.conf.urls import url
 
-from funkwhale_api.common.auth import TokenAuthMiddleware
+from django.conf.urls import url
 from funkwhale_api.instance import consumers
 
 application = ProtocolTypeRouter(
     {
         # Empty for now (http->django views is added by default)
-        "websocket": TokenAuthMiddleware(
+        "websocket": AuthMiddlewareStack(
             URLRouter([url("^api/v1/activity$", consumers.InstanceActivityConsumer)])
         )
     }

api/config/settings/common.py

 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, unicode_literals
 
+from collections import OrderedDict
 import datetime
 import logging.config
-import os
 import sys
 
 from urllib.parse import urlsplit
@@ -18,7 +18,7 @@ ROOT_DIR = environ.Path(__file__) - 3  # (/a/b/myfile.py - 3 = /)
 APPS_DIR = ROOT_DIR.path("funkwhale_api")
 
 env = environ.Env()
-
+ENV = env
 LOGLEVEL = env("LOGLEVEL", default="info").upper()
 """
 Default logging level for the Funkwhale processes"""  # pylint: disable=W0105
@@ -46,6 +46,12 @@ logging.config.dictConfig(