api/config/api_urls.py

@@ -13,6 +13,8 @@ from funkwhale_api.subsonic.views import SubsonicViewSet
 from funkwhale_api.tags import views as tags_views
 from funkwhale_api.users import jwt_views
 
+from config import plugins
+
 router = common_routers.OptionalSlashRouter()
 router.register(r"settings", GlobalPreferencesViewSet, basename="settings")
 router.register(r"activity", activity_views.ActivityViewSet, "activity")
@@ -98,3 +100,11 @@ v1_patterns += [
 urlpatterns = [
     url(r"^v1/", include((v1_patterns, "v1"), namespace="v1"))
 ] + format_suffix_patterns(subsonic_router.urls, allowed=["view"])
+
+plugin_urls = []
+for group in plugins.trigger_hook("urls"):
+    for u in group:
+        plugin_urls.append(u)
+urlpatterns += [
+    url("^plugins/", include((plugin_urls, "plugins"), namespace="plugins")),
+]

api/config/plugins.py

+from django.apps import AppConfig
+
+from pluggy import PluginManager, HookimplMarker, HookspecMarker
+
+plugins_manager = PluginManager("funkwhale")
+plugin_hook = HookimplMarker("funkwhale")
+plugin_spec = HookspecMarker("funkwhale")
+
+
+class ConfigError(ValueError):
+    pass
+
+
+class Plugin(AppConfig):
+    conf = {}
+    path = "noop"
+    conf_serializer = None
+
+    def get_conf(self):
+        return self.instance.conf
+
+    def set_conf(self, data):
+        if self.conf_serializer:
+            s = self.conf_serializer(data=data)
+            s.is_valid(raise_exception=True)
+            data = s.validated_data
+        instance = self.instance()
+        instance.conf = data
+        instance.save(update_fields=["conf"])
+
+    def instance(self):
+        """Return the DB object that match the plugin"""
+        from funkwhale_api.common import models
+
+        return models.PodPlugin.objects.get_or_create(code=self.name)[0]
+
+    def plugin_settings(self):
+        """
+        Return plugin specific settings from django.conf.settings
+        """
+        from django.conf import settings
+
+        d = {}
+        for key in dir(settings):
+            k = key.lower()
+            if not k.startswith("plugin_{}_".format(self.name.lower())):
+                continue
+
+            value = getattr(settings, key)
+            s_key = k.replace("plugin_{}_".format(self.name.lower()), "")
+            d[s_key] = value
+            return clean(d, self.conf, self.name)
+
+
+def clean(d, conf, plugin_name):
+    cleaned = {}
+    for key, c in conf.items():
+        if key in d:
+            try:
+                cleaned[key] = c["validator"](d[key])
+            except (ValueError, TypeError, AttributeError):
+                raise ConfigError(
+                    "Invalid value {} for setting {} in plugin {}".format(
+                        d[key], key, plugin_name
+                    )
+                )
+
+        else:
+            cleaned[key] = c["default"]
+
+    return cleaned
+
+
+class HookSpec:
+    @plugin_spec
+    def database_engine(self):
+        """
+        Customize the database engine with a new class
+        """
+
+    @plugin_spec
+    def register_apps(self):
+        """
+        Register additional apps in INSTALLED_APPS.
+
+        :rvalue: list"""
+
+    @plugin_spec
+    def middlewares_before(self):
+        """
+        Register additional middlewares at the outer level.
+
+        :rvalue: list"""
+
+    @plugin_spec
+    def middlewares_after(self):
+        """
+        Register additional middlewares at the inner level.
+
+        :rvalue: list"""
+
+    def urls(self):
+        """
+        Register additional urls.
+
+        :rvalue: list"""
+
+
+plugins_manager.add_hookspecs(HookSpec())
+
+
+def register(plugin_class):
+    return plugins_manager.register(plugin_class(plugin_class.name, "noop"))
+
+
+def save(plugin_class):
+    from funkwhale_api.common.models import PodPlugin
+
+    return PodPlugin.objects.get_or_create(code=plugin_class.name)[0]
+
+
+def trigger_hook(name, *args, **kwargs):
+    handler = getattr(plugins_manager.hook, name)
+    return handler(*args, **kwargs)
+
+
+@register
+class DefaultPlugin(Plugin):
+    name = "default"
+    verbose_name = "Default plugin"
+
+    @plugin_hook
+    def database_engine(self):
+        return "django.db.backends.postgresql"
+
+    @plugin_hook
+    def urls(self):
+        return []

api/config/settings/common.py

@@ -6,19 +6,37 @@ import logging.config
 import os
 import sys
 
-from urllib.parse import urlsplit
+import persisting_theory
 
-import environ
+from urllib.parse import urlsplit
 from celery.schedules import crontab
-
 from funkwhale_api import __version__
+import environ
 
-logger = logging.getLogger("funkwhale_api.config")
 ROOT_DIR = environ.Path(__file__) - 3  # (/a/b/myfile.py - 3 = /)
 APPS_DIR = ROOT_DIR.path("funkwhale_api")
-
+sys.path.append(os.path.join(APPS_DIR, "plugins"))
+logger = logging.getLogger("funkwhale_api.config")
 env = environ.Env()
 
+
+class Plugins(persisting_theory.Registry):
+    look_into = "entrypoint"
+
+
+PLUGINS = [
+    "funkwhale_plugin_{}".format(p)
+    for p in env.list("FUNKWHALE_PLUGINS", default=[])
+    if p
+]
+"""
+List of Funkwhale plugins to load.
+"""
+from config import plugins  # noqa
+
+plugins_registry = Plugins()
+plugins_registry.autodiscover(PLUGINS)
+
 LOGLEVEL = env("LOGLEVEL", default="info").upper()
 """
 Default logging level for the Funkwhale processes"""  # pylint: disable=W0105
@@ -248,46 +266,63 @@ LOCAL_APPS = (
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
 
 
-PLUGINS = [p for p in env.list("FUNKWHALE_PLUGINS", default=[]) if p]
-"""
-List of Funkwhale plugins to load.
-"""
-if PLUGINS:
-    logger.info("Running with the following plugins enabled: %s", ", ".join(PLUGINS))
-else:
-    logger.info("Running with no plugins")
-
 ADDITIONAL_APPS = env.list("ADDITIONAL_APPS", default=[])
 """
 List of Django apps to load in addition to Funkwhale plugins and apps.
 """
+
+PLUGINS_APPS = tuple()
+
+for p in plugins.trigger_hook("register_apps"):
+    PLUGINS_APPS += tuple(p)
+
 INSTALLED_APPS = (
     DJANGO_APPS
     + THIRD_PARTY_APPS
     + LOCAL_APPS
-    + tuple(["{}.apps.Plugin".format(p) for p in PLUGINS])
     + tuple(ADDITIONAL_APPS)
+    + tuple(PLUGINS)
+    + tuple(PLUGINS_APPS)
 )
 
+if PLUGINS:
+    logger.info("Running with the following plugins enabled: %s", ", ".join(PLUGINS))
+else:
+    logger.info("Running with no plugins")
+
 # MIDDLEWARE CONFIGURATION
 # ------------------------------------------------------------------------------
-ADDITIONAL_MIDDLEWARES_BEFORE = env.list("ADDITIONAL_MIDDLEWARES_BEFORE", default=[])
-MIDDLEWARE = tuple(ADDITIONAL_MIDDLEWARES_BEFORE) + (
-    "django.middleware.security.SecurityMiddleware",
-    "django.middleware.clickjacking.XFrameOptionsMiddleware",
-    "corsheaders.middleware.CorsMiddleware",
-    # needs to be before SPA middleware
-    "django.contrib.sessions.middleware.SessionMiddleware",
-    "django.middleware.common.CommonMiddleware",
-    "django.middleware.csrf.CsrfViewMiddleware",
-    # /end
-    "funkwhale_api.common.middleware.SPAFallbackMiddleware",
-    "django.contrib.auth.middleware.AuthenticationMiddleware",
-    "django.contrib.messages.middleware.MessageMiddleware",
-    "funkwhale_api.users.middleware.RecordActivityMiddleware",
-    "funkwhale_api.common.middleware.ThrottleStatusMiddleware",
+ADDITIONAL_MIDDLEWARES_START = env.list("ADDITIONAL_MIDDLEWARES_START", default=[])
+for group in plugins.trigger_hook("middlewares_before"):
+    for m in group:
+        ADDITIONAL_MIDDLEWARES_START.append(m)
+
+ADDITIONAL_MIDDLEWARES_END = env.list("ADDITIONAL_MIDDLEWARES_END", default=[])
+for group in plugins.trigger_hook("middlewares_after"):
+    for m in group:
+        ADDITIONAL_MIDDLEWARES_END.append(m)
+
+MIDDLEWARE = (
+    tuple(ADDITIONAL_MIDDLEWARES_START)
+    + (
+        "django.middleware.security.SecurityMiddleware",
+        "django.middleware.clickjacking.XFrameOptionsMiddleware",
+        "corsheaders.middleware.CorsMiddleware",
+        # needs to be before SPA middleware
+        "django.contrib.sessions.middleware.SessionMiddleware",
+        "django.middleware.common.CommonMiddleware",
+        "django.middleware.csrf.CsrfViewMiddleware",
+        # /end
+        "funkwhale_api.common.middleware.SPAFallbackMiddleware",
+        "django.contrib.auth.middleware.AuthenticationMiddleware",
+        "django.contrib.messages.middleware.MessageMiddleware",
+        "funkwhale_api.users.middleware.RecordActivityMiddleware",
+        "funkwhale_api.common.middleware.ThrottleStatusMiddleware",
+    )
+    + tuple(ADDITIONAL_MIDDLEWARES_END)
 )
 
+
 # DEBUG
 # ------------------------------------------------------------------------------
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#debug
@@ -370,6 +405,9 @@ DATABASES["default"]["ATOMIC_REQUESTS"] = True
 DB_CONN_MAX_AGE = DATABASES["default"]["CONN_MAX_AGE"] = env(
     "DB_CONN_MAX_AGE", default=60 * 5
 )
+
+engine = plugins.trigger_hook("database_engine")[-1]
+DATABASES["default"]["ENGINE"] = engine
 """
 Max time, in seconds, before database connections are closed.
 """
@@ -479,6 +517,7 @@ AWS_ACCESS_KEY_ID = env("AWS_ACCESS_KEY_ID", default=None)
 """
 Access-key ID for your S3 storage.
 """
+SECRET_KEY = env("DJANGO_SECRET_KEY")
 
 if AWS_ACCESS_KEY_ID:
     AWS_ACCESS_KEY_ID = AWS_ACCESS_KEY_ID
@@ -555,6 +594,7 @@ Delay in seconds before uploaded but unattached attachements are pruned from the
 # ------------------------------------------------------------------------------
 ROOT_URLCONF = "config.urls"
 SPA_URLCONF = "config.spa_urls"
+PLUGINS_URLCONF = "funkwhale_api.common.plugins"
 ASGI_APPLICATION = "config.routing.application"
 
 # This ensures that Django will be able to detect a secure connection

api/config/settings/local.py

@@ -17,14 +17,6 @@ DEBUG = env.bool("DJANGO_DEBUG", default=True)
 FORCE_HTTPS_URLS = env.bool("FORCE_HTTPS_URLS", default=False)
 TEMPLATES[0]["OPTIONS"]["debug"] = DEBUG
 
-# SECRET CONFIGURATION
-# ------------------------------------------------------------------------------
-# See: https://docs.djangoproject.com/en/dev/ref/settings/#secret-key
-# Note: This key only used for development and testing.
-SECRET_KEY = env(
-    "DJANGO_SECRET_KEY", default="mc$&b=5j#6^bv7tld1gyjp2&+^-qrdy=0sw@r5sua*1zp4fmxc"
-)
-
 # Mail settings
 # ------------------------------------------------------------------------------
 EMAIL_HOST = "localhost"

api/config/settings/production.py

@@ -16,8 +16,6 @@ from .common import *  # noqa
 # SECRET CONFIGURATION
 # ------------------------------------------------------------------------------
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#secret-key
-# Raises ImproperlyConfigured exception if DJANGO_SECRET_KEY not in os.environ
-SECRET_KEY = env("DJANGO_SECRET_KEY")
 
 # django-secure
 # ------------------------------------------------------------------------------

api/funkwhale_api/cli/main.py

@@ -4,6 +4,7 @@ import sys
 from . import base
 from . import library  # noqa
 from . import media  # noqa
+from . import plugins  # noqa
 from . import users  # noqa
 
 from rest_framework.exceptions import ValidationError

api/funkwhale_api/cli/plugins.py

+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+import click
+
+from django.conf import settings
+
+from . import base
+
+
+PIP = os.path.join(sys.prefix, "bin", "pip")
+
+
+@base.cli.group()
+def plugins():
+    """Install, configure and remove plugins"""
+    pass
+
+
+def get_all_plugins():
+    plugins = [
+        f.path
+        for f in os.scandir(settings.FUNKWHALE_PLUGINS_PATH)
+        if "/funkwhale_plugin_" in f.path
+    ]
+    plugins = [
+        p.split("-")[0].split("/")[-1].replace("funkwhale_plugin_", "") for p in plugins
+    ]
+    return plugins
+
+
+@plugins.command("install")
+@click.argument("name_or_url", nargs=-1)
+@click.option("--builtins", is_flag=True)
+@click.option("--pip-args")
+def install(name_or_url, builtins, pip_args):
+    """
+    Installed the specified plug using their name.
+    """
+    pip_args = pip_args or ""
+    all_plugins = []
+    for p in name_or_url:
+        builtin_path = os.path.join(
+            settings.APPS_DIR, "plugins", "funkwhale_plugin_{}".format(p)
+        )
+        if os.path.exists(builtin_path):
+            all_plugins.append(builtin_path)
+        else:
+            all_plugins.append(p)
+    install_plugins(pip_args, all_plugins)
+    click.echo(
+        "Installation completed, ensure FUNKWHALE_PLUGINS={} is present in your .env file".format(
+            ",".join(get_all_plugins())
+        )
+    )
+
+
+def install_plugins(pip_args, all_plugins):
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        command = "{} install {} --target {} --build={} {}".format(
+            PIP,
+            pip_args,
+            settings.FUNKWHALE_PLUGINS_PATH,
+            tmpdirname,
+            " ".join(all_plugins),
+        )
+        subprocess.run(
+            command, shell=True, check=True,
+        )
+
+
+@plugins.command("uninstall")
+@click.argument("name", nargs=-1)
+def uninstall(name):
+    """
+    Remove plugins
+    """
+    to_remove = ["funkwhale_plugin_{}".format(n) for n in name]
+    command = "{} uninstall -y {}".format(PIP, " ".join(to_remove))
+    subprocess.run(
+        command, shell=True, check=True,
+    )
+    for f in os.scandir(settings.FUNKWHALE_PLUGINS_PATH):
+        for n in name:
+            if "/funkwhale_plugin_{}".format(n) in f.path:
+                shutil.rmtree(f.path)
+    click.echo(
+        "Removal completed, set FUNKWHALE_PLUGINS={} in your .env file".format(
+            ",".join(get_all_plugins())
+        )
+    )

api/funkwhale_api/common/migrations/0008_auto_20200617_1902.py

+# Generated by Django 3.0.6 on 2020-06-17 19:02
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('common', '0007_auto_20200116_1610'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='PodPlugin',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('conf', django.contrib.postgres.fields.jsonb.JSONField(default=None, null=True, blank=True)),
+                ('code', models.CharField(max_length=100, unique=True)),
+                ('creation_date', models.DateTimeField(default=django.utils.timezone.now)),
+            ],
+        ),
+        migrations.AlterField(
+            model_name='attachment',
+            name='url',
+            field=models.URLField(blank=True, max_length=500, null=True),
+        ),
+    ]

api/funkwhale_api/common/models.py

@@ -18,6 +18,7 @@ from django.urls import reverse
 from versatileimagefield.fields import VersatileImageField
 from versatileimagefield.image_warmer import VersatileImageFieldWarmer
 
+from config import plugins
 from funkwhale_api.federation import utils as federation_utils
 
 from . import utils
@@ -363,3 +364,17 @@ def remove_attached_content(sender, instance, **kwargs):
                 getattr(instance, field).delete()
             except Content.DoesNotExist:
                 pass
+
+
+class PodPlugin(models.Model):
+    conf = JSONField(default=None, null=True, blank=True)
+    code = models.CharField(max_length=100, unique=True)
+    creation_date = models.DateTimeField(default=timezone.now)
+
+    @property
+    def plugin(self):
+        """Links to the Plugin instance in entryposint.py"""
+        candidates = plugins.plugins_manager.get_plugins()
+        for p in candidates:
+            if p.name == self.code:
+                return p

api/funkwhale_api/common/serializers.py

@@ -339,3 +339,21 @@ class NullToEmptDict(object):
         if not v:
             return v
         return super().to_representation(v)
+
+
+class PodPluginSerializer(serializers.Serializer):
+    code = serializers.CharField(read_only=True)
+    enabled = serializers.BooleanField()
+    conf = serializers.JSONField()
+    label = serializers.SerializerMethodField()
+
+    class Meta:
+        fields = [
+            "code",
+            "label",
+            "enabled",
+            "conf",
+        ]
+
+    def get_label(self, o):
+        return o.plugin.verbose_name

api/funkwhale_api/music/filters.py

@@ -85,8 +85,6 @@ class ArtistFilter(
         model = models.Artist
         fields = {
             "name": ["exact", "iexact", "startswith", "icontains"],
-            "playable": ["exact"],
-            "scope": ["exact"],
             "mbid": ["exact"],
         }
         hidden_content_fields_mapping = moderation_filters.USER_FILTER_CONFIG["ARTIST"]
@@ -122,11 +120,9 @@ class TrackFilter(
         model = models.Track
         fields = {
             "title": ["exact", "iexact", "startswith", "icontains"],
-            "playable": ["exact"],
             "id": ["exact"],
             "album": ["exact"],
             "license": ["exact"],
-            "scope": ["exact"],
             "mbid": ["exact"],
         }
         hidden_content_fields_mapping = moderation_filters.USER_FILTER_CONFIG["TRACK"]
@@ -173,7 +169,6 @@ class UploadFilter(audio_filters.IncludeChannelsFilterSet):
     class Meta:
         model = models.Upload
         fields = [
-            "playable",
             "import_status",
             "mimetype",
             "track",
@@ -181,7 +176,6 @@ class UploadFilter(audio_filters.IncludeChannelsFilterSet):
             "album_artist",
             "library",
             "import_reference",
-            "scope",
             "channel",
         ]
         include_channels_field = "track__artist__channel"
@@ -209,7 +203,7 @@ class AlbumFilter(
 
     class Meta:
         model = models.Album
-        fields = ["playable", "q", "artist", "scope", "mbid"]
+        fields = ["artist", "mbid"]
         hidden_content_fields_mapping = moderation_filters.USER_FILTER_CONFIG["ALBUM"]
         include_channels_field = "artist__channel"
         channel_filter_field = "track__album"
@@ -226,4 +220,4 @@ class LibraryFilter(filters.FilterSet):
 
     class Meta:
         model = models.Library
-        fields = ["privacy_level", "q", "scope"]
+        fields = ["privacy_level"]

api/funkwhale_api/playlists/filters.py

@@ -33,9 +33,6 @@ class PlaylistFilter(filters.FilterSet):
         fields = {
             "user": ["exact"],
             "name": ["exact", "icontains"],
-            "q": "exact",
-            "playable": "exact",
-            "scope": "exact",
         }
 
     def filter_playable(self, queryset, name, value):

api/funkwhale_api/plugins/funkwhale_plugin_prometheus/README.md

+Prometheus exporter for Funkwhale
+=================================
+
+Use the following prometheus config:
+
+.. code-block: yaml
+
+  global:
+    scrape_interval: 15s
+
+  scrape_configs:
+    - job_name: funkwhale
+      static_configs:
+        - targets: ['yourpod']
+      metrics_path: /api/plugins/prometheus/metrics?token=test
+
+    - job_name: prometheus
+      static_configs:
+        - targets: ['localhost:9090']

api/funkwhale_api/plugins/funkwhale_plugin_prometheus/entrypoint.py

+from django.conf.urls import url, include
+
+from config import plugins
+
+
+@plugins.register
+class Plugin(plugins.Plugin):
+    name = "funkwhale_plugin_prometheus"
+    verbose_name = "Prometheus metrics exporter"
+
+    @plugins.plugin_hook
+    def database_engine(self):
+        return "django_prometheus.db.backends.postgresql"
+
+    @plugins.plugin_hook
+    def register_apps(self):
+        return ["django_prometheus"]
+
+    @plugins.plugin_hook
+    def middlewares_before(self):
+        return [
+            "django_prometheus.middleware.PrometheusBeforeMiddleware",
+        ]
+
+    @plugins.plugin_hook
+    def middlewares_after(self):
+        return [
+            "django_prometheus.middleware.PrometheusAfterMiddleware",
+        ]
+
+    @plugins.plugin_hook
+    def urls(self):
+        return [url(r"^prometheus/", include("django_prometheus.urls"))]

api/funkwhale_api/plugins/funkwhale_plugin_prometheus/setup.cfg

+[metadata]
+name = funkwhale-plugin-prometheus
+description = "A prometheus metrics exporter for your Funkwhale pod"
+version = 0.1.dev0
+author = Agate Blue
+author_email = me@agate.blue
+url = https://dev.funkwhale.audio/funkwhale/funkwhale
+long_description = file: README.md
+license = AGPL3
+classifiers =
+    Development Status :: 3 - Alpha
+    License :: OSI Approved :: AGPL
+    Natural Language :: English
+    Programming Language :: Python :: 3.6
+
+[options]
+zip_safe = True
+include_package_data = True
+packages = find:
+install_requires =
+    django_prometheus
+
+[options.entry_points]
+funkwhale =
+	prometheus = prometheus_exporter.entrypoint
+
+
+[options.packages.find]
+exclude =
+    tests
+
+[bdist_wheel]
+universal = 1
+
+[tool:pytest]
+testpaths = tests

api/funkwhale_api/plugins/funkwhale_plugin_prometheus/setup.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+from setuptools import setup
+
+setup()

api/funkwhale_api/radios/filtersets.py

@@ -11,5 +11,4 @@ class RadioFilter(django_filters.FilterSet):
         model = models.Radio
         fields = {
             "name": ["exact", "iexact", "startswith", "icontains"],
-            "scope": "exact",
         }

api/funkwhale_api/tags/filters.py

@@ -18,4 +18,4 @@ class TagFilter(filters.FilterSet):
 
     class Meta:
         model = models.Tag
-        fields = {"q": ["exact"], "name": ["exact", "startswith"]}
+        fields = {"name": ["exact", "startswith"]}