Skip to content
Snippets Groups Projects
Unverified Commit 4c4ab591 authored by Agate's avatar Agate :speech_balloon:
Browse files

Fixed various issue with plugins, shaping prometheus plugin

parent 1cea82dc
No related branches found
No related tags found
No related merge requests found
Showing with 134 additions and 209 deletions
......@@ -13,6 +13,8 @@ from funkwhale_api.subsonic.views import SubsonicViewSet
from funkwhale_api.tags import views as tags_views
from funkwhale_api.users import jwt_views
from config import plugins
router = common_routers.OptionalSlashRouter()
router.register(r"settings", GlobalPreferencesViewSet, basename="settings")
router.register(r"activity", activity_views.ActivityViewSet, "activity")
......@@ -98,3 +100,11 @@ v1_patterns += [
urlpatterns = [
url(r"^v1/", include((v1_patterns, "v1"), namespace="v1"))
] + format_suffix_patterns(subsonic_router.urls, allowed=["view"])
plugin_urls = []
for group in plugins.trigger_hook("urls"):
for u in group:
plugin_urls.append(u)
urlpatterns += [
url("^plugins/", include((plugin_urls, "plugins"), namespace="plugins")),
]
from django import urls
from django.apps import AppConfig
from pluggy import PluginManager, HookimplMarker, HookspecMarker
plugins_manager = PluginManager("funkwhale")
hook = HookimplMarker("funkwhale")
hookspec = HookspecMarker("funkwhale")
class PluginViewMiddleware:
def __init__(self, get_response):
self.get_response = get_response
def __call__(self, request):
from django.conf import settings
response = self.get_response(request)
if response.status_code == 404 and request.path.startswith("/plugins/"):
match = urls.resolve(request.path, urlconf=settings.PLUGINS_URLCONF)
response = match.func(request, *match.args, **match.kwargs)
return response
plugin_hook = HookimplMarker("funkwhale")
plugin_spec = HookspecMarker("funkwhale")
class ConfigError(ValueError):
pass
class Plugin:
class Plugin(AppConfig):
conf = {}
path = "noop"
def get_conf(self):
return {"enabled": self.plugin_settings.enabled}
def register_api_view(self, path, name=None):
def register(view):
return urls.path(
"plugins/{}/{}".format(self.name.replace("_", "-"), path),
view,
name="plugins-{}-{}".format(self.name, name),
)
return register
def plugin_settings(self):
"""
Return plugin specific settings from django.conf.settings
"""
import ipdb
ipdb.set_trace()
from django.conf import settings
d = {}
......@@ -80,47 +55,34 @@ def clean(d, conf, plugin_name):
return cleaned
def reverse(name, **kwargs):
from django.conf import settings
return urls.reverse(name, settings.PLUGINS_URLCONF, **kwargs)
def resolve(name, **kwargs):
from django.conf import settings
return urls.resolve(name, settings.PLUGINS_URLCONF, **kwargs)
# def install_plugin(name_or_path):
# subprocess.check_call([sys.executable, "-m", "pip", "install", package])
# sub
class HookSpec:
@hookspec
@plugin_spec
def database_engine(self):
"""
Customize the database engine with a new class
"""
@plugin_spec
def register_apps(self):
"""
Register additional apps in INSTALLED_APPS.
:rvalue: list"""
@hookspec
@plugin_spec
def middlewares_before(self):
"""
Register additional middlewares at the outer level.
:rvalue: list"""
@hookspec
@plugin_spec
def middlewares_after(self):
"""
Register additional middlewares at the inner level.
:rvalue: list"""
@hookspec
def urls(self):
"""
Register additional urls.
......@@ -129,3 +91,19 @@ class HookSpec:
plugins_manager.add_hookspecs(HookSpec())
def register(plugin_class):
return plugins_manager.register(plugin_class("noop", "noop"))
def trigger_hook(name, *args, **kwargs):
handler = getattr(plugins_manager.hook, name)
return handler(*args, **kwargs)
@register
class DefaultPlugin(Plugin):
@plugin_hook
def database_engine(self):
return "django.db.backends.postgresql"
......@@ -6,6 +6,8 @@ import logging.config
import os
import sys
import persisting_theory
from urllib.parse import urlsplit
from celery.schedules import crontab
from funkwhale_api import __version__
......@@ -17,9 +19,21 @@ sys.path.append(os.path.join(APPS_DIR, "plugins"))
logger = logging.getLogger("funkwhale_api.config")
env = environ.Env()
from .. import plugins # noqa
total = plugins.plugins_manager.load_setuptools_entrypoints("funkwhale")
class Plugins(persisting_theory.Registry):
look_into = "entrypoint"
PLUGINS = [p for p in env.list("FUNKWHALE_PLUGINS", default=[]) if p]
"""
List of Funkwhale plugins to load.
"""
from config import plugins # noqa
plugins_registry = Plugins()
plugins_registry.autodiscover(PLUGINS)
# plugins.plugins_manager.register(Plugin("noop", "noop"))
LOGLEVEL = env("LOGLEVEL", default="info").upper()
"""
......@@ -250,18 +264,14 @@ LOCAL_APPS = (
# See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
PLUGINS = [p for p in env.list("FUNKWHALE_PLUGINS", default=[]) if p]
"""
List of Funkwhale plugins to load.
"""
ADDITIONAL_APPS = env.list("ADDITIONAL_APPS", default=[])
"""
List of Django apps to load in addition to Funkwhale plugins and apps.
"""
PLUGINS_APPS = tuple()
for p in plugins.plugins_manager.hook.register_apps():
for p in plugins.trigger_hook("register_apps"):
PLUGINS_APPS += (p,)
INSTALLED_APPS = (
......@@ -281,12 +291,12 @@ else:
# MIDDLEWARE CONFIGURATION
# ------------------------------------------------------------------------------
ADDITIONAL_MIDDLEWARES_START = env.list("ADDITIONAL_MIDDLEWARES_START", default=[])
for group in plugins.plugins_manager.hook.middlewares_before():
for group in plugins.trigger_hook("middlewares_before"):
for m in group:
ADDITIONAL_MIDDLEWARES_START.append(m)
ADDITIONAL_MIDDLEWARES_END = env.list("ADDITIONAL_MIDDLEWARES_END", default=[])
for group in plugins.plugins_manager.hook.middlewares_after():
for group in plugins.trigger_hook("middlewares_after"):
for m in group:
ADDITIONAL_MIDDLEWARES_END.append(m)
......@@ -306,7 +316,6 @@ MIDDLEWARE = (
"django.contrib.messages.middleware.MessageMiddleware",
"funkwhale_api.users.middleware.RecordActivityMiddleware",
"funkwhale_api.common.middleware.ThrottleStatusMiddleware",
"funkwhale_api.common.plugins.PluginViewMiddleware",
)
+ tuple(ADDITIONAL_MIDDLEWARES_END)
)
......@@ -394,6 +403,9 @@ DATABASES["default"]["ATOMIC_REQUESTS"] = True
DB_CONN_MAX_AGE = DATABASES["default"]["CONN_MAX_AGE"] = env(
"DB_CONN_MAX_AGE", default=60 * 5
)
engine = plugins.trigger_hook("database_engine")[-1]
DATABASES["default"]["ENGINE"] = engine
"""
Max time, in seconds, before database connections are closed.
"""
......
......@@ -8,8 +8,6 @@ from django.conf.urls.static import static
from funkwhale_api.common import admin
from django.views import defaults as default_views
from config import plugins
urlpatterns = [
# Django Admin, use {% url 'admin:index' %}
......@@ -26,9 +24,6 @@ urlpatterns = [
# Your stuff: custom urls includes go here
]
for group in plugins.plugins_manager.hook.urls():
urlpatterns += group
if settings.DEBUG:
# This allows the error pages to be debugged during development, just visit
# these url in browser to see how these error pages look like.
......
import json
from django import http
from .plugin import PLUGIN
@PLUGIN.register_api_view(path="prometheus")
def prometheus(request):
stats = get_stats()
return http.HttpResponse(json.dumps(stats))
def get_stats():
return {"foo": "bar"}
from funkwhale_api.common import plugins
class Plugin(plugins.Plugin):
name = "prometheus"
......@@ -4,6 +4,7 @@ import sys
from . import base
from . import library # noqa
from . import media # noqa
from . import plugins # noqa
from . import users # noqa
from rest_framework.exceptions import ValidationError
......
import os
import subprocess
import click
from django.conf import settings
from . import base
@base.cli.group()
def plugins():
"""Install, configure and remove plugins"""
pass
@plugins.command("install")
@click.argument("name_or_url", nargs=-1)
@click.option("--builtins", is_flag=True)
@click.option("--pip-args")
def install(name_or_url, builtins, pip_args):
"""
Installed the specified plug using their name.
If --builtins is provided, it will also install
plugins present at FUNKWHALE_PLUGINS_PATH
"""
pip_args = pip_args or ""
target_path = settings.FUNKWHALE_PLUGINS_PATH
builtins_path = os.path.join(settings.APPS_DIR, "plugins")
builtins_plugins = [f.path for f in os.scandir(builtins_path) if f.is_dir()]
command = "pip install {} --target={} {}".format(
pip_args, target_path, " ".join(builtins_plugins)
)
subprocess.run(
command, shell=True, check=True,
)
from django.apps import AppConfig
from django import urls
from django.conf import settings
urlpatterns = []
class PluginViewMiddleware:
def __init__(self, get_response):
self.get_response = get_response
def __call__(self, request):
response = self.get_response(request)
if response.status_code == 404 and request.path.startswith("/plugins/"):
match = urls.resolve(request.path, urlconf=settings.PLUGINS_URLCONF)
response = match.func(request, *match.args, **match.kwargs)
return response
class Plugin(AppConfig):
def ready(self):
from . import main # noqa
return super().ready()
def register_api_view(self, path, name=None):
def register(view):
urlpatterns.append(
urls.path(
"plugins/{}/{}".format(self.name.replace("_", "-"), path),
view,
name="plugins-{}-{}".format(self.name, name),
)
),
return register
def reverse(name, **kwargs):
return urls.reverse(name, settings.PLUGINS_URLCONF, **kwargs)
def resolve(name, **kwargs):
return urls.resolve(name, settings.PLUGINS_URLCONF, **kwargs)
Prometheus exporter for Funkwhale
=================================
Use the following prometheus config:
.. code-block: yaml
global:
scrape_interval: 15s
scrape_configs:
- job_name: funkwhale
static_configs:
- targets: ['yourpod']
metrics_path: /api/plugins/prometheus/metrics?token=test
- job_name: prometheus
static_configs:
- targets: ['localhost:9090']
import json
from django import http
from django import urls
from django.conf.urls import url, include
from config import plugins
@plugins.register
class Plugin(plugins.Plugin):
name = "prometheus_exporter"
@plugins.hook
@plugins.plugin_hook
def database_engine(self):
return "django_prometheus.db.backends.postgresql"
@plugins.plugin_hook
def register_apps(self):
return "django_prometheus"
@plugins.hook
@plugins.plugin_hook
def middlewares_before(self):
return [
"django_prometheus.middleware.PrometheusBeforeMiddleware",
]
@plugins.hook
@plugins.plugin_hook
def middlewares_after(self):
return [
"django_prometheus.middleware.PrometheusAfterMiddleware",
]
@plugins.hook
@plugins.plugin_hook
def urls(self):
return [urls.url(r"^plugins/prometheus/exporter/?$", prometheus)]
plugins.plugins_manager.register(Plugin())
def prometheus(request):
stats = {"foo": "bar"}
return http.HttpResponse(json.dumps(stats))
return [url(r"^prometheus/", include("django_prometheus.urls"))]
......@@ -21,8 +21,8 @@ install_requires =
django_prometheus
[options.entry_points]
funkwhale-plugin =
prometheus = prometheus_exporter.main
funkwhale =
prometheus = prometheus_exporter.entrypoint
[options.packages.find]
......
import os
import pytest
from django.urls import resolvers
from funkwhale_api.common import plugins
class P(plugins.Plugin):
name = "test_plugin"
path = os.path.abspath(__file__)
@pytest.fixture
def plugin(settings):
yield P(app_name="test_plugin", app_module="tests.common.test_plugins.main.P")
@pytest.fixture(autouse=True)
def clear_patterns():
plugins.urlpatterns.clear()
resolvers._get_cached_resolver.cache_clear()
yield
resolvers._get_cached_resolver.cache_clear()
def test_can_register_view(plugin, mocker, settings):
view = mocker.Mock()
plugin.register_api_view("hello", name="hello")(view)
expected = "/plugins/test-plugin/hello"
assert plugins.reverse("plugins-test_plugin-hello") == expected
assert plugins.resolve(expected).func == view
def test_plugin_view_middleware_not_matching(api_client, plugin, mocker, settings):
view = mocker.Mock()
get_response = mocker.Mock()
middleware = plugins.PluginViewMiddleware(get_response)
plugin.register_api_view("hello", name="hello")(view)
request = mocker.Mock(path=plugins.reverse("plugins-test_plugin-hello"))
response = middleware(request)
assert response == get_response.return_value
view.assert_not_called()
def test_plugin_view_middleware_matching(api_client, plugin, mocker, settings):
view = mocker.Mock()
get_response = mocker.Mock(return_value=mocker.Mock(status_code=404))
middleware = plugins.PluginViewMiddleware(get_response)
plugin.register_api_view("hello/<slug:slug>", name="hello")(view)
request = mocker.Mock(
path=plugins.reverse("plugins-test_plugin-hello", kwargs={"slug": "world"})
)
response = middleware(request)
assert response == view.return_value
view.assert_called_once_with(request, slug="world")
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment