Merge branch 'develop'

.gitlab-ci.yml

 variables:
   IMAGE_NAME: funkwhale/funkwhale
   IMAGE: $IMAGE_NAME:$CI_COMMIT_REF_NAME
+  IMAGE_LATEST: $IMAGE_NAME:latest
   ALL_IN_ONE_IMAGE_NAME: funkwhale/all-in-one
   ALL_IN_ONE_IMAGE: $ALL_IN_ONE_IMAGE_NAME:$CI_COMMIT_REF_NAME
+  ALL_IN_ONE_IMAGE_LATEST: $ALL_IN_ONE_IMAGE_NAME:latest
   PIP_CACHE_DIR: "$CI_PROJECT_DIR/pip-cache"
   PYTHONDONTWRITEBYTECODE: "true"
   REVIEW_DOMAIN: preview.funkwhale.audio
@@ -16,6 +18,7 @@ stages:
   - deploy
 
 review_front:
+  interruptible: true
   stage: review
   image: node:11
   when: manual
@@ -54,8 +57,8 @@ review_front:
     name: review/front/$CI_COMMIT_REF_NAME
     url: http://$CI_PROJECT_NAMESPACE.pages.funkwhale.audio/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/front-review/index.html
 
-
 review_docs:
+  interruptible: true
   stage: review
   when: manual
   allow_failure: true
@@ -68,7 +71,7 @@ review_docs:
     - cd docs
     - apt-get update
     - apt-get install -y graphviz
-    - pip install sphinx
+    - pip install sphinx sphinx_rtd_theme
   script:
     - ./build_docs.sh
   cache:
@@ -87,8 +90,8 @@ review_docs:
     name: review/docs/$CI_COMMIT_REF_NAME
     url: http://$CI_PROJECT_NAMESPACE.pages.funkwhale.audio/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/docs-review/index.html
 
-
 black:
+  interruptible: true
   image: python:3.6
   stage: lint
   variables:
@@ -99,6 +102,7 @@ black:
     - black --check --diff api/
 
 flake8:
+  interruptible: true
   image: python:3.6
   stage: lint
   variables:
@@ -113,6 +117,7 @@ flake8:
       - "$PIP_CACHE_DIR"
 
 test_api:
+  interruptible: true
   services:
     - postgres:11
     - redis:3
@@ -129,6 +134,7 @@ test_api:
   only:
     - branches
   before_script:
+    - apk add make
     - cd api
     - sed -i '/Pillow/d' requirements/base.txt
     - pip3 install -r requirements/base.txt
@@ -140,6 +146,7 @@ test_api:
     - docker
 
 test_front:
+  interruptible: true
   stage: test
   image: node:11
   before_script:
@@ -196,7 +203,7 @@ pages:
     - cd docs
     - apt-get update
     - apt-get install -y graphviz
-    - pip install sphinx
+    - pip install sphinx sphinx_rtd_theme
   script:
     - ./build_docs.sh
   cache:
@@ -218,10 +225,12 @@ docker_release:
     - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
     - cp -r front/dist api/frontend
     - (if [ "$CI_COMMIT_REF_NAME" == "develop" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
-    - cd api
   script:
-    - docker build -t $IMAGE .
+    - if [[ ! -z "$CI_COMMIT_TAG" ]]; then (./docs/get-releases-json.py | scripts/is-docker-latest.py $CI_COMMIT_TAG -) && export DOCKER_LATEST_TAG="-t $IMAGE_LATEST" || export DOCKER_LATEST_TAG=; fi
+    - cd api
+    - docker build -t $IMAGE $DOCKER_LATEST_TAG .
     - docker push $IMAGE
+    - if [[ ! -z "$DOCKER_LATEST_TAG" ]]; then docker push $IMAGE_LATEST; fi
   only:
     - develop@funkwhale/funkwhale
     - tags@funkwhale/funkwhale
@@ -239,6 +248,7 @@ docker_all_in_one_release:
     - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
     - (if [ "$CI_COMMIT_REF_NAME" == "develop" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
   script:
+    - if [[ ! -z "$CI_COMMIT_TAG" ]]; then (./docs/get-releases-json.py | scripts/is-docker-latest.py $CI_COMMIT_TAG -) && export DOCKER_LATEST_TAG="-t $ALL_IN_ONE_IMAGE_LATEST" || export DOCKER_LATEST_TAG=; fi
     - wget $ALL_IN_ONE_ARTIFACT_URL -O all_in_one.zip
     - unzip -o all_in_one.zip -d tmpdir
     - mv tmpdir/docker-funkwhale-$ALL_IN_ONE_REF $BUILD_PATH && rmdir tmpdir
@@ -246,8 +256,9 @@ docker_all_in_one_release:
     - cp -r front $BUILD_PATH/src/front
     - cd $BUILD_PATH
     - ./scripts/download-nginx-template.sh src/ $CI_COMMIT_REF_NAME
-    - docker build -t $ALL_IN_ONE_IMAGE .
+    - docker build -t $ALL_IN_ONE_IMAGE $DOCKER_LATEST_TAG .
     - docker push $ALL_IN_ONE_IMAGE
+    - if [[ ! -z "$DOCKER_LATEST_TAG" ]]; then docker push $ALL_IN_ONE_IMAGE_LATEST; fi
   only:
     - develop@funkwhale/funkwhale
     - tags@funkwhale/funkwhale

CONTRIBUTING.rst

@@ -185,9 +185,9 @@ If you prefer to launch them in the background instead, use the ``-d`` flag, and
 
 Once everything is up, you can access the various funkwhale's components:
 
-- The Vue webapp, on http://localhost:8080
-- The API, on http://localhost:8080/api/v1/
-- The django admin, on http://localhost:8080/api/admin/
+- The Vue webapp, on http://localhost:8000
+- The API, on http://localhost:8000/api/v1/
+- The django admin, on http://localhost:800/api/admin/
 
 Stopping everything
 ^^^^^^^^^^^^^^^^^^^

api/Dockerfile

@@ -17,6 +17,7 @@ RUN \
     libpq              \
     libmagic           \
     libffi-dev         \
+    make         \
     zlib-dev           \
     openldap-dev && \
     \
@@ -44,7 +45,7 @@ RUN \
     if [ "$install_dev_deps" = "1" ] ; then echo "Installing dev dependencies" && pip3 install --no-cache-dir -r /requirements/local.txt -r /requirements/test.txt ; else echo "Skipping dev deps installation" ; fi
 
 ENTRYPOINT ["./compose/django/entrypoint.sh"]
-CMD ["./compose/django/daphne.sh"]
+CMD ["./compose/django/server.sh"]
 
 COPY . /app
 WORKDIR /app

api/compose/django/daphne.sh → api/compose/django/server.sh

 #!/bin/bash -eux
 python /app/manage.py collectstatic --noinput
-daphne -b 0.0.0.0 -p 5000 config.asgi:application --proxy-headers
+gunicorn config.asgi:application -w ${FUNKWHALE_WEB_WORKERS-1} -k uvicorn.workers.UvicornWorker -b 0.0.0.0:5000

api/config/api_urls.py

@@ -2,18 +2,20 @@ from django.conf.urls import include, url
 from dynamic_preferences.api.viewsets import GlobalPreferencesViewSet
 from rest_framework import routers
 from rest_framework.urlpatterns import format_suffix_patterns
-from rest_framework_jwt import views as jwt_views
 
 from funkwhale_api.activity import views as activity_views
 from funkwhale_api.common import views as common_views
+from funkwhale_api.common import routers as common_routers
 from funkwhale_api.music import views
 from funkwhale_api.playlists import views as playlists_views
 from funkwhale_api.subsonic.views import SubsonicViewSet
+from funkwhale_api.tags import views as tags_views
+from funkwhale_api.users import jwt_views
 
-router = routers.SimpleRouter()
+router = common_routers.OptionalSlashRouter()
 router.register(r"settings", GlobalPreferencesViewSet, basename="settings")
 router.register(r"activity", activity_views.ActivityViewSet, "activity")
-router.register(r"tags", views.TagViewSet, "tags")
+router.register(r"tags", tags_views.TagViewSet, "tags")
 router.register(r"tracks", views.TrackViewSet, "tracks")
 router.register(r"uploads", views.UploadViewSet, "uploads")
 router.register(r"libraries", views.LibraryViewSet, "libraries")
@@ -79,8 +81,9 @@ v1_patterns += [
         r"^oauth/",
         include(("funkwhale_api.users.oauth.urls", "oauth"), namespace="oauth"),
     ),
-    url(r"^token/$", jwt_views.obtain_jwt_token, name="token"),
-    url(r"^token/refresh/$", jwt_views.refresh_jwt_token, name="token_refresh"),
+    url(r"^token/?$", jwt_views.obtain_jwt_token, name="token"),
+    url(r"^token/refresh/?$", jwt_views.refresh_jwt_token, name="token_refresh"),
+    url(r"^rate-limit/?$", common_views.RateLimitView.as_view(), name="rate-limit"),
 ]
 
 urlpatterns = [

api/config/settings/common.py

@@ -11,7 +11,8 @@ https://docs.djangoproject.com/en/dev/ref/settings/
 from __future__ import absolute_import, unicode_literals
 
 import datetime
-import logging
+import logging.config
+import sys
 
 from urllib.parse import urlsplit
 
@@ -20,13 +21,44 @@ from celery.schedules import crontab
 
 from funkwhale_api import __version__
 
-logger = logging.getLogger(__name__)
+logger = logging.getLogger("funkwhale_api.config")
 ROOT_DIR = environ.Path(__file__) - 3  # (/a/b/myfile.py - 3 = /)
 APPS_DIR = ROOT_DIR.path("funkwhale_api")
 
 env = environ.Env()
+
+LOGLEVEL = env("LOGLEVEL", default="info").upper()
+LOGGING_CONFIG = None
+logging.config.dictConfig(
+    {
+        "version": 1,
+        "disable_existing_loggers": False,
+        "formatters": {
+            "console": {"format": "%(asctime)s %(name)-12s %(levelname)-8s %(message)s"}
+        },
+        "handlers": {
+            "console": {"class": "logging.StreamHandler", "formatter": "console"},
+            # # Add Handler for Sentry for `warning` and above
+            # 'sentry': {
+            #     'level': 'WARNING',
+            #     'class': 'raven.contrib.django.raven_compat.handlers.SentryHandler',
+            # },
+        },
+        "loggers": {
+            "funkwhale_api": {
+                "level": LOGLEVEL,
+                "handlers": ["console"],
+                # required to avoid double logging with root logger
+                "propagate": False,
+            },
+            "": {"level": "WARNING", "handlers": ["console"]},
+        },
+    }
+)
+
 env_file = env("ENV_FILE", default=None)
 if env_file:
+    logger.info("Loading specified env file at %s", env_file)
     # we have an explicitely specified env file
     # so we try to load and it fail loudly if it does not exist
     env.read_env(env_file)
@@ -49,6 +81,11 @@ else:
         logger.info("Loaded env file at %s/.env", path)
         break
 
+FUNKWHALE_PLUGINS_PATH = env(
+    "FUNKWHALE_PLUGINS_PATH", default="/srv/funkwhale/plugins/"
+)
+sys.path.append(FUNKWHALE_PLUGINS_PATH)
+
 FUNKWHALE_HOSTNAME = None
 FUNKWHALE_HOSTNAME_SUFFIX = env("FUNKWHALE_HOSTNAME_SUFFIX", default=None)
 FUNKWHALE_HOSTNAME_PREFIX = env("FUNKWHALE_HOSTNAME_PREFIX", default=None)
@@ -124,7 +161,6 @@ THIRD_PARTY_APPS = (
     "oauth2_provider",
     "rest_framework",
     "rest_framework.authtoken",
-    "taggit",
     "rest_auth",
     "rest_auth.registration",
     "dynamic_preferences",
@@ -147,7 +183,6 @@ if RAVEN_ENABLED:
     }
     THIRD_PARTY_APPS += ("raven.contrib.django.raven_compat",)
 
-
 # Apps specific for this project go here.
 LOCAL_APPS = (
     "funkwhale_api.common.apps.CommonConfig",
@@ -160,29 +195,44 @@ LOCAL_APPS = (
     "funkwhale_api.requests",
     "funkwhale_api.favorites",
     "funkwhale_api.federation",
-    "funkwhale_api.moderation",
+    "funkwhale_api.moderation.apps.ModerationConfig",
     "funkwhale_api.radios",
     "funkwhale_api.history",
     "funkwhale_api.playlists",
     "funkwhale_api.subsonic",
+    "funkwhale_api.tags",
 )
 
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
 
-INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS
+
+PLUGINS = [p for p in env.list("FUNKWHALE_PLUGINS", default=[]) if p]
+if PLUGINS:
+    logger.info("Running with the following plugins enabled: %s", ", ".join(PLUGINS))
+else:
+    logger.info("Running with no plugins")
+
+INSTALLED_APPS = (
+    DJANGO_APPS
+    + THIRD_PARTY_APPS
+    + LOCAL_APPS
+    + tuple(["{}.apps.Plugin".format(p) for p in PLUGINS])
+)
 
 # MIDDLEWARE CONFIGURATION
 # ------------------------------------------------------------------------------
 MIDDLEWARE = (
+    "django.middleware.security.SecurityMiddleware",
+    "django.middleware.clickjacking.XFrameOptionsMiddleware",
+    "corsheaders.middleware.CorsMiddleware",
     "funkwhale_api.common.middleware.SPAFallbackMiddleware",
     "django.contrib.sessions.middleware.SessionMiddleware",
-    "corsheaders.middleware.CorsMiddleware",
     "django.middleware.common.CommonMiddleware",
     "django.middleware.csrf.CsrfViewMiddleware",
     "django.contrib.auth.middleware.AuthenticationMiddleware",
     "django.contrib.messages.middleware.MessageMiddleware",
-    "django.middleware.clickjacking.XFrameOptionsMiddleware",
     "funkwhale_api.users.middleware.RecordActivityMiddleware",
+    "funkwhale_api.common.middleware.ThrottleStatusMiddleware",
 )
 
 # DEBUG
@@ -350,6 +400,8 @@ ASGI_APPLICATION = "config.routing.application"
 
 # This ensures that Django will be able to detect a secure connection
 SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
+SECURE_BROWSER_XSS_FILTER = True
+SECURE_CONTENT_TYPE_NOSNIFF = True
 
 # AUTHENTICATION CONFIGURATION
 # ------------------------------------------------------------------------------
@@ -516,14 +568,32 @@ CELERY_BEAT_SCHEDULE = {
 
 NODEINFO_REFRESH_DELAY = env.int("NODEINFO_REFRESH_DELAY", default=3600 * 24)
 
+
+def get_user_secret_key(user):
+    from django.conf import settings
+
+    return settings.SECRET_KEY + str(user.secret_key)
+
+
 JWT_AUTH = {
     "JWT_ALLOW_REFRESH": True,
     "JWT_EXPIRATION_DELTA": datetime.timedelta(days=7),
     "JWT_REFRESH_EXPIRATION_DELTA": datetime.timedelta(days=30),
     "JWT_AUTH_HEADER_PREFIX": "JWT",
-    "JWT_GET_USER_SECRET_KEY": lambda user: user.secret_key,
+    "JWT_GET_USER_SECRET_KEY": get_user_secret_key,
 }
 OLD_PASSWORD_FIELD_ENABLED = True
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator"
+    },
+    {
+        "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
+        "OPTIONS": {"min_length": env.int("PASSWORD_MIN_LENGTH", default=8)},
+    },
+    {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator"},
+    {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"},
+]
 ACCOUNT_ADAPTER = "funkwhale_api.users.adapters.FunkwhaleAccountAdapter"
 CORS_ORIGIN_ALLOW_ALL = True
 # CORS_ORIGIN_WHITELIST = (
@@ -557,8 +627,151 @@ REST_FRAMEWORK = {
         "django_filters.rest_framework.DjangoFilterBackend",
     ),
     "DEFAULT_RENDERER_CLASSES": ("rest_framework.renderers.JSONRenderer",),
+    "NUM_PROXIES": env.int("NUM_PROXIES", default=1),
+}
+THROTTLING_ENABLED = env.bool("THROTTLING_ENABLED", default=True)
+if THROTTLING_ENABLED:
+    REST_FRAMEWORK["DEFAULT_THROTTLE_CLASSES"] = env.list(
+        "THROTTLE_CLASSES",
+        default=["funkwhale_api.common.throttling.FunkwhaleThrottle"],
+    )
+
+THROTTLING_SCOPES = {
+    "*": {"anonymous": "anonymous-wildcard", "authenticated": "authenticated-wildcard"},
+    "create": {
+        "authenticated": "authenticated-create",
+        "anonymous": "anonymous-create",
+    },
+    "list": {"authenticated": "authenticated-list", "anonymous": "anonymous-list"},
+    "retrieve": {
+        "authenticated": "authenticated-retrieve",
+        "anonymous": "anonymous-retrieve",
+    },
+    "destroy": {
+        "authenticated": "authenticated-destroy",
+        "anonymous": "anonymous-destroy",
+    },
+    "update": {
+        "authenticated": "authenticated-update",
+        "anonymous": "anonymous-update",
+    },
+    "partial_update": {
+        "authenticated": "authenticated-update",
+        "anonymous": "anonymous-update",
+    },
 }
 
+THROTTLING_USER_RATES = env.dict("THROTTLING_RATES", default={})
+
+THROTTLING_RATES = {
+    "anonymous-wildcard": {
+        "rate": THROTTLING_USER_RATES.get("anonymous-wildcard", "1000/h"),
+        "description": "Anonymous requests not covered by other limits",
+    },
+    "authenticated-wildcard": {
+        "rate": THROTTLING_USER_RATES.get("authenticated-wildcard", "2000/h"),
+        "description": "Authenticated requests not covered by other limits",
+    },
+    "authenticated-create": {
+        "rate": THROTTLING_USER_RATES.get("authenticated-create", "1000/hour"),
+        "description": "Authenticated POST requests",
+    },
+    "anonymous-create": {
+        "rate": THROTTLING_USER_RATES.get("anonymous-create", "1000/day"),
+        "description": "Anonymous POST requests",
+    },
+    "authenticated-list": {
+        "rate": THROTTLING_USER_RATES.get("authenticated-list", "10000/hour"),
+        "description": "Authenticated GET requests on resource lists",
+    },
+    "anonymous-list": {
+        "rate": THROTTLING_USER_RATES.get("anonymous-list", "10000/day"),
+        "description": "Anonymous GET requests on resource lists",
+    },
+    "authenticated-retrieve": {
+        "rate": THROTTLING_USER_RATES.get("authenticated-retrieve", "10000/hour"),
+        "description": "Authenticated GET requests on resource detail",
+    },
+    "anonymous-retrieve": {
+        "rate": THROTTLING_USER_RATES.get("anonymous-retrieve", "10000/day"),
+        "description": "Anonymous GET requests on resource detail",
+    },
+    "authenticated-destroy": {
+        "rate": THROTTLING_USER_RATES.get("authenticated-destroy", "500/hour"),
+        "description": "Authenticated DELETE requests on resource detail",
+    },
+    "anonymous-destroy": {
+        "rate": THROTTLING_USER_RATES.get("anonymous-destroy", "1000/day"),
+        "description": "Anonymous DELETE requests on resource detail",
+    },
+    "authenticated-update": {
+        "rate": THROTTLING_USER_RATES.get("authenticated-update", "1000/hour"),
+        "description": "Authenticated PATCH and PUT requests on resource detail",
+    },
+    "anonymous-update": {
+        "rate": THROTTLING_USER_RATES.get("anonymous-update", "1000/day"),
+        "description": "Anonymous PATCH and PUT requests on resource detail",
+    },
+    # potentially spammy / dangerous endpoints
+    "authenticated-reports": {
+        "rate": THROTTLING_USER_RATES.get("authenticated-reports", "100/day"),
+        "description": "Authenticated report submission",
+    },
+    "anonymous-reports": {
+        "rate": THROTTLING_USER_RATES.get("anonymous-reports", "10/day"),
+        "description": "Anonymous report submission",
+    },
+    "authenticated-oauth-app": {
+        "rate": THROTTLING_USER_RATES.get("authenticated-oauth-app", "10/hour"),
+        "description": "Authenticated OAuth app creation",
+    },
+    "anonymous-oauth-app": {
+        "rate": THROTTLING_USER_RATES.get("anonymous-oauth-app", "10/day"),
+        "description": "Anonymous OAuth app creation",
+    },
+    "oauth-authorize": {
+        "rate": THROTTLING_USER_RATES.get("oauth-authorize", "100/hour"),
+        "description": "OAuth app authorization",
+    },
+    "oauth-token": {
+        "rate": THROTTLING_USER_RATES.get("oauth-token", "100/hour"),
+        "description": "OAuth token creation",
+    },
+    "oauth-revoke-token": {
+        "rate": THROTTLING_USER_RATES.get("oauth-revoke-token", "100/hour"),
+        "description": "OAuth token deletion",
+    },
+    "jwt-login": {
+        "rate": THROTTLING_USER_RATES.get("jwt-login", "30/hour"),
+        "description": "JWT token creation",
+    },
+    "jwt-refresh": {
+        "rate": THROTTLING_USER_RATES.get("jwt-refresh", "30/hour"),
+        "description": "JWT token refresh",
+    },
+    "signup": {
+        "rate": THROTTLING_USER_RATES.get("signup", "10/day"),
+        "description": "Account creation",
+    },
+    "verify-email": {
+        "rate": THROTTLING_USER_RATES.get("verify-email", "20/h"),
+        "description": "Email address confirmation",
+    },
+    "password-change": {
+        "rate": THROTTLING_USER_RATES.get("password-change", "20/h"),
+        "description": "Password change (when authenticated)",
+    },
+    "password-reset": {
+        "rate": THROTTLING_USER_RATES.get("password-reset", "20/h"),
+        "description": "Password reset request",
+    },
+    "password-reset-confirm": {
+        "rate": THROTTLING_USER_RATES.get("password-reset-confirm", "20/h"),
+        "description": "Password reset confirmation",
+    },
+}
+
+
 BROWSABLE_API_ENABLED = env.bool("BROWSABLE_API_ENABLED", default=False)
 if BROWSABLE_API_ENABLED:
     REST_FRAMEWORK["DEFAULT_RENDERER_CLASSES"] += (
@@ -661,3 +874,17 @@ ACTOR_KEY_ROTATION_DELAY = env.int("ACTOR_KEY_ROTATION_DELAY", default=3600 * 48
 SUBSONIC_DEFAULT_TRANSCODING_FORMAT = (
     env("SUBSONIC_DEFAULT_TRANSCODING_FORMAT", default="mp3") or None
 )
+
+# extra tags will be ignored
+TAGS_MAX_BY_OBJ = env.int("TAGS_MAX_BY_OBJ", default=30)
+FEDERATION_OBJECT_FETCH_DELAY = env.int(
+    "FEDERATION_OBJECT_FETCH_DELAY", default=60 * 24 * 3
+)
+
+MODERATION_EMAIL_NOTIFICATIONS_ENABLED = env.bool(
+    "MODERATION_EMAIL_NOTIFICATIONS_ENABLED", default=True
+)
+
+# Delay in days after signup before we show the "support us" messages
+INSTANCE_SUPPORT_MESSAGE_DELAY = env.int("INSTANCE_SUPPORT_MESSAGE_DELAY", default=15)
+FUNKWHALE_SUPPORT_MESSAGE_DELAY = env.int("FUNKWHALE_SUPPORT_MESSAGE_DELAY", default=15)

api/config/settings/production.py

@@ -73,50 +73,4 @@ TEMPLATES[0]["OPTIONS"]["loaders"] = [
 # ------------------------------------------------------------------------------
 # Heroku URL does not pass the DB number, so we parse it in
 
-
-# LOGGING CONFIGURATION
-# ------------------------------------------------------------------------------
-# See: https://docs.djangoproject.com/en/dev/ref/settings/#logging
-# A sample logging configuration. The only tangible logging
-# performed by this configuration is to send an email to
-# the site admins on every HTTP 500 error when DEBUG=False.
-# See http://docs.djangoproject.com/en/dev/topics/logging for
-# more details on how to customize your logging configuration.
-LOGGING = {
-    "version": 1,
-    "disable_existing_loggers": False,
-    "filters": {"require_debug_false": {"()": "django.utils.log.RequireDebugFalse"}},
-    "formatters": {
-        "verbose": {
-            "format": "%(levelname)s %(asctime)s %(module)s "
-            "%(process)d %(thread)d %(message)s"
-        }
-    },
-    "handlers": {
-        "mail_admins": {
-            "level": "ERROR",
-            "filters": ["require_debug_false"],
-            "class": "django.utils.log.AdminEmailHandler",
-        },
-        "console": {
-            "level": "DEBUG",
-            "class": "logging.StreamHandler",
-            "formatter": "verbose",
-        },
-    },
-    "loggers": {
-        "django.request": {
-            "handlers": ["mail_admins"],
-            "level": "ERROR",
-            "propagate": True,
-        },
-        "django.security.DisallowedHost": {
-            "level": "ERROR",
-            "handlers": ["console", "mail_admins"],
-            "propagate": True,
-        },
-    },
-}
-
-
 # Your production stuff: Below this line define 3rd party library settings

api/config/spa_urls.py

@@ -15,4 +15,9 @@ urlpatterns = [
         spa_views.library_artist,
         name="library_artist",
     ),
+    urls.re_path(
+        r"^library/playlists/(?P<pk>\d+)/?$",
+        spa_views.library_playlist,
+        name="library_playlist",
+    ),
 ]

api/config/urls.py

@@ -19,8 +19,7 @@ urlpatterns = [
             ("funkwhale_api.federation.urls", "federation"), namespace="federation"
         ),
     ),
-    url(r"^api/v1/auth/", include("rest_auth.urls")),
-    url(r"^api/v1/auth/registration/", include("funkwhale_api.users.rest_auth_urls")),
+    url(r"^api/v1/auth/", include("funkwhale_api.users.rest_auth_urls")),
     url(r"^accounts/", include("allauth.urls")),
     # Your stuff: custom urls includes go here
 ]

api/funkwhale_api/__init__.py

 # -*- coding: utf-8 -*-
-__version__ = "0.19.1"
+__version__ = "0.20.0-rc1"
 __version_info__ = tuple(
     [
         int(num) if num.isdigit() else num

api/funkwhale_api/common/fields.py

 import django_filters
 from django import forms
+from django.core.serializers.json import DjangoJSONEncoder
 from django.db import models
 
+from rest_framework import serializers
+
 from . import search
 
 PRIVACY_LEVEL_CHOICES = [
@@ -52,3 +55,118 @@ class SmartSearchFilter(django_filters.CharFilter):
         except (forms.ValidationError):
             return qs.none()
         return search.apply(qs, cleaned)
+
+
+def get_generic_filter_query(value, relation_name, choices):
+    parts = value.split(":", 1)
+    type = parts[0]
+    try:
+        conf = choices[type]
+    except KeyError:
+        raise forms.ValidationError("Invalid type")
+    related_queryset = conf["queryset"]
+    related_model = related_queryset.model
+    filter_query = models.Q(
+        **{
+            "{}_content_type__app_label".format(
+                relation_name
+            ): related_model._meta.app_label,
+            "{}_content_type__model".format(
+                relation_name
+            ): related_model._meta.model_name,
+        }
+    )
+    if len(parts) > 1:
+        id_attr = conf.get("id_attr", "id")
+        id_field = conf.get("id_field", serializers.IntegerField(min_value=1))
+        try:
+            id_value = parts[1]
+            id_value = id_field.to_internal_value(id_value)
+        except (TypeError, KeyError, serializers.ValidationError):
+            raise forms.ValidationError("Invalid id")
+        query_getter = conf.get(
+            "get_query", lambda attr, value: models.Q(**{attr: value})
+        )
+        obj_query = query_getter(id_attr, id_value)
+        try:
+            obj = related_queryset.get(obj_query)
+        except related_queryset.model.DoesNotExist:
+            raise forms.ValidationError("Invalid object")
+        filter_query &= models.Q(**{"{}_id".format(relation_name): obj.id})
+
+    return filter_query
+
+
+class GenericRelationFilter(django_filters.CharFilter):
+    def __init__(self, relation_name, choices, *args, **kwargs):
+        self.relation_name = relation_name
+        self.choices = choices
+        super().__init__(*args, **kwargs)
+
+    def filter(self, qs, value):
+        if not value:
+            return qs
+        try:
+            filter_query = get_generic_filter_query(
+                value, relation_name=self.relation_name, choices=self.choices
+            )
+        except forms.ValidationError:
+            return qs.none()
+        return qs.filter(filter_query)
+
+
+class GenericRelation(serializers.JSONField):
+    def __init__(self, choices, *args, **kwargs):
+        self.choices = choices
+        self.encoder = kwargs.setdefault("encoder", DjangoJSONEncoder)
+        super().__init__(*args, **kwargs)
+
+    def to_representation(self, value):
+        if not value:
+            return
+        type = None
+        id = None
+        id_attr = None
+        for key, choice in self.choices.items():
+            if isinstance(value, choice["queryset"].model):
+                type = key
+                id_attr = choice.get("id_attr", "id")
+                id = getattr(value, id_attr)
+                break
+
+        if type:
+            return {"type": type, id_attr: id}
+
+    def to_internal_value(self, v):
+        v = super().to_internal_value(v)
+
+        if not v or not isinstance(v, dict):
+            raise serializers.ValidationError("Invalid data")
+
+        try:
+            type = v["type"]
+            field = serializers.ChoiceField(choices=list(self.choices.keys()))
+            type = field.to_internal_value(type)
+        except (TypeError, KeyError, serializers.ValidationError):
+            raise serializers.ValidationError("Invalid type")
+
+        conf = self.choices[type]
+        id_attr = conf.get("id_attr", "id")
+        id_field = conf.get("id_field", serializers.IntegerField(min_value=1))
+        queryset = conf["queryset"]
+        try:
+            id_value = v[id_attr]
+            id_value = id_field.to_internal_value(id_value)
+        except (TypeError, KeyError, serializers.ValidationError):
+            raise serializers.ValidationError("Invalid {}".format(id_attr))
+
+        query_getter = conf.get(
+            "get_query", lambda attr, value: models.Q(**{attr: value})
+        )
+        query = query_getter(id_attr, id_value)
+        try:
+            obj = queryset.get(query)
+        except queryset.model.DoesNotExist:
+            raise serializers.ValidationError("Object not found")
+
+        return obj

api/funkwhale_api/common/filters.py

@@ -15,7 +15,7 @@ class NoneObject(object):
 
 
 NONE = NoneObject()
-NULL_BOOLEAN_CHOICES = [
+BOOLEAN_CHOICES = [
     (True, True),
     ("true", True),
     ("True", True),
@@ -26,6 +26,8 @@ NULL_BOOLEAN_CHOICES = [
     ("False", False),
     ("0", False),
     ("no", False),
+]
+NULL_BOOLEAN_CHOICES = BOOLEAN_CHOICES + [
     ("None", NONE),
     ("none", NONE),
     ("Null", NONE),
@@ -76,10 +78,26 @@ def clean_null_boolean_filter(v):
     return v
 
 
+def clean_boolean_filter(v):
+    return CoerceChoiceField(choices=BOOLEAN_CHOICES).clean(v)
+
+
 def get_null_boolean_filter(name):
     return {"handler": lambda v: Q(**{name: clean_null_boolean_filter(v)})}
 
 
+def get_boolean_filter(name):
+    return {"handler": lambda v: Q(**{name: clean_boolean_filter(v)})}
+
+
+def get_generic_relation_filter(relation_name, choices):
+    return {
+        "handler": lambda v: fields.get_generic_filter_query(
+            v, relation_name=relation_name, choices=choices
+        )
+    }
+
+
 class DummyTypedMultipleChoiceField(forms.TypedMultipleChoiceField):
     def valid_value(self, value):
         return True
@@ -142,7 +160,7 @@ class MutationFilter(filters.FilterSet):
                 "domain": {"to": "created_by__domain__name__iexact"},
                 "is_approved": get_null_boolean_filter("is_approved"),
                 "target": {"handler": filter_target},
-                "is_applied": {"to": "is_applied"},
+                "is_applied": get_boolean_filter("is_applied"),
             },
         )
     )

api/funkwhale_api/common/management/commands/load_test_data.py

+import math
+import random
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from django.db import transaction
+
+
+from funkwhale_api.federation import keys
+from funkwhale_api.federation import models as federation_models
+from funkwhale_api.music import models as music_models
+from funkwhale_api.tags import models as tags_models
+from funkwhale_api.users import models as users_models
+
+
+BATCH_SIZE = 500
+
+
+def create_local_accounts(factories, count, dependencies):
+    password = factories["users.User"].build().password
+    users = factories["users.User"].build_batch(size=count)
+    for user in users:
+        # we set the hashed password by hand, because computing one for each user
+        # is CPU intensive
+        user.password = password
+    users = users_models.User.objects.bulk_create(users, batch_size=BATCH_SIZE)
+    actors = []
+    domain = federation_models.Domain.objects.get_or_create(
+        name=settings.FEDERATION_HOSTNAME
+    )[0]
+    users = [u for u in users if u.pk]
+    private, public = keys.get_key_pair()
+    for user in users:
+        if not user.pk:
+            continue
+        actor = federation_models.Actor(
+            private_key=private.decode("utf-8"),
+            public_key=public.decode("utf-8"),
+            **users_models.get_actor_data(user.username, domain=domain)
+        )
+        actors.append(actor)
+    actors = federation_models.Actor.objects.bulk_create(actors, batch_size=BATCH_SIZE)
+    for user, actor in zip(users, actors):
+        user.actor = actor
+    users_models.User.objects.bulk_update(users, ["actor"])
+    return actors
+
+
+def create_taggable_items(dependency):
+    def inner(factories, count, dependencies):
+
+        objs = []
+        tagged_objects = dependencies.get(
+            dependency, list(CONFIG_BY_ID[dependency]["model"].objects.all().only("pk"))
+        )
+        tags = dependencies.get("tags", list(tags_models.Tag.objects.all().only("pk")))
+        for i in range(count):
+            tag = random.choice(tags)
+            tagged_object = random.choice(tagged_objects)
+            objs.append(
+                factories["tags.TaggedItem"].build(
+                    content_object=tagged_object, tag=tag
+                )
+            )
+
+        return tags_models.TaggedItem.objects.bulk_create(
+            objs, batch_size=BATCH_SIZE, ignore_conflicts=True
+        )
+
+    return inner
+
+
+CONFIG = [
+    {
+        "id": "tracks",
+        "model": music_models.Track,
+        "factory": "music.Track",
+        "factory_kwargs": {"artist": None, "album": None},
+        "depends_on": [
+            {"field": "album", "id": "albums", "default_factor": 0.1},
+            {"field": "artist", "id": "artists", "default_factor": 0.05},
+        ],
+    },
+    {
+        "id": "albums",
+        "model": music_models.Album,
+        "factory": "music.Album",
+        "factory_kwargs": {"artist": None},
+        "depends_on": [{"field": "artist", "id": "artists", "default_factor": 0.3}],
+    },
+    {"id": "artists", "model": music_models.Artist, "factory": "music.Artist"},
+    {
+        "id": "local_accounts",
+        "model": federation_models.Actor,
+        "handler": create_local_accounts,
+    },
+    {
+        "id": "local_libraries",
+        "model": music_models.Library,
+        "factory": "music.Library",
+        "factory_kwargs": {"actor": None},
+        "depends_on": [{"field": "actor", "id": "local_accounts", "default_factor": 1}],
+    },
+    {
+        "id": "local_uploads",
+        "model": music_models.Upload,
+        "factory": "music.Upload",
+        "factory_kwargs": {"import_status": "finished", "library": None, "track": None},
+        "depends_on": [
+            {
+                "field": "library",
+                "id": "local_libraries",
+                "default_factor": 0.05,
+                "queryset": music_models.Library.objects.all().select_related(
+                    "actor__user"
+                ),
+            },
+            {"field": "track", "id": "tracks", "default_factor": 1},
+        ],
+    },
+    {"id": "tags", "model": tags_models.Tag, "factory": "tags.Tag"},
+    {
+        "id": "track_tags",
+        "model": tags_models.TaggedItem,
+        "queryset": tags_models.TaggedItem.objects.filter(
+            content_type__app_label="music", content_type__model="track"
+        ),
+        "handler": create_taggable_items("tracks"),
+        "depends_on": [
+            {
+                "field": "tag",
+                "id": "tags",
+                "default_factor": 0.1,
+                "queryset": tags_models.Tag.objects.all(),
+                "set": False,
+            },
+            {
+                "field": "content_object",
+                "id": "tracks",
+                "default_factor": 1,
+                "set": False,
+            },
+        ],
+    },
+    {
+        "id": "album_tags",
+        "model": tags_models.TaggedItem,
+        "queryset": tags_models.TaggedItem.objects.filter(
+            content_type__app_label="music", content_type__model="album"
+        ),
+        "handler": create_taggable_items("albums"),
+        "depends_on": [
+            {
+                "field": "tag",
+                "id": "tags",
+                "default_factor": 0.1,
+                "queryset": tags_models.Tag.objects.all(),
+                "set": False,
+            },
+            {
+                "field": "content_object",
+                "id": "albums",
+                "default_factor": 1,
+                "set": False,
+            },
+        ],
+    },
+    {
+        "id": "artist_tags",
+        "model": tags_models.TaggedItem,
+        "queryset": tags_models.TaggedItem.objects.filter(
+            content_type__app_label="music", content_type__model="artist"
+        ),
+        "handler": create_taggable_items("artists"),
+        "depends_on": [
+            {
+                "field": "tag",
+                "id": "tags",
+                "default_factor": 0.1,
+                "queryset": tags_models.Tag.objects.all(),
+                "set": False,
+            },
+            {
+                "field": "content_object",
+                "id": "artists",
+                "default_factor": 1,
+                "set": False,
+            },
+        ],
+    },
+]
+
+CONFIG_BY_ID = {c["id"]: c for c in CONFIG}
+
+
+class Rollback(Exception):
+    pass
+
+
+def create_objects(row, factories, count, **factory_kwargs):
+    return factories[row["factory"]].build_batch(size=count, **factory_kwargs)
+
+
+class Command(BaseCommand):
+    help = """
+    Inject demo data into your database. Useful for load testing, or setting up a demo instance.
+
+    Use with caution and only if you know what you are doing.
+    """
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--no-dry-run",
+            action="store_false",
+            dest="dry_run",
+            help="Commit the changes to the database",
+        )
+        parser.add_argument(
+            "--create-dependencies", action="store_true", dest="create_dependencies"
+        )
+        for row in CONFIG:
+            parser.add_argument(
+                "--{}".format(row["id"].replace("_", "-")),
+                dest=row["id"],
+                type=int,
+                help="Number of {} objects to create".format(row["id"]),
+            )
+            dependencies = row.get("depends_on", [])
+            for dependency in dependencies:
+                parser.add_argument(
+                    "--{}-{}-factor".format(row["id"], dependency["field"]),
+                    dest="{}_{}_factor".format(row["id"], dependency["field"]),
+                    type=float,
+                    help="Number of {} objects to create per {} object".format(
+                        dependency["id"], row["id"]
+                    ),
+                )
+
+    def handle(self, *args, **options):
+        from django.apps import apps
+        from funkwhale_api import factories
+
+        app_names = [app.name for app in apps.app_configs.values()]
+        factories.registry.autodiscover(app_names)
+        try:
+            return self.inner_handle(*args, **options)
+        except Rollback:
+            pass
+
+    @transaction.atomic
+    def inner_handle(self, *args, **options):
+        results = {}
+        for row in CONFIG:
+            self.create_batch(row, results, options, count=options.get(row["id"]))
+
+        self.stdout.write("\nFinal state of database:\n\n")
+        for row in CONFIG:
+            qs = row.get("queryset", row["model"].objects.all())
+            total = qs.count()
+            self.stdout.write("- {} {} objects".format(total, row["id"]))
+
+        self.stdout.write("")
+        if options["dry_run"]:
+
+            self.stdout.write(
+                "Run this command with --no-dry-run to commit the changes to the database"
+            )
+            raise Rollback()
+
+        self.stdout.write(self.style.SUCCESS("Done!"))
+
+    def create_batch(self, row, results, options, count):
+        from funkwhale_api import factories
+
+        if row["id"] in results:
+            # already generated
+            return results[row["id"]]
+        if not count:
+            return []
+        dependencies = row.get("depends_on", [])
+        create_dependencies = options.get("create_dependencies")
+        for dependency in dependencies:
+            dep_count = options.get(dependency["id"])
+            if not create_dependencies and dep_count is None:
+                continue
+            if dep_count is None:
+                factor = options[
+                    "{}_{}_factor".format(row["id"], dependency["field"])
+                ] or dependency.get("default_factor")
+                dep_count = math.ceil(factor * count)
+
+            results[dependency["id"]] = self.create_batch(
+                CONFIG_BY_ID[dependency["id"]], results, options, count=dep_count
+            )
+        self.stdout.write("Creating {} {}…".format(count, row["id"]))
+        handler = row.get("handler")
+        if handler:
+            objects = handler(factories.registry, count, dependencies=results)
+        else:
+            objects = create_objects(
+                row, factories.registry, count, **row.get("factory_kwargs", {})
+            )
+        for dependency in dependencies:
+            if not dependency.get("set", True):
+                continue
+            if create_dependencies:
+                candidates = results[dependency["id"]]
+            else:
+                # we use existing objects in the database
+                queryset = dependency.get(
+                    "queryset", CONFIG_BY_ID[dependency["id"]]["model"].objects.all()
+                )
+                candidates = list(queryset.values_list("pk", flat=True))
+                picked_pks = [random.choice(candidates) for _ in objects]
+                picked_objects = {o.pk: o for o in queryset.filter(pk__in=picked_pks)}
+            for i, obj in enumerate(objects):
+                if create_dependencies:
+                    value = random.choice(candidates)
+                else:
+                    value = picked_objects[picked_pks[i]]
+                setattr(obj, dependency["field"], value)
+        if not handler:
+            objects = row["model"].objects.bulk_create(objects, batch_size=BATCH_SIZE)
+        results[row["id"]] = objects
+        return objects

api/funkwhale_api/common/middleware.py

 import html
 import requests
+import time
 import xml.sax.saxutils
 
 from django import http
 from django.conf import settings
 from django.core.cache import caches
 from django import urls
+from rest_framework import views
 
 from . import preferences
+from . import throttling
 from . import utils
 
 EXCLUDED_PATHS = ["/api", "/federation", "/.well-known"]
@@ -176,3 +179,66 @@ class DevHttpsMiddleware:
                 lambda: request.__class__.get_host(request).replace(":80", ":443"),
             )
         return self.get_response(request)
+
+
+def monkey_patch_rest_initialize_request():
+    """
+    Rest framework use it's own APIRequest, meaning we can't easily
+    access our throttling info in the middleware. So me monkey patch the
+    `initialize_request` method from rest_framework to keep a link between both requests
+    """
+    original = views.APIView.initialize_request
+
+    def replacement(self, request, *args, **kwargs):
+        r = original(self, request, *args, **kwargs)
+        setattr(request, "_api_request", r)
+        return r
+
+    setattr(views.APIView, "initialize_request", replacement)
+
+
+monkey_patch_rest_initialize_request()
+
+
+class ThrottleStatusMiddleware:
+    """
+    Include useful information regarding throttling in API responses to
+    ensure clients can adapt.
+    """
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        try:
+            response = self.get_response(request)
+        except throttling.TooManyRequests:
+            # manual throttling in non rest_framework view, we have to return
+            # the proper response ourselves
+            response = http.HttpResponse(status=429)
+        request_to_check = request
+        try:
+            request_to_check = request._api_request
+        except AttributeError:
+            pass
+        throttle_status = getattr(request_to_check, "_throttle_status", None)
+        if throttle_status:
+            response["X-RateLimit-Limit"] = str(throttle_status["num_requests"])
+            response["X-RateLimit-Scope"] = str(throttle_status["scope"])
+            response["X-RateLimit-Remaining"] = throttle_status["num_requests"] - len(
+                throttle_status["history"]
+            )
+            response["X-RateLimit-Duration"] = str(throttle_status["duration"])
+            if throttle_status["history"]:
+                now = int(time.time())
+                # At this point, the client can send additional requests
+                oldtest_request = throttle_status["history"][-1]
+                remaining = throttle_status["duration"] - (now - int(oldtest_request))
+                response["Retry-After"] = str(remaining)
+                # At this point, all Rate Limit is reset to 0
+                latest_request = throttle_status["history"][0]
+                remaining = throttle_status["duration"] - (now - int(latest_request))
+                response["X-RateLimit-Reset"] = str(now + remaining)
+                response["X-RateLimit-ResetSeconds"] = str(remaining)
+
+        return response

api/funkwhale_api/common/migrations/0003_cit_extension.py

+# Generated by Django 2.0.2 on 2018-02-27 18:43
+from django.db import migrations
+from django.contrib.postgres.operations import CITextExtension
+
+
+class CustomCITExtension(CITextExtension):
+    def database_forwards(self, app_label, schema_editor, from_state, to_state):
+        check_sql = "SELECT 1 FROM pg_extension WHERE extname = 'citext'"
+        with schema_editor.connection.cursor() as cursor:
+            cursor.execute(check_sql)
+            result = cursor.fetchall()
+
+        if result:
+            return
+        return super().database_forwards(app_label, schema_editor, from_state, to_state)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("common", "0002_mutation")]
+
+    operations = [CustomCITExtension()]

api/funkwhale_api/common/models.py

@@ -73,7 +73,7 @@ class LocalFromFidQuerySet:
             return self.filter(~query)
 
 
-class MutationQuerySet(models.QuerySet):
+class GenericTargetQuerySet(models.QuerySet):
     def get_for_target(self, target):
         content_type = ContentType.objects.get_for_model(target)
         return self.filter(target_content_type=content_type, target_id=target.pk)
@@ -119,7 +119,7 @@ class Mutation(models.Model):
     )
     target = GenericForeignKey("target_content_type", "target_id")
 
-    objects = MutationQuerySet.as_manager()
+    objects = GenericTargetQuerySet.as_manager()
 
     def get_federation_id(self):
         if self.fid:

api/funkwhale_api/common/mutations.py

@@ -86,6 +86,7 @@ class MutationSerializer(serializers.Serializer):
 
 class UpdateMutationSerializer(serializers.ModelSerializer, MutationSerializer):
     serialized_relations = {}
+    previous_state_handlers = {}
 
     def __init__(self, *args, **kwargs):
         # we force partial mode, because update mutations are partial
@@ -139,16 +140,20 @@ class UpdateMutationSerializer(serializers.ModelSerializer, MutationSerializer):
         return get_update_previous_state(
             obj,
             *list(validated_data.keys()),
-            serialized_relations=self.serialized_relations
+            serialized_relations=self.serialized_relations,
+            handlers=self.previous_state_handlers,
         )
 
 
-def get_update_previous_state(obj, *fields, serialized_relations={}):
+def get_update_previous_state(obj, *fields, serialized_relations={}, handlers={}):
     if not fields:
         raise ValueError("You need to provide at least one field")
 
     state = {}
     for field in fields:
+        if field in handlers:
+            state[field] = handlers[field](obj)
+            continue
         value = getattr(obj, field)
         if isinstance(value, models.Model):
             # we store the related object id and repr for better UX

api/funkwhale_api/common/preferences.py

@@ -14,6 +14,11 @@ def get(pref):
     return manager[pref]
 
 
+def all():
+    manager = global_preferences_registry.manager()
+    return manager.all()
+
+
 def set(pref, value):
     manager = global_preferences_registry.manager()
     manager[pref] = value

api/funkwhale_api/common/routers.py

+from rest_framework.routers import SimpleRouter
+
+
+class OptionalSlashRouter(SimpleRouter):
+    def __init__(self):
+        super().__init__()
+        self.trailing_slash = "/?"

api/funkwhale_api/common/throttling.py

+import collections
+
+from django.core.cache import cache
+from rest_framework import throttling as rest_throttling
+
+from django.conf import settings
+
+
+def get_ident(request):
+    if hasattr(request, "user") and request.user.is_authenticated:
+        return {"type": "authenticated", "id": request.user.pk}
+    ident = rest_throttling.BaseThrottle().get_ident(request)
+
+    return {"type": "anonymous", "id": ident}
+
+
+def get_cache_key(scope, ident):
+    parts = ["throttling", scope, ident["type"], str(ident["id"])]
+    return ":".join(parts)
+
+
+def get_scope_for_action_and_ident_type(action, ident_type, view_conf={}):
+    config = collections.ChainMap(view_conf, settings.THROTTLING_SCOPES)
+
+    try:
+        action_config = config[action]
+    except KeyError:
+        action_config = config.get("*", {})
+
+    try:
+        return action_config[ident_type]
+    except KeyError:
+        return
+
+
+def get_status(ident, now):
+    data = []
+    throttle = FunkwhaleThrottle()
+    for key in sorted(settings.THROTTLING_RATES.keys()):
+        conf = settings.THROTTLING_RATES[key]
+        row_data = {"id": key, "rate": conf["rate"], "description": conf["description"]}
+        if conf["rate"]:
+            num_requests, duration = throttle.parse_rate(conf["rate"])
+            history = cache.get(get_cache_key(key, ident)) or []
+
+            relevant_history = [h for h in history if h > now - duration]
+            row_data["limit"] = num_requests
+            row_data["duration"] = duration
+            row_data["remaining"] = num_requests - len(relevant_history)
+            if relevant_history and len(relevant_history) >= num_requests:
+                # At this point, the endpoint becomes available again
+                now_request = relevant_history[-1]
+                remaining = duration - (now - int(now_request))
+                row_data["available"] = int(now + remaining) or None
+                row_data["available_seconds"] = int(remaining) or None
+            else:
+                row_data["available"] = None
+                row_data["available_seconds"] = None
+
+            if relevant_history:
+                # At this point, all Rate Limit is reset to 0
+                latest_request = relevant_history[0]
+                remaining = duration - (now - int(latest_request))
+                row_data["reset"] = int(now + remaining)
+                row_data["reset_seconds"] = int(remaining)
+            else:
+                row_data["reset"] = None
+                row_data["reset_seconds"] = None
+        else:
+            row_data["limit"] = None
+            row_data["duration"] = None
+            row_data["remaining"] = None
+            row_data["available"] = None
+            row_data["available_seconds"] = None
+            row_data["reset"] = None
+            row_data["reset_seconds"] = None
+
+        data.append(row_data)
+
+    return data
+
+
+class FunkwhaleThrottle(rest_throttling.SimpleRateThrottle):
+    def __init__(self):
+        pass
+
+    def get_cache_key(self, request, view):
+        return get_cache_key(self.scope, self.ident)
+
+    def allow_request(self, request, view):
+        self.request = request
+        self.ident = get_ident(request)
+        action = getattr(view, "action", "*")
+        view_scopes = getattr(view, "throttling_scopes", {})
+        if view_scopes is None:
+            return True
+        self.scope = get_scope_for_action_and_ident_type(
+            action=action, ident_type=self.ident["type"], view_conf=view_scopes
+        )
+        if not self.scope or self.scope not in settings.THROTTLING_RATES:
+            return True
+        self.rate = settings.THROTTLING_RATES[self.scope].get("rate")
+        self.num_requests, self.duration = self.parse_rate(self.rate)
+        self.request = request
+
+        return super().allow_request(request, view)
+
+    def attach_info(self):
+        info = {
+            "num_requests": self.num_requests,
+            "duration": self.duration,
+            "scope": self.scope,
+            "history": self.history or [],
+            "wait": self.wait(),
+        }
+        setattr(self.request, "_throttle_status", info)
+
+    def throttle_success(self):
+        self.attach_info()
+        return super().throttle_success()
+
+    def throttle_failure(self):
+        self.attach_info()
+        return super().throttle_failure()
+
+
+class TooManyRequests(Exception):
+    pass
+
+
+DummyView = collections.namedtuple("DummyView", "action throttling_scopes")
+
+
+def check_request(request, scope):
+    """
+    A simple wrapper around FunkwhaleThrottle for views that aren't API views
+    or cannot use rest_framework automatic throttling.
+
+    Raise TooManyRequests if limit is reached.
+    """
+    if not settings.THROTTLING_ENABLED:
+        return True
+
+    view = DummyView(
+        action=scope,
+        throttling_scopes={scope: {"anonymous": scope, "authenticated": scope}},
+    )
+    throttle = FunkwhaleThrottle()
+    if not throttle.allow_request(request, view):
+        raise TooManyRequests()
+    return True