Commit 853cd833 authored by Eliot Berriot's avatar Eliot Berriot
Browse files

Merge branch 'rate-limiting' into 'develop'

: Rate limiting

Closes #261

See merge request funkwhale/funkwhale!877
parents 8666afc6 d28bf65d
......@@ -2,7 +2,6 @@ from django.conf.urls import include, url
from dynamic_preferences.api.viewsets import GlobalPreferencesViewSet
from rest_framework import routers
from rest_framework.urlpatterns import format_suffix_patterns
from rest_framework_jwt import views as jwt_views
from funkwhale_api.activity import views as activity_views
from funkwhale_api.common import views as common_views
......@@ -11,6 +10,7 @@ from funkwhale_api.music import views
from funkwhale_api.playlists import views as playlists_views
from funkwhale_api.subsonic.views import SubsonicViewSet
from funkwhale_api.tags import views as tags_views
from funkwhale_api.users import jwt_views
router = common_routers.OptionalSlashRouter()
router.register(r"settings", GlobalPreferencesViewSet, basename="settings")
......@@ -83,6 +83,7 @@ v1_patterns += [
),
url(r"^token/?$", jwt_views.obtain_jwt_token, name="token"),
url(r"^token/refresh/?$", jwt_views.refresh_jwt_token, name="token_refresh"),
url(r"^rate-limit/?$", common_views.RateLimitView.as_view(), name="rate-limit"),
]
urlpatterns = [
......
......@@ -232,6 +232,7 @@ MIDDLEWARE = (
"django.contrib.auth.middleware.AuthenticationMiddleware",
"django.contrib.messages.middleware.MessageMiddleware",
"funkwhale_api.users.middleware.RecordActivityMiddleware",
"funkwhale_api.common.middleware.ThrottleStatusMiddleware",
)
# DEBUG
......@@ -615,7 +616,150 @@ REST_FRAMEWORK = {
"django_filters.rest_framework.DjangoFilterBackend",
),
"DEFAULT_RENDERER_CLASSES": ("rest_framework.renderers.JSONRenderer",),
"NUM_PROXIES": env.int("NUM_PROXIES", default=1),
}
THROTTLING_ENABLED = env.bool("THROTTLING_ENABLED", default=True)
if THROTTLING_ENABLED:
REST_FRAMEWORK["DEFAULT_THROTTLE_CLASSES"] = env.list(
"THROTTLE_CLASSES",
default=["funkwhale_api.common.throttling.FunkwhaleThrottle"],
)
THROTTLING_SCOPES = {
"*": {"anonymous": "anonymous-wildcard", "authenticated": "authenticated-wildcard"},
"create": {
"authenticated": "authenticated-create",
"anonymous": "anonymous-create",
},
"list": {"authenticated": "authenticated-list", "anonymous": "anonymous-list"},
"retrieve": {
"authenticated": "authenticated-retrieve",
"anonymous": "anonymous-retrieve",
},
"destroy": {
"authenticated": "authenticated-destroy",
"anonymous": "anonymous-destroy",
},
"update": {
"authenticated": "authenticated-update",
"anonymous": "anonymous-update",
},
"partial_update": {
"authenticated": "authenticated-update",
"anonymous": "anonymous-update",
},
}
THROTTLING_USER_RATES = env.dict("THROTTLING_RATES", default={})
THROTTLING_RATES = {
"anonymous-wildcard": {
"rate": THROTTLING_USER_RATES.get("anonymous-wildcard", "1000/h"),
"description": "Anonymous requests not covered by other limits",
},
"authenticated-wildcard": {
"rate": THROTTLING_USER_RATES.get("authenticated-wildcard", "2000/h"),
"description": "Authenticated requests not covered by other limits",
},
"authenticated-create": {
"rate": THROTTLING_USER_RATES.get("authenticated-create", "1000/hour"),
"description": "Authenticated POST requests",
},
"anonymous-create": {
"rate": THROTTLING_USER_RATES.get("anonymous-create", "1000/day"),
"description": "Anonymous POST requests",
},
"authenticated-list": {
"rate": THROTTLING_USER_RATES.get("authenticated-list", "10000/hour"),
"description": "Authenticated GET requests on resource lists",
},
"anonymous-list": {
"rate": THROTTLING_USER_RATES.get("anonymous-list", "10000/day"),
"description": "Anonymous GET requests on resource lists",
},
"authenticated-retrieve": {
"rate": THROTTLING_USER_RATES.get("authenticated-retrieve", "10000/hour"),
"description": "Authenticated GET requests on resource detail",
},
"anonymous-retrieve": {
"rate": THROTTLING_USER_RATES.get("anonymous-retrieve", "10000/day"),
"description": "Anonymous GET requests on resource detail",
},
"authenticated-destroy": {
"rate": THROTTLING_USER_RATES.get("authenticated-destroy", "500/hour"),
"description": "Authenticated DELETE requests on resource detail",
},
"anonymous-destroy": {
"rate": THROTTLING_USER_RATES.get("anonymous-destroy", "1000/day"),
"description": "Anonymous DELETE requests on resource detail",
},
"authenticated-update": {
"rate": THROTTLING_USER_RATES.get("authenticated-update", "1000/hour"),
"description": "Authenticated PATCH and PUT requests on resource detail",
},
"anonymous-update": {
"rate": THROTTLING_USER_RATES.get("anonymous-update", "1000/day"),
"description": "Anonymous PATCH and PUT requests on resource detail",
},
# potentially spammy / dangerous endpoints
"authenticated-reports": {
"rate": THROTTLING_USER_RATES.get("authenticated-reports", "100/day"),
"description": "Authenticated report submission",
},
"anonymous-reports": {
"rate": THROTTLING_USER_RATES.get("anonymous-reports", "10/day"),
"description": "Anonymous report submission",
},
"authenticated-oauth-app": {
"rate": THROTTLING_USER_RATES.get("authenticated-oauth-app", "10/hour"),
"description": "Authenticated OAuth app creation",
},
"anonymous-oauth-app": {
"rate": THROTTLING_USER_RATES.get("anonymous-oauth-app", "10/day"),
"description": "Anonymous OAuth app creation",
},
"oauth-authorize": {
"rate": THROTTLING_USER_RATES.get("oauth-authorize", "100/hour"),
"description": "OAuth app authorization",
},
"oauth-token": {
"rate": THROTTLING_USER_RATES.get("oauth-token", "100/hour"),
"description": "OAuth token creation",
},
"oauth-revoke-token": {
"rate": THROTTLING_USER_RATES.get("oauth-revoke-token", "100/hour"),
"description": "OAuth token deletion",
},
"jwt-login": {
"rate": THROTTLING_USER_RATES.get("jwt-login", "30/hour"),
"description": "JWT token creation",
},
"jwt-refresh": {
"rate": THROTTLING_USER_RATES.get("jwt-refresh", "30/hour"),
"description": "JWT token refresh",
},
"signup": {
"rate": THROTTLING_USER_RATES.get("signup", "10/day"),
"description": "Account creation",
},
"verify-email": {
"rate": THROTTLING_USER_RATES.get("verify-email", "20/h"),
"description": "Email address confirmation",
},
"password-change": {
"rate": THROTTLING_USER_RATES.get("password-change", "20/h"),
"description": "Password change (when authenticated)",
},
"password-reset": {
"rate": THROTTLING_USER_RATES.get("password-reset", "20/h"),
"description": "Password reset request",
},
"password-reset-confirm": {
"rate": THROTTLING_USER_RATES.get("password-reset-confirm", "20/h"),
"description": "Password reset confirmation",
},
}
BROWSABLE_API_ENABLED = env.bool("BROWSABLE_API_ENABLED", default=False)
if BROWSABLE_API_ENABLED:
......
......@@ -19,8 +19,7 @@ urlpatterns = [
("funkwhale_api.federation.urls", "federation"), namespace="federation"
),
),
url(r"^api/v1/auth/", include("rest_auth.urls")),
url(r"^api/v1/auth/registration/", include("funkwhale_api.users.rest_auth_urls")),
url(r"^api/v1/auth/", include("funkwhale_api.users.rest_auth_urls")),
url(r"^accounts/", include("allauth.urls")),
# Your stuff: custom urls includes go here
]
......
import html
import requests
import time
import xml.sax.saxutils
from django import http
from django.conf import settings
from django.core.cache import caches
from django import urls
from rest_framework import views
from . import preferences
from . import throttling
from . import utils
EXCLUDED_PATHS = ["/api", "/federation", "/.well-known"]
......@@ -176,3 +179,66 @@ class DevHttpsMiddleware:
lambda: request.__class__.get_host(request).replace(":80", ":443"),
)
return self.get_response(request)
def monkey_patch_rest_initialize_request():
"""
Rest framework use it's own APIRequest, meaning we can't easily
access our throttling info in the middleware. So me monkey patch the
`initialize_request` method from rest_framework to keep a link between both requests
"""
original = views.APIView.initialize_request
def replacement(self, request, *args, **kwargs):
r = original(self, request, *args, **kwargs)
setattr(request, "_api_request", r)
return r
setattr(views.APIView, "initialize_request", replacement)
monkey_patch_rest_initialize_request()
class ThrottleStatusMiddleware:
"""
Include useful information regarding throttling in API responses to
ensure clients can adapt.
"""
def __init__(self, get_response):
self.get_response = get_response
def __call__(self, request):
try:
response = self.get_response(request)
except throttling.TooManyRequests:
# manual throttling in non rest_framework view, we have to return
# the proper response ourselves
response = http.HttpResponse(status=429)
request_to_check = request
try:
request_to_check = request._api_request
except AttributeError:
pass
throttle_status = getattr(request_to_check, "_throttle_status", None)
if throttle_status:
response["X-RateLimit-Limit"] = str(throttle_status["num_requests"])
response["X-RateLimit-Scope"] = str(throttle_status["scope"])
response["X-RateLimit-Remaining"] = throttle_status["num_requests"] - len(
throttle_status["history"]
)
response["X-RateLimit-Duration"] = str(throttle_status["duration"])
if throttle_status["history"]:
now = int(time.time())
# At this point, the client can send additional requests
oldtest_request = throttle_status["history"][-1]
remaining = throttle_status["duration"] - (now - int(oldtest_request))
response["Retry-After"] = str(remaining)
# At this point, all Rate Limit is reset to 0
latest_request = throttle_status["history"][0]
remaining = throttle_status["duration"] - (now - int(latest_request))
response["X-RateLimit-Reset"] = str(now + remaining)
response["X-RateLimit-ResetSeconds"] = str(remaining)
return response
import collections
from django.core.cache import cache
from rest_framework import throttling as rest_throttling
from django.conf import settings
def get_ident(request):
if hasattr(request, "user") and request.user.is_authenticated:
return {"type": "authenticated", "id": request.user.pk}
ident = rest_throttling.BaseThrottle().get_ident(request)
return {"type": "anonymous", "id": ident}
def get_cache_key(scope, ident):
parts = ["throttling", scope, ident["type"], str(ident["id"])]
return ":".join(parts)
def get_scope_for_action_and_ident_type(action, ident_type, view_conf={}):
config = collections.ChainMap(view_conf, settings.THROTTLING_SCOPES)
try:
action_config = config[action]
except KeyError:
action_config = config.get("*", {})
try:
return action_config[ident_type]
except KeyError:
return
def get_status(ident, now):
data = []
throttle = FunkwhaleThrottle()
for key in sorted(settings.THROTTLING_RATES.keys()):
conf = settings.THROTTLING_RATES[key]
row_data = {"id": key, "rate": conf["rate"], "description": conf["description"]}
if conf["rate"]:
num_requests, duration = throttle.parse_rate(conf["rate"])
history = cache.get(get_cache_key(key, ident)) or []
relevant_history = [h for h in history if h > now - duration]
row_data["limit"] = num_requests
row_data["duration"] = duration
row_data["remaining"] = num_requests - len(relevant_history)
if relevant_history and len(relevant_history) >= num_requests:
# At this point, the endpoint becomes available again
now_request = relevant_history[-1]
remaining = duration - (now - int(now_request))
row_data["available"] = int(now + remaining) or None
row_data["available_seconds"] = int(remaining) or None
else:
row_data["available"] = None
row_data["available_seconds"] = None
if relevant_history:
# At this point, all Rate Limit is reset to 0
latest_request = relevant_history[0]
remaining = duration - (now - int(latest_request))
row_data["reset"] = int(now + remaining)
row_data["reset_seconds"] = int(remaining)
else:
row_data["reset"] = None
row_data["reset_seconds"] = None
else:
row_data["limit"] = None
row_data["duration"] = None
row_data["remaining"] = None
row_data["available"] = None
row_data["available_seconds"] = None
row_data["reset"] = None
row_data["reset_seconds"] = None
data.append(row_data)
return data
class FunkwhaleThrottle(rest_throttling.SimpleRateThrottle):
def __init__(self):
pass
def get_cache_key(self, request, view):
return get_cache_key(self.scope, self.ident)
def allow_request(self, request, view):
self.request = request
self.ident = get_ident(request)
action = getattr(view, "action", "*")
view_scopes = getattr(view, "throttling_scopes", {})
if view_scopes is None:
return True
self.scope = get_scope_for_action_and_ident_type(
action=action, ident_type=self.ident["type"], view_conf=view_scopes
)
if not self.scope or self.scope not in settings.THROTTLING_RATES:
return True
self.rate = settings.THROTTLING_RATES[self.scope].get("rate")
self.num_requests, self.duration = self.parse_rate(self.rate)
self.request = request
return super().allow_request(request, view)
def attach_info(self):
info = {
"num_requests": self.num_requests,
"duration": self.duration,
"scope": self.scope,
"history": self.history or [],
"wait": self.wait(),
}
setattr(self.request, "_throttle_status", info)
def throttle_success(self):
self.attach_info()
return super().throttle_success()
def throttle_failure(self):
self.attach_info()
return super().throttle_failure()
class TooManyRequests(Exception):
pass
DummyView = collections.namedtuple("DummyView", "action throttling_scopes")
def check_request(request, scope):
"""
A simple wrapper around FunkwhaleThrottle for views that aren't API views
or cannot use rest_framework automatic throttling.
Raise TooManyRequests if limit is reached.
"""
if not settings.THROTTLING_ENABLED:
return True
view = DummyView(
action=scope,
throttling_scopes={scope: {"anonymous": scope, "authenticated": scope}},
)
throttle = FunkwhaleThrottle()
if not throttle.allow_request(request, view):
raise TooManyRequests()
return True
import time
from django.conf import settings
from django.db import transaction
from rest_framework.decorators import action
......@@ -5,6 +8,7 @@ from rest_framework import exceptions
from rest_framework import mixins
from rest_framework import permissions
from rest_framework import response
from rest_framework import views
from rest_framework import viewsets
from . import filters
......@@ -13,6 +17,7 @@ from . import mutations
from . import serializers
from . import signals
from . import tasks
from . import throttling
from . import utils
......@@ -121,3 +126,17 @@ class MutationViewSet(
new_is_approved=instance.is_approved,
)
return response.Response({}, status=200)
class RateLimitView(views.APIView):
permission_classes = []
throttle_classes = []
def get(self, request, *args, **kwargs):
ident = throttling.get_ident(request)
data = {
"enabled": settings.THROTTLING_ENABLED,
"ident": ident,
"scopes": throttling.get_status(ident, time.time()),
}
return response.Response(data, status=200)
......@@ -49,6 +49,12 @@ class ReportsViewSet(mixins.CreateModelMixin, viewsets.GenericViewSet):
ordering_fields = ("creation_date",)
anonymous_policy = "setting"
anonymous_scopes = {"write:reports"}
throttling_scopes = {
"create": {
"anonymous": "anonymous-reports",
"authenticated": "authenticated-reports",
}
}
def get_serializer_context(self):
context = super().get_serializer_context()
......
from rest_framework_jwt import views as jwt_views
class ObtainJSONWebToken(jwt_views.ObtainJSONWebToken):
throttling_scopes = {"*": {"anonymous": "jwt-login", "authenticated": "jwt-login"}}
class RefreshJSONWebToken(jwt_views.RefreshJSONWebToken):
throttling_scopes = {
"*": {"anonymous": "jwt-refresh", "authenticated": "jwt-refresh"}
}
obtain_jwt_token = ObtainJSONWebToken.as_view()
refresh_jwt_token = RefreshJSONWebToken.as_view()
......@@ -10,6 +10,8 @@ from oauth2_provider import exceptions as oauth2_exceptions
from oauth2_provider import views as oauth_views
from oauth2_provider.settings import oauth2_settings
from funkwhale_api.common import throttling
from .. import models
from .permissions import ScopePermission
from . import serializers
......@@ -35,6 +37,12 @@ class ApplicationViewSet(
lookup_field = "client_id"
queryset = models.Application.objects.all().order_by("-created")
serializer_class = serializers.ApplicationSerializer
throttling_scopes = {
"create": {
"anonymous": "anonymous-oauth-app",
"authenticated": "authenticated-oauth-app",
}
}
def get_serializer_class(self):
if self.request.method.lower() == "post":
......@@ -141,6 +149,10 @@ class AuthorizeView(views.APIView, oauth_views.AuthorizationView):
return self.json_payload(errors, status_code=400)
def post(self, request, *args, **kwargs):
throttling.check_request(request, "oauth-authorize")
return super().post(request, *args, **kwargs)
def form_valid(self, form):
try:
response = super().form_valid(form)
......@@ -175,8 +187,12 @@ class AuthorizeView(views.APIView, oauth_views.AuthorizationView):
class TokenView(oauth_views.TokenView):
pass
def post(self, request, *args, **kwargs):
throttling.check_request(request, "oauth-token")
return super().post(request, *args, **kwargs)
class RevokeTokenView(oauth_views.RevokeTokenView):
pass
def post(self, request, *args, **kwargs):
throttling.check_request(request, "oauth-revoke-token")
return super().post(request, *args, **kwargs)
from django.conf.urls import url
from django.views.generic import TemplateView
from rest_auth import views as rest_auth_views
from rest_auth.registration import views as registration_views
from . import views
urlpatterns = [
url(r"^$", views.RegisterView.as_view(), name="rest_register"),
# URLs that do not require a session or valid token
url(
r"^verify-email/?$",
registration_views.VerifyEmailView.as_view(),
r"^password/reset/$",
views.PasswordResetView.as_view(),
name="rest_password_reset",
),
url(
r"^password/reset/confirm/$",
views.PasswordResetConfirmView.as_view(),
name="rest_password_reset_confirm",
),
# URLs that require a user to be logged in with a valid session / token.
url(
r"^user/$", rest_auth_views.UserDetailsView.as_view(), name="rest_user_details"
),
url(
r"^password/change/$",
views.PasswordChangeView.as_view(),
name="rest_password_change",
),
# Registration URLs
url(r"^registration/$", views.RegisterView.as_view(), name="rest_register"),
url(
r"^registration/verify-email/?$",
views.VerifyEmailView.as_view(),
name="rest_verify_email",
),
url(
r"^change-password/?$",
rest_auth_views.PasswordChangeView.as_view(),
r"^registration/change-password/?$",
views.PasswordChangeView.as_view(),
name="change_password",
),
# This url is used by django-allauth and empty TemplateView is
......@@ -28,7 +48,7 @@ urlpatterns = [
# view from:
# djang-allauth https://github.com/pennersr/django-allauth/blob/master/allauth/account/views.py#L190
url(
r"^account-confirm-email/(?P<key>\w+)/?$",
r"^registration/account-confirm-email/(?P<key>\w+)/?$",
TemplateView.as_view(),
name="account_confirm_email",