Commit 62f401ed authored by Eliot Berriot's avatar Eliot Berriot 💬

Merge branch 'develop'

parents 0488eddf 9e743392
......@@ -12,6 +12,7 @@ MUSIC_DIRECTORY_PATH=/music
BROWSABLE_API_ENABLED=True
FORWARDED_PROTO=http
LDAP_ENABLED=False
FUNKWHALE_SPA_HTML_ROOT=http://nginx/front/
# Uncomment this if you're using traefik/https
# FORCE_HTTPS_URLS=True
......
......@@ -114,7 +114,7 @@ black:
before_script:
- pip install black
script:
- black --exclude "/(\.git|\.hg|\.mypy_cache|\.tox|\.venv|_build|buck-out|build|dist|migrations)/" --check --diff api/
- black --check --diff api/
flake8:
image: python:3.6
......@@ -281,6 +281,7 @@ build_api:
paths:
- api
script:
- rm -rf api/tests
- (if [ "$CI_COMMIT_REF_NAME" == "develop" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
- chmod -R 750 api
- echo Done!
......
<!--
Hi there! You are reporting a bug on this project, and we want to thank you!
If it's the first time you post here, please take a moment to read our Code of Conduct
(https://funkwhale.audio/code-of-conduct/) and ensure your issue respect our guidelines.
To ensure your bug report is as useful as possible, please try to stick
to the following structure. You can leave the parts text between `<!- ->`
markers untouched, they won't be displayed in your final message.
......
<!--
Hi there! You are about to share feature request or an idea, and we want to thank you!
If it's the first time you post here, please take a moment to read our Code of Conduct
(https://funkwhale.audio/code-of-conduct/) and ensure your issue respect our guidelines.
To ensure we can deal with your idea or request, please try to stick
to the following structure. You can leave the parts text between `<!- ->`
markers untouched, they won't be displayed in your final message.
......
......@@ -172,6 +172,10 @@ and metadata.
Launch all services
^^^^^^^^^^^^^^^^^^^
Before the first Funkwhale launch, it is required to run this::
docker-compose -f dev.yml run --rm front yarn run i18n-compile
Then you can run everything with::
docker-compose -f dev.yml up front api nginx celeryworker
......@@ -276,7 +280,8 @@ When working on federation with traefik, ensure you have this in your ``env``::
EXTERNAL_REQUESTS_VERIFY_SSL=false
# this ensure you don't have incorrect urls pointing to http resources
FUNKWHALE_PROTOCOL=https
# Disable host ports binding for the nginx container, as traefik is serving everything
NGINX_PORTS_MAPPING=80
Typical workflow for a contribution
-----------------------------------
......@@ -513,13 +518,15 @@ It's possible to nest multiple component parts to reach a higher level of detail
- ``Content/*/Form.Help text``
Collecting translatable strings
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
If you want to ensure your translatable strings are correctly marked for translation,
you can try to extract them.
Extraction is done by calling ``yarn run i18n-extract``, which
will pull all the strings from source files and put them in a PO file.
will pull all the strings from source files and put them in a PO files.
You can then inspect the PO files to ensure everything is fine (but don't commit them, it's not needed).
Contributing to the API
-----------------------
......
......@@ -31,4 +31,9 @@ are outlined in `CONTRIBUTING <CONTRIBUTING.rst>`_.
Translate
^^^^^^^^^
Translators willing to help can refer to `TRANSLATORS <TRANSLATORS>`_ for instructions.
Translators willing to help can refer to `TRANSLATORS <TRANSLATORS.rst>`_ for instructions.
Code of Conduct
---------------
`Our Code of Conduct <https://funkwhale.audio/code-of-conduct/>`_ applies to all the community spaces, including our GitLab instance. Please, take a moment to read it.
......@@ -22,5 +22,6 @@ fi
if [ -d "frontend" ]; then
mkdir -p /frontend
cp -r frontend/* /frontend/
export FUNKWHALE_SPA_HTML_ROOT=/frontend/index.html
fi
exec "$@"
......@@ -5,6 +5,7 @@ from rest_framework.urlpatterns import format_suffix_patterns
from rest_framework_jwt import views as jwt_views
from funkwhale_api.activity import views as activity_views
from funkwhale_api.common import views as common_views
from funkwhale_api.music import views
from funkwhale_api.playlists import views as playlists_views
from funkwhale_api.subsonic.views import SubsonicViewSet
......@@ -24,6 +25,7 @@ router.register(r"playlists", playlists_views.PlaylistViewSet, "playlists")
router.register(
r"playlist-tracks", playlists_views.PlaylistTrackViewSet, "playlist-tracks"
)
router.register(r"mutations", common_views.MutationViewSet, "mutations")
v1_patterns = router.urls
subsonic_router = routers.SimpleRouter(trailing_slash=False)
......@@ -40,6 +42,12 @@ v1_patterns += [
r"^manage/",
include(("funkwhale_api.manage.urls", "manage"), namespace="manage"),
),
url(
r"^moderation/",
include(
("funkwhale_api.moderation.urls", "moderation"), namespace="moderation"
),
),
url(
r"^federation/",
include(
......@@ -67,6 +75,10 @@ v1_patterns += [
r"^users/",
include(("funkwhale_api.users.api_urls", "users"), namespace="users"),
),
url(
r"^oauth/",
include(("funkwhale_api.users.oauth.urls", "oauth"), namespace="oauth"),
),
url(r"^token/$", jwt_views.obtain_jwt_token, name="token"),
url(r"^token/refresh/$", jwt_views.refresh_jwt_token, name="token_refresh"),
]
......
import os
import django
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.production")
import django # noqa
django.setup()
from .routing import application # noqa
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.production")
......@@ -29,7 +29,6 @@ env_file = env("ENV_FILE", default=None)
if env_file:
# we have an explicitely specified env file
# so we try to load and it fail loudly if it does not exist
print("ENV_FILE", env_file)
env.read_env(env_file)
else:
# we try to load from .env and config/.env
......@@ -79,7 +78,7 @@ FUNKWHALE_SPA_HTML_CACHE_DURATION = env.int(
"FUNKWHALE_SPA_HTML_CACHE_DURATION", default=60 * 15
)
FUNKWHALE_EMBED_URL = env(
"FUNKWHALE_EMBED_URL", default=FUNKWHALE_SPA_HTML_ROOT + "embed.html"
"FUNKWHALE_EMBED_URL", default=FUNKWHALE_URL + "/front/embed.html"
)
APP_NAME = "Funkwhale"
......@@ -94,6 +93,9 @@ FEDERATION_MUSIC_NEEDS_APPROVAL = env.bool(
)
# XXX: deprecated, see #186
FEDERATION_ACTOR_FETCH_DELAY = env.int("FEDERATION_ACTOR_FETCH_DELAY", default=60 * 12)
FEDERATION_SERVICE_ACTOR_USERNAME = env(
"FEDERATION_SERVICE_ACTOR_USERNAME", default="service"
)
ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=[]) + [FUNKWHALE_HOSTNAME]
# APP CONFIGURATION
......@@ -119,6 +121,7 @@ THIRD_PARTY_APPS = (
"allauth.account", # registration
"allauth.socialaccount", # registration
"corsheaders",
"oauth2_provider",
"rest_framework",
"rest_framework.authtoken",
"taggit",
......@@ -147,9 +150,10 @@ if RAVEN_ENABLED:
# Apps specific for this project go here.
LOCAL_APPS = (
"funkwhale_api.common",
"funkwhale_api.common.apps.CommonConfig",
"funkwhale_api.activity.apps.ActivityConfig",
"funkwhale_api.users", # custom users app
"funkwhale_api.users.oauth",
# Your stuff: custom apps go here
"funkwhale_api.instance",
"funkwhale_api.music",
......@@ -181,10 +185,6 @@ MIDDLEWARE = (
"funkwhale_api.users.middleware.RecordActivityMiddleware",
)
# MIGRATIONS CONFIGURATION
# ------------------------------------------------------------------------------
MIGRATION_MODULES = {"sites": "funkwhale_api.contrib.sites.migrations"}
# DEBUG
# ------------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#debug
......@@ -220,6 +220,16 @@ DATABASES = {
"default": env.db("DATABASE_URL")
}
DATABASES["default"]["ATOMIC_REQUESTS"] = True
DATABASES["default"]["CONN_MAX_AGE"] = env("DB_CONN_MAX_AGE", default=60 * 60)
MIGRATION_MODULES = {
# see https://github.com/jazzband/django-oauth-toolkit/issues/634
# swappable models are badly designed in oauth2_provider
# ignore migrations and provide our own models.
"oauth2_provider": None,
"sites": "funkwhale_api.contrib.sites.migrations",
}
#
# DATABASES = {
# 'default': {
......@@ -296,6 +306,25 @@ STATIC_ROOT = env("STATIC_ROOT", default=str(ROOT_DIR("staticfiles")))
STATIC_URL = env("STATIC_URL", default="/staticfiles/")
DEFAULT_FILE_STORAGE = "funkwhale_api.common.storage.ASCIIFileSystemStorage"
PROXY_MEDIA = env.bool("PROXY_MEDIA", default=True)
AWS_DEFAULT_ACL = None
AWS_QUERYSTRING_AUTH = env.bool("AWS_QUERYSTRING_AUTH", default=not PROXY_MEDIA)
AWS_S3_MAX_MEMORY_SIZE = env.int(
"AWS_S3_MAX_MEMORY_SIZE", default=1000 * 1000 * 1000 * 20
)
AWS_QUERYSTRING_EXPIRE = env.int("AWS_QUERYSTRING_EXPIRE", default=3600)
AWS_ACCESS_KEY_ID = env("AWS_ACCESS_KEY_ID", default=None)
if AWS_ACCESS_KEY_ID:
AWS_ACCESS_KEY_ID = AWS_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY = env("AWS_SECRET_ACCESS_KEY")
AWS_STORAGE_BUCKET_NAME = env("AWS_STORAGE_BUCKET_NAME")
AWS_S3_ENDPOINT_URL = env("AWS_S3_ENDPOINT_URL", default=None)
AWS_S3_REGION_NAME = env("AWS_S3_REGION_NAME", default=None)
AWS_S3_SIGNATURE_VERSION = "s3v4"
AWS_LOCATION = env("AWS_LOCATION", default="")
DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
# See: https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#std:setting-STATICFILES_DIRS
STATICFILES_DIRS = (str(APPS_DIR.path("static")),)
......@@ -341,6 +370,23 @@ AUTH_USER_MODEL = "users.User"
LOGIN_REDIRECT_URL = "users:redirect"
LOGIN_URL = "account_login"
# OAuth configuration
from funkwhale_api.users.oauth import scopes # noqa
OAUTH2_PROVIDER = {
"SCOPES": {s.id: s.label for s in scopes.SCOPES_BY_ID.values()},
"ALLOWED_REDIRECT_URI_SCHEMES": ["http", "https", "urn"],
# we keep expired tokens for 15 days, for tracability
"REFRESH_TOKEN_EXPIRE_SECONDS": 3600 * 24 * 15,
"AUTHORIZATION_CODE_EXPIRE_SECONDS": 5 * 60,
"ACCESS_TOKEN_EXPIRE_SECONDS": 60 * 60 * 10,
"OAUTH2_SERVER_CLASS": "funkwhale_api.users.oauth.server.OAuth2Server",
}
OAUTH2_PROVIDER_APPLICATION_MODEL = "users.Application"
OAUTH2_PROVIDER_ACCESS_TOKEN_MODEL = "users.AccessToken"
OAUTH2_PROVIDER_GRANT_MODEL = "users.Grant"
OAUTH2_PROVIDER_REFRESH_TOKEN_MODEL = "users.RefreshToken"
# LDAP AUTHENTICATION CONFIGURATION
# ------------------------------------------------------------------------------
AUTH_LDAP_ENABLED = env.bool("LDAP_ENABLED", default=False)
......@@ -448,16 +494,28 @@ CELERY_TASK_TIME_LIMIT = 300
CELERY_BEAT_SCHEDULE = {
"federation.clean_music_cache": {
"task": "federation.clean_music_cache",
"schedule": crontab(hour="*/2"),
"schedule": crontab(minute="0", hour="*/2"),
"options": {"expires": 60 * 2},
},
"music.clean_transcoding_cache": {
"task": "music.clean_transcoding_cache",
"schedule": crontab(hour="*"),
"schedule": crontab(minute="0", hour="*"),
"options": {"expires": 60 * 2},
},
"oauth.clear_expired_tokens": {
"task": "oauth.clear_expired_tokens",
"schedule": crontab(minute="0", hour="0"),
"options": {"expires": 60 * 60 * 24},
},
"federation.refresh_nodeinfo_known_nodes": {
"task": "federation.refresh_nodeinfo_known_nodes",
"schedule": crontab(minute="0", hour="*"),
"options": {"expires": 60 * 60},
},
}
NODEINFO_REFRESH_DELAY = env.int("NODEINFO_REFRESH_DELAY", default=3600 * 24)
JWT_AUTH = {
"JWT_ALLOW_REFRESH": True,
"JWT_EXPIRATION_DELTA": datetime.timedelta(days=7),
......@@ -475,7 +533,6 @@ CORS_ORIGIN_ALLOW_ALL = True
CORS_ALLOW_CREDENTIALS = True
REST_FRAMEWORK = {
"DEFAULT_PERMISSION_CLASSES": ("rest_framework.permissions.IsAuthenticated",),
"DEFAULT_PAGINATION_CLASS": "funkwhale_api.common.pagination.FunkwhalePagination",
"PAGE_SIZE": 25,
"DEFAULT_PARSER_CLASSES": (
......@@ -485,11 +542,15 @@ REST_FRAMEWORK = {
"funkwhale_api.federation.parsers.ActivityParser",
),
"DEFAULT_AUTHENTICATION_CLASSES": (
"oauth2_provider.contrib.rest_framework.OAuth2Authentication",
"funkwhale_api.common.authentication.JSONWebTokenAuthenticationQS",
"funkwhale_api.common.authentication.BearerTokenHeaderAuth",
"funkwhale_api.common.authentication.JSONWebTokenAuthentication",
"rest_framework.authentication.SessionAuthentication",
"rest_framework.authentication.BasicAuthentication",
"rest_framework.authentication.SessionAuthentication",
),
"DEFAULT_PERMISSION_CLASSES": (
"funkwhale_api.users.oauth.permissions.ScopePermission",
),
"DEFAULT_FILTER_BACKENDS": (
"rest_framework.filters.OrderingFilter",
......
......@@ -62,19 +62,6 @@ CELERY_TASK_ALWAYS_EAGER = False
# Your local stuff: Below this line define 3rd party library settings
LOGGING = {
"version": 1,
"handlers": {"console": {"level": "DEBUG", "class": "logging.StreamHandler"}},
"loggers": {
"django.request": {
"handlers": ["console"],
"propagate": True,
"level": "DEBUG",
},
"django_auth_ldap": {"handlers": ["console"], "level": "DEBUG"},
"": {"level": "DEBUG", "handlers": ["console"]},
},
}
CSRF_TRUSTED_ORIGINS = [o for o in ALLOWED_HOSTS]
......
# -*- coding: utf-8 -*-
__version__ = "0.18.3"
__version__ = "0.19.0-rc2"
__version_info__ = tuple(
[
int(num) if num.isdigit() else num
......
from django.contrib.admin import register as initial_register, site, ModelAdmin # noqa
from django.db.models.fields.related import RelatedField
from . import models
from . import tasks
def register(model):
"""
......@@ -17,3 +20,28 @@ def register(model):
return initial_register(model)(modeladmin)
return decorator
def apply(modeladmin, request, queryset):
queryset.update(is_approved=True)
for id in queryset.values_list("id", flat=True):
tasks.apply_mutation.delay(mutation_id=id)
apply.short_description = "Approve and apply"
@register(models.Mutation)
class MutationAdmin(ModelAdmin):
list_display = [
"uuid",
"type",
"created_by",
"creation_date",
"applied_date",
"is_approved",
"is_applied",
]
search_fields = ["created_by__preferred_username"]
list_filter = ["type", "is_approved", "is_applied"]
actions = [apply]
from django.apps import AppConfig, apps
from . import mutations
class CommonConfig(AppConfig):
name = "funkwhale_api.common"
def ready(self):
super().ready()
app_names = [app.name for app in apps.app_configs.values()]
mutations.registry.autodiscover(app_names)
from rest_framework import response
from django.db import transaction
from rest_framework import decorators
from rest_framework import exceptions
from rest_framework import response
from rest_framework import status
from . import filters
from . import models
from . import mutations as common_mutations
from . import serializers
from . import signals
from . import tasks
from . import utils
def action_route(serializer_class):
......@@ -12,3 +24,69 @@ def action_route(serializer_class):
return response.Response(result, status=200)
return action
def mutations_route(types):
"""
Given a queryset and a list of mutation types, return a view
that can be included in any viewset, and serve:
GET /{id}/mutations/ - list of mutations for the given object
POST /{id}/mutations/ - create a mutation for the given object
"""
@transaction.atomic
def mutations(self, request, *args, **kwargs):
obj = self.get_object()
if request.method == "GET":
queryset = models.Mutation.objects.get_for_target(obj).filter(
type__in=types
)
queryset = queryset.order_by("-creation_date")
filterset = filters.MutationFilter(request.GET, queryset=queryset)
page = self.paginate_queryset(filterset.qs)
if page is not None:
serializer = serializers.APIMutationSerializer(page, many=True)
return self.get_paginated_response(serializer.data)
serializer = serializers.APIMutationSerializer(queryset, many=True)
return response.Response(serializer.data)
if request.method == "POST":
if not request.user.is_authenticated:
raise exceptions.NotAuthenticated()
serializer = serializers.APIMutationSerializer(
data=request.data, context={"registry": common_mutations.registry}
)
serializer.is_valid(raise_exception=True)
if not common_mutations.registry.has_perm(
actor=request.user.actor,
type=serializer.validated_data["type"],
obj=obj,
perm="approve"
if serializer.validated_data.get("is_approved", False)
else "suggest",
):
raise exceptions.PermissionDenied()
final_payload = common_mutations.registry.get_validated_payload(
type=serializer.validated_data["type"],
payload=serializer.validated_data["payload"],
obj=obj,
)
mutation = serializer.save(
created_by=request.user.actor,
target=obj,
payload=final_payload,
is_approved=serializer.validated_data.get("is_approved", None),
)
if mutation.is_approved:
utils.on_commit(tasks.apply_mutation.delay, mutation_id=mutation.pk)
utils.on_commit(
signals.mutation_created.send, sender=None, mutation=mutation
)
return response.Response(serializer.data, status=status.HTTP_201_CREATED)
return decorators.action(
methods=["get", "post"], detail=True, required_scope="edits"
)(mutations)
import factory
from funkwhale_api.factories import registry, NoUpdateOnCreate
from funkwhale_api.federation import factories as federation_factories
@registry.register
class MutationFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
fid = factory.Faker("federation_url")
uuid = factory.Faker("uuid4")
created_by = factory.SubFactory(federation_factories.ActorFactory)
summary = factory.Faker("paragraph")
type = "update"
class Meta:
model = "common.Mutation"
@factory.post_generation