Verified Commit 338e1a85 authored by Eliot Berriot's avatar Eliot Berriot 💬

Merge branch 'release/0.18'

parents de7b1314 f44d2b06
Pipeline #3033 passed with stages
in 5 minutes and 53 seconds
......@@ -12,3 +12,9 @@ MUSIC_DIRECTORY_PATH=/music
BROWSABLE_API_ENABLED=True
FORWARDED_PROTO=http
LDAP_ENABLED=False
# Uncomment this if you're using traefik/https
# FORCE_HTTPS_URLS=True
# Customize to your needs
POSTGRES_VERSION=11
......@@ -93,4 +93,6 @@ po/*.po
docs/swagger
_build
front/src/translations.json
front/src/translations/*.json
front/locales/en_US/LC_MESSAGES/app.po
*.prof
variables:
IMAGE_NAME: funkwhale/funkwhale
IMAGE: $IMAGE_NAME:$CI_COMMIT_REF_NAME
IMAGE_LATEST: $IMAGE_NAME:latest
ALL_IN_ONE_IMAGE_NAME: funkwhale/all-in-one
ALL_IN_ONE_IMAGE: $ALL_IN_ONE_IMAGE_NAME:$CI_COMMIT_REF_NAME
PIP_CACHE_DIR: "$CI_PROJECT_DIR/pip-cache"
PYTHONDONTWRITEBYTECODE: "true"
REVIEW_DOMAIN: preview.funkwhale.audio
......@@ -131,16 +132,15 @@ flake8:
test_api:
services:
- postgres:9.4
- postgres:11
- redis:3
stage: test
image: funkwhale/funkwhale:latest
image: funkwhale/funkwhale:develop
cache:
key: "$CI_PROJECT_ID__pip_cache"
paths:
- "$PIP_CACHE_DIR"
variables:
DJANGO_ALLOWED_HOSTS: "localhost"
DATABASE_URL: "postgresql://postgres@postgres/postgres"
FUNKWHALE_URL: "https://funkwhale.ci"
DJANGO_SETTINGS_MODULE: config.settings.local
......@@ -148,11 +148,10 @@ test_api:
- branches
before_script:
- cd api
- apt-get update
- grep "^[^#;]" requirements.apt | grep -Fv "python3-dev" | xargs apt-get install -y --no-install-recommends
- pip install -r requirements/base.txt
- pip install -r requirements/local.txt
- pip install -r requirements/test.txt
- sed -i '/Pillow/d' requirements/base.txt
- pip3 install -r requirements/base.txt
- pip3 install -r requirements/local.txt
- pip3 install -r requirements/test.txt
script:
- pytest --cov=funkwhale_api tests/
tags:
......@@ -166,7 +165,7 @@ test_front:
only:
- branches
script:
- yarn install
- yarn install --check-files
- yarn test:unit
cache:
key: "funkwhale__front_dependencies"
......@@ -194,11 +193,6 @@ build_front:
# cf https://dev.funkwhale.audio/funkwhale/funkwhale/issues/169
- yarn build | tee /dev/stderr | (! grep -i 'ERROR in')
- chmod -R 755 dist
cache:
key: "funkwhale__front_dependencies"
paths:
- front/node_modules
- front/yarn.lock
artifacts:
name: "front_${CI_COMMIT_REF_NAME}"
paths:
......@@ -207,6 +201,7 @@ build_front:
- tags@funkwhale/funkwhale
- master@funkwhale/funkwhale
- develop@funkwhale/funkwhale
tags:
- docker
......@@ -236,9 +231,11 @@ pages:
docker_release:
stage: deploy
image: bash
before_script:
- docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
- cp -r front/dist api/frontend
- (if [ "$CI_COMMIT_REF_NAME" == "develop" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
- cd api
script:
- docker build -t $IMAGE .
......@@ -249,15 +246,42 @@ docker_release:
tags:
- docker-build
docker_all_in_one_release:
stage: deploy
image: bash
variables:
ALL_IN_ONE_REF: master
ALL_IN_ONE_ARTIFACT_URL: https://github.com/thetarkus/docker-funkwhale/archive/$ALL_IN_ONE_REF.zip
BUILD_PATH: all_in_one
before_script:
- docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
- (if [ "$CI_COMMIT_REF_NAME" == "develop" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
script:
- wget $ALL_IN_ONE_ARTIFACT_URL -O all_in_one.zip
- unzip -o all_in_one.zip -d tmpdir
- mv tmpdir/docker-funkwhale-$ALL_IN_ONE_REF $BUILD_PATH && rmdir tmpdir
- cp -r api $BUILD_PATH/src/api
- cp -r front $BUILD_PATH/src/front
- cd $BUILD_PATH
- ./scripts/download-nginx-template.sh src/ $CI_COMMIT_REF_NAME
- docker build -t $ALL_IN_ONE_IMAGE .
- docker push $ALL_IN_ONE_IMAGE
only:
- develop@funkwhale/funkwhale
- tags@funkwhale/funkwhale
tags:
- docker-build
build_api:
# Simply publish a zip containing api/ directory
stage: deploy
image: busybox
image: bash
artifacts:
name: "api_${CI_COMMIT_REF_NAME}"
paths:
- api
script:
- (if [ "$CI_COMMIT_REF_NAME" == "develop" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
- chmod -R 750 api
- echo Done!
only:
......
......@@ -34,6 +34,12 @@ Describe the expected behaviour.
## Context
<!--
The version of your instance can be found on the footer : Source code (x.y)
-->
**Funkwhale version(s) affected**: x.y
<!--
If relevant, share additional context here like:
......
This diff is collapsed.
......@@ -35,18 +35,20 @@ Setup front-end only development environment
cd funkwhale
cd front
2. Install [nodejs](https://nodejs.org/en/download/package-manager/) and [yarn](https://yarnpkg.com/lang/en/docs/install/#debian-stable)
2. Install `nodejs <https://nodejs.org/en/download/package-manager/>`_ and `yarn <https://yarnpkg.com/lang/en/docs/install/#debian-stable>`_
3. Install the dependencies::
yarn install
4. Launch the development server::
# this will serve the front-end on http://localhost:8000
# this will serve the front-end on http://localhost:8000/front/
VUE_PORT=8000 yarn serve
5. Make the front-end talk with an existing server (like https://demo.funkwhale.audio),
by clicking on the corresponding link in the footer
6. Start hacking!
Setup your development environment
......@@ -307,7 +309,7 @@ A typical fragment looks like that:
Fixed broken audio player on Chrome 42 for ogg files (#567)
If the work fixes one or more issues, the issue number should be included at the
end of the fragment (``(#567)`` is the issue number in the previous example.
end of the fragment (``(#567)`` is the issue number in the previous example).
If your work is not related to a specific issue, use the merge request
identifier instead, like this:
......@@ -389,7 +391,7 @@ This is regular pytest, so you can use any arguments/options that pytest usually
# Stop on first failure
docker-compose -f dev.yml run --rm api pytest -x
# Run a specific test file
docker-compose -f dev.yml run --rm api pytest tests/test_acoustid.py
docker-compose -f dev.yml run --rm api pytest tests/music/test_models.py
Writing tests
^^^^^^^^^^^^^
......@@ -507,7 +509,7 @@ useful when testing components that depend on each other:
# here, we ensure no email was sent
mocked_notify.assert_not_called()
Views: you can find some readable views tests in :file:`tests/users/test_views.py`
Views: you can find some readable views tests in file: ``api/tests/users/test_views.py``
.. note::
......
FROM python:3.6
FROM alpine:3.8
ENV PYTHONUNBUFFERED 1
RUN \
echo 'installing dependencies' && \
apk add \
bash \
git \
gettext \
musl-dev \
gcc \
postgresql-dev \
python3-dev \
py3-psycopg2 \
py3-pillow \
libldap \
ffmpeg \
libpq \
libmagic \
libffi-dev \
zlib-dev \
openldap-dev && \
\
\
ln -s /usr/bin/python3 /usr/bin/python
# Requirements have to be pulled and installed here, otherwise caching won't work
RUN echo 'deb http://httpredir.debian.org/debian/ jessie-backports main' > /etc/apt/sources.list.d/ffmpeg.list
COPY ./requirements.apt /requirements.apt
RUN apt-get update; \
grep "^[^#;]" requirements.apt | \
grep -Fv "python3-dev" | \
xargs apt-get install -y --no-install-recommends; \
rm -rf /usr/share/doc/* /usr/share/locale/*
RUN curl -L https://github.com/acoustid/chromaprint/releases/download/v1.4.2/chromaprint-fpcalc-1.4.2-linux-x86_64.tar.gz | tar -xz -C /usr/local/bin --strip 1
RUN mkdir /requirements
COPY ./requirements/base.txt /requirements/base.txt
RUN pip install -r /requirements/base.txt
COPY ./requirements/production.txt /requirements/production.txt
RUN pip install -r /requirements/production.txt
RUN \
echo 'fixing requirements file for alpine' && \
sed -i '/Pillow/d' /requirements/base.txt && \
\
\
echo 'installing pip requirements' && \
pip3 install --no-cache-dir --upgrade pip && \
pip3 install --no-cache-dir setuptools wheel && \
pip3 install --no-cache-dir -r /requirements/base.txt
COPY . /app
# Since youtube-dl code is updated fairly often, we split it here
RUN pip install --upgrade youtube-dl
WORKDIR /app
ARG install_dev_deps=0
COPY ./requirements/*.txt /requirements/
RUN \
if [ "$install_dev_deps" = "1" ] ; then echo "Installing dev dependencies" && pip3 install --no-cache-dir -r /requirements/local.txt -r /requirements/test.txt ; else echo "Skipping dev deps installation" ; fi
ENTRYPOINT ["./compose/django/entrypoint.sh"]
CMD ["./compose/django/daphne.sh"]
COPY . /app
WORKDIR /app
#!/bin/bash -eux
python /app/manage.py collectstatic --noinput
/usr/local/bin/daphne -b 0.0.0.0 -p 5000 config.asgi:application --proxy-headers
daphne -b 0.0.0.0 -p 5000 config.asgi:application --proxy-headers
#!/bin/bash
#!/bin/sh
set -e
# This entrypoint is used to play nicely with the current cookiecutter configuration.
# Since docker-compose relies heavily on environment variables itself for configuration, we'd have to define multiple
......
......@@ -10,7 +10,7 @@ from funkwhale_api.playlists import views as playlists_views
from funkwhale_api.subsonic.views import SubsonicViewSet
router = routers.SimpleRouter()
router.register(r"settings", GlobalPreferencesViewSet, base_name="settings")
router.register(r"settings", GlobalPreferencesViewSet, basename="settings")
router.register(r"activity", activity_views.ActivityViewSet, "activity")
router.register(r"tags", views.TagViewSet, "tags")
router.register(r"tracks", views.TrackViewSet, "tracks")
......@@ -19,6 +19,7 @@ router.register(r"libraries", views.LibraryViewSet, "libraries")
router.register(r"listen", views.ListenViewSet, "listen")
router.register(r"artists", views.ArtistViewSet, "artists")
router.register(r"albums", views.AlbumViewSet, "albums")
router.register(r"licenses", views.LicenseViewSet, "licenses")
router.register(r"playlists", playlists_views.PlaylistViewSet, "playlists")
router.register(
r"playlist-tracks", playlists_views.PlaylistTrackViewSet, "playlist-tracks"
......@@ -26,10 +27,11 @@ router.register(
v1_patterns = router.urls
subsonic_router = routers.SimpleRouter(trailing_slash=False)
subsonic_router.register(r"subsonic/rest", SubsonicViewSet, base_name="subsonic")
subsonic_router.register(r"subsonic/rest", SubsonicViewSet, basename="subsonic")
v1_patterns += [
url(r"^oembed/$", views.OembedView.as_view(), name="oembed"),
url(
r"^instance/",
include(("funkwhale_api.instance.urls", "instance"), namespace="instance"),
......
......@@ -13,7 +13,7 @@ from __future__ import absolute_import, unicode_literals
import datetime
import logging
from urllib.parse import urlparse, urlsplit
from urllib.parse import urlsplit
import environ
from celery.schedules import crontab
......@@ -69,12 +69,23 @@ else:
FUNKWHALE_HOSTNAME = _parsed.netloc
FUNKWHALE_PROTOCOL = _parsed.scheme
FUNKWHALE_PROTOCOL = FUNKWHALE_PROTOCOL.lower()
FUNKWHALE_HOSTNAME = FUNKWHALE_HOSTNAME.lower()
FUNKWHALE_URL = "{}://{}".format(FUNKWHALE_PROTOCOL, FUNKWHALE_HOSTNAME)
FUNKWHALE_SPA_HTML_ROOT = env(
"FUNKWHALE_SPA_HTML_ROOT", default=FUNKWHALE_URL + "/front/"
)
FUNKWHALE_SPA_HTML_CACHE_DURATION = env.int(
"FUNKWHALE_SPA_HTML_CACHE_DURATION", default=60 * 15
)
FUNKWHALE_EMBED_URL = env(
"FUNKWHALE_EMBED_URL", default=FUNKWHALE_SPA_HTML_ROOT + "embed.html"
)
APP_NAME = "Funkwhale"
# XXX: deprecated, see #186
FEDERATION_ENABLED = env.bool("FEDERATION_ENABLED", default=True)
FEDERATION_HOSTNAME = env("FEDERATION_HOSTNAME", default=FUNKWHALE_HOSTNAME)
FEDERATION_HOSTNAME = env("FEDERATION_HOSTNAME", default=FUNKWHALE_HOSTNAME).lower()
# XXX: deprecated, see #186
FEDERATION_COLLECTION_PAGE_SIZE = env.int("FEDERATION_COLLECTION_PAGE_SIZE", default=50)
# XXX: deprecated, see #186
......@@ -83,7 +94,7 @@ FEDERATION_MUSIC_NEEDS_APPROVAL = env.bool(
)
# XXX: deprecated, see #186
FEDERATION_ACTOR_FETCH_DELAY = env.int("FEDERATION_ACTOR_FETCH_DELAY", default=60 * 12)
ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS")
ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=[]) + [FUNKWHALE_HOSTNAME]
# APP CONFIGURATION
# ------------------------------------------------------------------------------
......@@ -145,10 +156,10 @@ LOCAL_APPS = (
"funkwhale_api.requests",
"funkwhale_api.favorites",
"funkwhale_api.federation",
"funkwhale_api.moderation",
"funkwhale_api.radios",
"funkwhale_api.history",
"funkwhale_api.playlists",
"funkwhale_api.providers.acoustid",
"funkwhale_api.subsonic",
)
......@@ -159,7 +170,7 @@ INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS
# MIDDLEWARE CONFIGURATION
# ------------------------------------------------------------------------------
MIDDLEWARE = (
# Make sure djangosecure.middleware.SecurityMiddleware is listed first
"funkwhale_api.common.middleware.SPAFallbackMiddleware",
"django.contrib.sessions.middleware.SessionMiddleware",
"corsheaders.middleware.CorsMiddleware",
"django.middleware.common.CommonMiddleware",
......@@ -305,8 +316,7 @@ FILE_UPLOAD_PERMISSIONS = 0o644
# URL Configuration
# ------------------------------------------------------------------------------
ROOT_URLCONF = "config.urls"
# See: https://docs.djangoproject.com/en/dev/ref/settings/#wsgi-application
WSGI_APPLICATION = "config.wsgi.application"
SPA_URLCONF = "config.spa_urls"
ASGI_APPLICATION = "config.routing.application"
# This ensures that Django will be able to detect a secure connection
......@@ -315,7 +325,7 @@ SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
# AUTHENTICATION CONFIGURATION
# ------------------------------------------------------------------------------
AUTHENTICATION_BACKENDS = (
"django.contrib.auth.backends.ModelBackend",
"funkwhale_api.users.auth_backends.ModelBackend",
"allauth.account.auth_backends.AuthenticationBackend",
)
SESSION_COOKIE_HTTPONLY = False
......@@ -400,15 +410,20 @@ if AUTH_LDAP_ENABLED:
AUTOSLUG_SLUGIFY_FUNCTION = "slugify.slugify"
CACHE_DEFAULT = "redis://127.0.0.1:6379/0"
CACHES = {"default": env.cache_url("CACHE_URL", default=CACHE_DEFAULT)}
CACHES = {
"default": env.cache_url("CACHE_URL", default=CACHE_DEFAULT),
"local": {
"BACKEND": "django.core.cache.backends.locmem.LocMemCache",
"LOCATION": "local-cache",
},
}
CACHES["default"]["BACKEND"] = "django_redis.cache.RedisCache"
cache_url = urlparse(CACHES["default"]["LOCATION"])
CHANNEL_LAYERS = {
"default": {
"BACKEND": "channels_redis.core.RedisChannelLayer",
"CONFIG": {"hosts": [(cache_url.hostname, cache_url.port)]},
"CONFIG": {"hosts": [CACHES["default"]["LOCATION"]]},
}
}
......@@ -435,7 +450,12 @@ CELERY_BEAT_SCHEDULE = {
"task": "federation.clean_music_cache",
"schedule": crontab(hour="*/2"),
"options": {"expires": 60 * 2},
}
},
"music.clean_transcoding_cache": {
"task": "music.clean_transcoding_cache",
"schedule": crontab(hour="*"),
"options": {"expires": 60 * 2},
},
}
JWT_AUTH = {
......@@ -516,6 +536,7 @@ MUSICBRAINZ_HOSTNAME = env("MUSICBRAINZ_HOSTNAME", default="musicbrainz.org")
# Custom Admin URL, use {% url 'admin:index' %}
ADMIN_URL = env("DJANGO_ADMIN_URL", default="^api/admin/")
CSRF_USE_SESSIONS = True
SESSION_ENGINE = "django.contrib.sessions.backends.cache"
# Playlist settings
# XXX: deprecated, see #186
......@@ -570,3 +591,9 @@ VERSATILEIMAGEFIELD_RENDITION_KEY_SETS = {
]
}
VERSATILEIMAGEFIELD_SETTINGS = {"create_images_on_demand": False}
RSA_KEY_SIZE = 2048
# for performance gain in tests, since we don't need to actually create the
# thumbnails
CREATE_IMAGE_THUMBNAILS = env.bool("CREATE_IMAGE_THUMBNAILS", default=True)
# we rotate actor keys at most every two days by default
ACTOR_KEY_ROTATION_DELAY = env.int("ACTOR_KEY_ROTATION_DELAY", default=3600 * 48)
......@@ -14,6 +14,7 @@ from .common import * # noqa
# DEBUG
# ------------------------------------------------------------------------------
DEBUG = env.bool("DJANGO_DEBUG", default=True)
FORCE_HTTPS_URLS = env.bool("FORCE_HTTPS_URLS", default=False)
TEMPLATES[0]["OPTIONS"]["debug"] = DEBUG
# SECRET CONFIGURATION
......@@ -31,7 +32,6 @@ EMAIL_PORT = 1025
# django-debug-toolbar
# ------------------------------------------------------------------------------
MIDDLEWARE += ("debug_toolbar.middleware.DebugToolbarMiddleware",)
# INTERNAL_IPS = ('127.0.0.1', '10.0.2.2',)
......@@ -39,20 +39,24 @@ DEBUG_TOOLBAR_CONFIG = {
"DISABLE_PANELS": ["debug_toolbar.panels.redirects.RedirectsPanel"],
"SHOW_TEMPLATE_CONTEXT": True,
"SHOW_TOOLBAR_CALLBACK": lambda request: True,
"JQUERY_URL": "",
"JQUERY_URL": "/staticfiles/admin/js/vendor/jquery/jquery.js",
}
# django-extensions
# ------------------------------------------------------------------------------
# INSTALLED_APPS += ('django_extensions', )
INSTALLED_APPS += ("debug_toolbar",)
# Debug toolbar is slow, we disable it for tests
DEBUG_TOOLBAR_ENABLED = env.bool("DEBUG_TOOLBAR_ENABLED", default=DEBUG)
if DEBUG_TOOLBAR_ENABLED:
MIDDLEWARE += ("debug_toolbar.middleware.DebugToolbarMiddleware",)
INSTALLED_APPS += ("debug_toolbar",)
# TESTING
# ------------------------------------------------------------------------------
TEST_RUNNER = "django.test.runner.DiscoverRunner"
# CELERY
# In development, all tasks will be executed locally by blocking until the task returns
CELERY_TASK_ALWAYS_EAGER = False
# END CELERY
......@@ -72,3 +76,10 @@ LOGGING = {
},
}
CSRF_TRUSTED_ORIGINS = [o for o in ALLOWED_HOSTS]
if env.bool("WEAK_PASSWORDS", default=False):
# Faster during tests
PASSWORD_HASHERS = ("django.contrib.auth.hashers.MD5PasswordHasher",)
MIDDLEWARE = ("funkwhale_api.common.middleware.DevHttpsMiddleware",) + MIDDLEWARE
from django import urls
from funkwhale_api.music import spa_views
urlpatterns = [
urls.re_path(
r"^library/tracks/(?P<pk>\d+)/?$", spa_views.library_track, name="library_track"
),
urls.re_path(
r"^library/albums/(?P<pk>\d+)/?$", spa_views.library_album, name="library_album"
),
urls.re_path(
r"^library/artists/(?P<pk>\d+)/?$",
spa_views.library_artist,
name="library_artist",
),
]
"""
WSGI config for funkwhale_api project.
This module contains the WSGI application used by Django's development server
and any production WSGI deployments. It should expose a module-level variable
named ``application``. Django's ``runserver`` and ``runfcgi`` commands discover
this application via the ``WSGI_APPLICATION`` setting.
Usually you will have the standard Django WSGI application here, but it also
might make sense to replace the whole Django WSGI application with a custom one
that later delegates to the Django one. For example, you could introduce WSGI
middleware here, or combine a Django application with an application of another
framework.
"""
import os
from django.core.wsgi import get_wsgi_application
from whitenoise.django import DjangoWhiteNoise
# We defer to a DJANGO_SETTINGS_MODULE already in the environment. This breaks
# if running multiple sites in the same mod_wsgi process. To fix this, use
# mod_wsgi daemon mode with each site in its own daemon process, or use
# os.environ["DJANGO_SETTINGS_MODULE"] = "config.settings.production"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.production")
# This application object is used by any WSGI server configured to use this
# file. This includes Django's development server, if the WSGI_APPLICATION
# setting points here.
application = get_wsgi_application()
# Use Whitenoise to serve static files
# See: https://whitenoise.readthedocs.org/
application = DjangoWhiteNoise(application)
# Apply WSGI middleware here.
# from helloworld.wsgi import HelloWorldApplication
# application = HelloWorldApplication(application)
FROM python:3.6
ENV PYTHONUNBUFFERED 1
# Requirements have to be pulled and installed here, otherwise caching won't work
RUN echo 'deb http://httpredir.debian.org/debian/ jessie-backports main' > /etc/apt/sources.list.d/ffmpeg.list
COPY ./requirements.apt /requirements.apt
RUN apt-get update; \
grep "^[^#;]" requirements.apt | \
grep -Fv "python3-dev" | \
xargs apt-get install -y --no-install-recommends; \
rm -rf /usr/share/doc/* /usr/share/locale/*
RUN curl -L https://github.com/acoustid/chromaprint/releases/download/v1.4.2/chromaprint-fpcalc-1.4.2-linux-x86_64.tar.gz | tar -xz -C /usr/local/bin --strip 1
RUN mkdir /requirements
COPY ./requirements/base.txt /requirements/base.txt
RUN pip install -r /requirements/base.txt
COPY ./requirements/local.txt /requirements/local.txt
RUN pip install -r /requirements/local.txt
COPY ./requirements/test.txt /requirements/test.txt
RUN pip install -r /requirements/test.txt