Skip to content
Snippets Groups Projects
Commit 4c9469ac authored by Eliot Berriot's avatar Eliot Berriot
Browse files

Merge branch 'feature/22-debian-installation' into 'develop'

Initial documentation on debian

See merge request funkwhale/funkwhale!15
parents a91a8b79 d63e7677
No related branches found
No related tags found
No related merge requests found
Showing
with 179 additions and 118 deletions
......@@ -18,6 +18,8 @@ test_api:
- pip install -r requirements/test.txt
script:
- pytest
variables:
DATABASE_URL: "sqlite://"
tags:
- docker
......
......@@ -4,7 +4,7 @@ set -e
# Since docker-compose relies heavily on environment variables itself for configuration, we'd have to define multiple
# environment variables just to support cookiecutter out of the box. That makes no sense, so this little entrypoint
# does all this for us.
export REDIS_URL=redis://redis:6379/0
export CACHE_URL=redis://redis:6379/0
# the official postgres image uses 'postgres' as default user if not set explictly.
if [ -z "$POSTGRES_ENV_POSTGRES_USER" ]; then
......@@ -13,7 +13,7 @@ fi
export DATABASE_URL=postgres://$POSTGRES_ENV_POSTGRES_USER:$POSTGRES_ENV_POSTGRES_PASSWORD@postgres:5432/$POSTGRES_ENV_POSTGRES_USER
export CELERY_BROKER_URL=$REDIS_URL
export CELERY_BROKER_URL=$CACHE_URL
# we copy the frontend files, if any so we can serve them from the outside
if [ -d "frontend" ]; then
......
......@@ -124,7 +124,7 @@ MANAGERS = ADMINS
# See: https://docs.djangoproject.com/en/dev/ref/settings/#databases
DATABASES = {
# Raises ImproperlyConfigured exception if DATABASE_URL not in os.environ
'default': env.db("DATABASE_URL", default="postgresql://postgres@postgres/postgres"),
'default': env.db("DATABASE_URL"),
}
DATABASES['default']['ATOMIC_REQUESTS'] = True
#
......@@ -199,7 +199,7 @@ CRISPY_TEMPLATE_PACK = 'bootstrap3'
# STATIC FILE CONFIGURATION
# ------------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#static-root
STATIC_ROOT = str(ROOT_DIR('staticfiles'))
STATIC_ROOT = env("STATIC_ROOT", default=str(ROOT_DIR('staticfiles')))
# See: https://docs.djangoproject.com/en/dev/ref/settings/#static-url
STATIC_URL = env("STATIC_URL", default='/staticfiles/')
......@@ -218,12 +218,10 @@ STATICFILES_FINDERS = (
# MEDIA CONFIGURATION
# ------------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#media-root
MEDIA_ROOT = str(APPS_DIR('media'))
MEDIA_ROOT = env("MEDIA_ROOT", default=str(APPS_DIR('media')))
# See: https://docs.djangoproject.com/en/dev/ref/settings/#media-url
MEDIA_URL = '/media/'
MEDIA_URL = env("MEDIA_URL", default='/media/')
# URL Configuration
# ------------------------------------------------------------------------------
......@@ -253,26 +251,24 @@ LOGIN_URL = 'account_login'
# SLUGLIFIER
AUTOSLUG_SLUGIFY_FUNCTION = 'slugify.slugify'
########## CELERY
INSTALLED_APPS += ('funkwhale_api.taskapp.celery.CeleryConfig',)
# if you are not using the django database broker (e.g. rabbitmq, redis, memcached), you can remove the next line.
INSTALLED_APPS += ('kombu.transport.django',)
BROKER_URL = env("CELERY_BROKER_URL", default='django://')
########## END CELERY
CACHE_DEFAULT = "redis://127.0.0.1:6379/0"
CACHES = {
"default": {
"BACKEND": "django_redis.cache.RedisCache",
"LOCATION": "{0}/{1}".format(env.cache_url('REDIS_URL', default="redis://127.0.0.1:6379"), 0),
"OPTIONS": {
"CLIENT_CLASS": "django_redis.client.DefaultClient",
"IGNORE_EXCEPTIONS": True, # mimics memcache behavior.
# http://niwinz.github.io/django-redis/latest/#_memcached_exceptions_behavior
}
}
"default": env.cache_url('CACHE_URL', default=CACHE_DEFAULT)
}
CACHES["default"]["BACKEND"] = "django_redis.cache.RedisCache"
CACHES["default"]["OPTIONS"] = {
"CLIENT_CLASS": "django_redis.client.DefaultClient",
"IGNORE_EXCEPTIONS": True, # mimics memcache behavior.
# http://niwinz.github.io/django-redis/latest/#_memcached_exceptions_behavior
}
########## CELERY
INSTALLED_APPS += ('funkwhale_api.taskapp.celery.CeleryConfig',)
BROKER_URL = env(
"CELERY_BROKER_URL", default=env('CACHE_URL', default=CACHE_DEFAULT))
########## END CELERY
# Location of root django.contrib.admin URL, use {% url 'admin:index' %}
ADMIN_URL = r'^admin/'
# Your common stuff: Below this line define 3rd party library settings
......@@ -336,3 +332,7 @@ MUSICBRAINZ_CACHE_DURATION = env.int(
)
CACHALOT_ENABLED = env.bool('CACHALOT_ENABLED', default=True)
# Custom Admin URL, use {% url 'admin:index' %}
ADMIN_URL = env('DJANGO_ADMIN_URL', default='^api/admin/')
......@@ -54,7 +54,7 @@ SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
# ------------------------------------------------------------------------------
# Hosts/domain names that are valid for this site
# See https://docs.djangoproject.com/en/1.6/ref/settings/#allowed-hosts
ALLOWED_HOSTS = env.list('DJANGO_ALLOWED_HOSTS', default=['funkwhale.io'])
ALLOWED_HOSTS = env.list('DJANGO_ALLOWED_HOSTS')
# END SITE CONFIGURATION
INSTALLED_APPS += ("gunicorn", )
......@@ -65,10 +65,6 @@ INSTALLED_APPS += ("gunicorn", )
# ------------------------
DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage'
# URL that handles the media served from MEDIA_ROOT, used for managing
# stored files.
MEDIA_URL = '/media/'
# Static Assets
# ------------------------
STATICFILES_STORAGE = 'django.contrib.staticfiles.storage.StaticFilesStorage'
......@@ -92,11 +88,6 @@ TEMPLATES[0]['OPTIONS']['loaders'] = [
'django.template.loaders.filesystem.Loader', 'django.template.loaders.app_directories.Loader', ]),
]
# DATABASE CONFIGURATION
# ------------------------------------------------------------------------------
# Raises ImproperlyConfigured exception if DATABASE_URL not in os.environ
DATABASES['default'] = env.db("DATABASE_URL")
# CACHING
# ------------------------------------------------------------------------------
# Heroku URL does not pass the DB number, so we parse it in
......@@ -151,7 +142,5 @@ LOGGING = {
}
}
# Custom Admin URL, use {% url 'admin:index' %}
ADMIN_URL = env('DJANGO_ADMIN_URL')
# Your production stuff: Below this line define 3rd party library settings
......@@ -22,6 +22,9 @@ CACHES = {
'LOCATION': ''
}
}
INSTALLED_APPS += ('kombu.transport.django',)
BROKER_URL = 'django://'
# TESTING
# ------------------------------------------------------------------------------
TEST_RUNNER = 'django.test.runner.DiscoverRunner'
......
import glob
from django.core.management.base import BaseCommand, CommandError
from funkwhale_api.providers.audiofile import importer
from funkwhale_api.providers.audiofile import tasks
class Command(BaseCommand):
......@@ -61,7 +61,7 @@ class Command(BaseCommand):
for path in matching:
self.stdout.write(message.format(path))
try:
importer.from_path(path)
tasks.from_path(path)
except Exception as e:
self.stdout.write('Error: {}'.format(e))
......
......@@ -3,7 +3,7 @@ import datetime
import unittest
from test_plus.test import TestCase
from funkwhale_api.providers.audiofile import importer
from funkwhale_api.providers.audiofile import tasks
DATA_DIR = os.path.dirname(os.path.abspath(__file__))
......@@ -27,7 +27,7 @@ class TestAudioFile(TestCase):
return_value='OggVorbis',
)
with m1, m2:
track_file = importer.from_path(
track_file = tasks.from_path(
os.path.join(DATA_DIR, 'dummy_file.ogg'))
self.assertEqual(
......
......@@ -2,8 +2,11 @@
import os
import sys
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
if __name__ == "__main__":
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local")
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.production")
from django.core.management import execute_from_command_line
......
##basic build dependencies of various Django apps for Ubuntu 14.04
#build-essential metapackage install: make, gcc, g++,
build-essential
#required to translate
gettext
#python-dev
##shared dependencies of:
##Pillow, pylibmc
zlib1g-dev
##Postgresql and psycopg2 dependencies
libjpeg-dev
zlib1g-dev
libpq-dev
postgresql-client
##Pillow dependencies
#libtiff4-dev
#libjpeg8-dev
#libfreetype6-dev
#liblcms1-dev
#libwebp-dev
##django-extensions
#graphviz-dev
##hitch
#python-setuptools
#python3-dev
#python-virtualenv
#python-pip
#firefox
#automake
#libtool
#libreadline6
#libreadline6-dev
#libreadline-dev
libsqlite3-dev
#libxml2
#libxml2-dev
#libssl-dev
#libbz2-dev
#wget
#curl
#llvm
libav-tools
python3-dev
......@@ -4,3 +4,5 @@ test:
command: pytest
volumes:
- .:/app
environment:
- "DATABASE_URL=sqlite://"
# If you're tweaking this file from the template, ensure you edit at lest the
# If you're tweaking this file from the template, ensure you edit at least the
# following variables:
# - DJANGO_SECRET_KEY
# - DJANGO_ALLOWED_HOSTS
# Additionaly, on non-docker setup, you'll also have to tweak/uncomment those
# variables:
# - DATABASE_URL
# - CACHE_URL
# - STATIC_ROOT
# - MEDIA_ROOT
# Docker only
# -----------
# The tag of the image we should use
# (it will be interpolated in docker-compose file)
# You can comment or ignore this if you're not using docker
FUNKWHALE_VERSION=latest
......@@ -17,26 +25,52 @@ FUNKWHALE_VERSION=latest
# Set this variables to bind the API server to another interface/port
# example: FUNKWHALE_API_IP=0.0.0.0
# example: FUNKWHALE_API_PORT=5678
FUNKWHALE_API_IP=
FUNKWHALE_API_PORT=
FUNKWHALE_API_IP=127.0.0.1
FUNKWHALE_API_PORT=5000
# API/Django configuration
# which settings module should django use?
# You don't have to touch this unless you really know what you're doing
DJANGO_SETTINGS_MODULE=config.settings.production
# Database configuration
# Examples:
# DATABASE_URL=postgresql://<user>:<password>@<host>:<port>/<database>
# DATABASE_URL=postgresql://funkwhale:passw0rd@localhost:5432/funkwhale_database
# Use the next one if you followed Debian installation guide
# DATABASE_URL=postgresql://funkwhale@:5432/funkwhale
# Generate one using `openssl rand -base64 45`, for example
DJANGO_SECRET_KEY=
# Cache configuration
# Examples:
# CACHE_URL=redis://<host>:<port>/<database>
# CACHE_URL=redis://localhost:6379/0
# Use the next one if you followed Debian installation guide
# CACHE_URL=redis://127.0.0.1:6379/0
# Where media files (such as album covers or audio tracks) should be stored
# on your system?
# (Ensure this directory actually exists)
# MEDIA_ROOT=/srv/funkwhale/data/media
# You don't have to edit this
DJANGO_ADMIN_URL=^api/admin/
# Where static files (such as API css or icons) should be compiled
# on your system?
# (Ensure this directory actually exists)
# STATIC_ROOT=/srv/funkwhale/data/static
# Update it to match the domain that will be used to reach your funkwhale
# instance
# Example: DJANGO_ALLOWED_HOSTS=funkwhale.yourdomain.com
DJANGO_ALLOWED_HOSTS=yourdomain
# which settings module should django use?
# You don't have to touch this unless you really know what you're doing
DJANGO_SETTINGS_MODULE=config.settings.production
# Generate one using `openssl rand -base64 45`, for example
DJANGO_SECRET_KEY=
# You don't have to edit this, but you can put the admin on another URL if you
# want to
# DJANGO_ADMIN_URL=^api/admin/
# If True, unauthenticated users won't be able to query the API
API_AUTHENTICATION_REQUIRED=True
......
[Unit]
Description=Funkwhale application server
After=redis.service postgresql.service
PartOf=funkwhale.target
[Service]
User=funkwhale
# adapt this depending on the path of your funkwhale installation
WorkingDirectory=/srv/funkwhale/api
EnvironmentFile=/srv/funkwhale/config/.env
ExecStart=/srv/funkwhale/virtualenv/bin/gunicorn config.wsgi:application -b ${FUNKWHALE_API_IP}:${FUNKWHALE_API_PORT}
[Install]
WantedBy=multi-user.target
[Unit]
Description=Funkwhale celery worker
After=redis.service postgresql.service
PartOf=funkwhale.target
[Service]
User=funkwhale
# adapt this depending on the path of your funkwhale installation
WorkingDirectory=/srv/funkwhale/api
EnvironmentFile=/srv/funkwhale/config/.env
ExecStart=/srv/funkwhale/virtualenv/bin/python manage.py celery worker
[Install]
WantedBy=multi-user.target
[Unit]
Description=Funkwhale
Wants=funkwhale-server.service funkwhale-worker.service
# Ensure you update at least the server_name variables to match your own
# domain
upstream funkwhale-api {
# depending on your setup, you may want to udpate this
server localhost:5000;
}
server {
listen 80;
listen [::]:80;
server_name demo.funkwhale.audio;
# useful for Let's Encrypt
location /.well-known/acme-challenge/ { allow all; }
location / { return 301 https://$host$request_uri; }
listen 80;
listen [::]:80;
# update this to match your instance name
server_name demo.funkwhale.audio;
# useful for Let's Encrypt
location /.well-known/acme-challenge/ { allow all; }
location / { return 301 https://$host$request_uri; }
}
server {
listen 443 ssl http2;
listen [::]:443 ssl http2;
# update this to match your instance name
server_name demo.funkwhale.audio;
# TLS
# Feel free to use your own configuration for SSL here or simply remove the
# lines and move the configuration to the previous server block if you
# don't want to run funkwhale behind https (this is not recommanded)
# have a look here for let's encrypt configuration:
# https://certbot.eff.org/all-instructions/#debian-9-stretch-nginx
ssl_protocols TLSv1.2;
ssl_ciphers HIGH:!MEDIUM:!LOW:!aNULL:!NULL:!SHA;
ssl_prefer_server_ciphers on;
ssl_session_cache shared:SSL:10m;
ssl_certificate /etc/letsencrypt/live/example.com/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/example.com/privkey.pem;
ssl_certificate /etc/letsencrypt/live/demo.funkwhale.audio/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/demo.funkwhale.audio/privkey.pem;
# HSTS
add_header Strict-Transport-Security "max-age=31536000";
......
......@@ -34,6 +34,8 @@ services:
command: python manage.py celery worker
environment:
- C_FORCE_ROOT=true
- "DATABASE_URL=postgresql://postgres@postgres/postgres"
- "CACHE_URL=redis://redis:6379/0"
volumes:
- ./api:/app
- ./data/music:/music
......@@ -46,12 +48,14 @@ services:
volumes:
- ./api:/app
- ./data/music:/music
environment:
- "DATABASE_URL=postgresql://postgres@postgres/postgres"
- "CACHE_URL=redis://redis:6379/0"
ports:
- "12081:12081"
links:
- postgres
- redis
- celeryworker
nginx:
env_file: .env.dev
......
......@@ -21,9 +21,7 @@ Changelog
* [feature] can now import artist and releases from youtube and musicbrainz.
This requires a YouTube API key for the search
* [breaking] we now check for user permission before serving audio files, which requires
a specific configuration block in your reverse proxy configuration:
.. code-block::
a specific configuration block in your reverse proxy configuration::
location /_protected/media {
internal;
......
......@@ -17,10 +17,12 @@
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
# import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))
import os
import sys
sys.path.insert(0, os.path.abspath('../api'))
import funkwhale_api # NOQA
# -- General configuration ------------------------------------------------
......@@ -55,9 +57,11 @@ author = 'Eliot Berriot'
# built documents.
#
# The short X.Y version.
version = '0.1'
# version = funkwhale_api.__version__
# @TODO use real version here
version = 'feature/22-debian-installation'
# The full version, including alpha/beta/rc tags.
release = '0.1'
release = version
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
......@@ -152,6 +156,3 @@ texinfo_documents = [
author, 'funkwhale', 'One line description of project.',
'Miscellaneous'),
]
......@@ -5,10 +5,18 @@ From music directory on the server
----------------------------------
You can import music files in funkwhale assuming they are located on the server
and readable by the funkwhale application.
and readable by the funkwhale application. Your music files should contain at
least an ``artist``, ``album`` and ``title`` tags.
Assuming your music is located at ``/music`` and your music files contains at
least an ``artist``, ``album`` and ``title`` tag, you can import those tracks as follows:
You can import those tracks as follows, assuming they are located in
``/srv/funkwhale/data/music``:
.. code-block:: bash
python api/manage.py import_files "/srv/funkwhale/data/music/**/*.ogg" --recursive --noinput
When you use docker, the ``/srv/funkwhale/data/music`` is mounted from the host
to the ``/music`` directory on the container:
.. code-block:: bash
......@@ -17,6 +25,7 @@ least an ``artist``, ``album`` and ``title`` tag, you can import those tracks as
For the best results, we recommand tagging your music collection through
`Picard <http://picard.musicbrainz.org/>`_ in order to have the best quality metadata.
.. note::
This command is idempotent, meaning you can run it multiple times on the same
......@@ -26,6 +35,18 @@ For the best results, we recommand tagging your music collection through
At the moment, only OGG/Vorbis and MP3 files with ID3 tags are supported
.. note::
The --recursive flag will work only on Python 3.5+, which is the default
version When using Docker or Debian 9. If you use an older version of Python,
remove the --recursive flag and use more explicit import patterns instead::
# this will only import ogg files at the second level
"/srv/funkwhale/data/music/*/*.ogg"
# this will only import ogg files in the fiven directory
"/srv/funkwhale/data/music/System-of-a-down/*.ogg"
Getting demo tracks
^^^^^^^^^^^^^^^^^^^
......@@ -34,10 +55,10 @@ If you do not have any music on your server but still want to test the import
process, you can call the following methods do download a few albums licenced
under creative commons (courtesy of Jamendo):
.. code-block:: bash
.. parsed-literal::
curl -L -o download-tracks.sh "https://code.eliotberriot.com/funkwhale/funkwhale/raw/master/demo/download-tracks.sh"
curl -L -o music.txt "https://code.eliotberriot.com/funkwhale/funkwhale/raw/master/demo/music.txt"
curl -L -o download-tracks.sh "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/demo/download-tracks.sh"
curl -L -o music.txt "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/demo/music.txt"
chmod +x download-tracks.sh
./download-tracks.sh music.txt
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment