Commit d63e7677 authored by Eliot Berriot's avatar Eliot Berriot 💬

WIP: Debian installation instructions

parent 6adc8f0c
Pipeline #154 passed with stage
in 30 seconds
......@@ -18,6 +18,8 @@ test_api:
- pip install -r requirements/test.txt
script:
- pytest
variables:
DATABASE_URL: "sqlite://"
tags:
- docker
......
......@@ -4,7 +4,7 @@ set -e
# Since docker-compose relies heavily on environment variables itself for configuration, we'd have to define multiple
# environment variables just to support cookiecutter out of the box. That makes no sense, so this little entrypoint
# does all this for us.
export REDIS_URL=redis://redis:6379/0
export CACHE_URL=redis://redis:6379/0
# the official postgres image uses 'postgres' as default user if not set explictly.
if [ -z "$POSTGRES_ENV_POSTGRES_USER" ]; then
......@@ -13,7 +13,7 @@ fi
export DATABASE_URL=postgres://$POSTGRES_ENV_POSTGRES_USER:$POSTGRES_ENV_POSTGRES_PASSWORD@postgres:5432/$POSTGRES_ENV_POSTGRES_USER
export CELERY_BROKER_URL=$REDIS_URL
export CELERY_BROKER_URL=$CACHE_URL
# we copy the frontend files, if any so we can serve them from the outside
if [ -d "frontend" ]; then
......
......@@ -123,7 +123,7 @@ MANAGERS = ADMINS
# See: https://docs.djangoproject.com/en/dev/ref/settings/#databases
DATABASES = {
# Raises ImproperlyConfigured exception if DATABASE_URL not in os.environ
'default': env.db("DATABASE_URL", default="postgresql://postgres@postgres/postgres"),
'default': env.db("DATABASE_URL"),
}
DATABASES['default']['ATOMIC_REQUESTS'] = True
#
......@@ -198,7 +198,7 @@ CRISPY_TEMPLATE_PACK = 'bootstrap3'
# STATIC FILE CONFIGURATION
# ------------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#static-root
STATIC_ROOT = str(ROOT_DIR('staticfiles'))
STATIC_ROOT = env("STATIC_ROOT", default=str(ROOT_DIR('staticfiles')))
# See: https://docs.djangoproject.com/en/dev/ref/settings/#static-url
STATIC_URL = env("STATIC_URL", default='/staticfiles/')
......@@ -217,12 +217,10 @@ STATICFILES_FINDERS = (
# MEDIA CONFIGURATION
# ------------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#media-root
MEDIA_ROOT = str(APPS_DIR('media'))
MEDIA_ROOT = env("MEDIA_ROOT", default=str(APPS_DIR('media')))
# See: https://docs.djangoproject.com/en/dev/ref/settings/#media-url
MEDIA_URL = '/media/'
MEDIA_URL = env("MEDIA_URL", default='/media/')
# URL Configuration
# ------------------------------------------------------------------------------
......@@ -252,26 +250,24 @@ LOGIN_URL = 'account_login'
# SLUGLIFIER
AUTOSLUG_SLUGIFY_FUNCTION = 'slugify.slugify'
########## CELERY
INSTALLED_APPS += ('funkwhale_api.taskapp.celery.CeleryConfig',)
# if you are not using the django database broker (e.g. rabbitmq, redis, memcached), you can remove the next line.
INSTALLED_APPS += ('kombu.transport.django',)
BROKER_URL = env("CELERY_BROKER_URL", default='django://')
########## END CELERY
CACHE_DEFAULT = "redis://127.0.0.1:6379/0"
CACHES = {
"default": {
"BACKEND": "django_redis.cache.RedisCache",
"LOCATION": "{0}/{1}".format(env.cache_url('REDIS_URL', default="redis://127.0.0.1:6379"), 0),
"OPTIONS": {
"CLIENT_CLASS": "django_redis.client.DefaultClient",
"IGNORE_EXCEPTIONS": True, # mimics memcache behavior.
# http://niwinz.github.io/django-redis/latest/#_memcached_exceptions_behavior
}
}
"default": env.cache_url('CACHE_URL', default=CACHE_DEFAULT)
}
CACHES["default"]["BACKEND"] = "django_redis.cache.RedisCache"
CACHES["default"]["OPTIONS"] = {
"CLIENT_CLASS": "django_redis.client.DefaultClient",
"IGNORE_EXCEPTIONS": True, # mimics memcache behavior.
# http://niwinz.github.io/django-redis/latest/#_memcached_exceptions_behavior
}
########## CELERY
INSTALLED_APPS += ('funkwhale_api.taskapp.celery.CeleryConfig',)
BROKER_URL = env(
"CELERY_BROKER_URL", default=env('CACHE_URL', default=CACHE_DEFAULT))
########## END CELERY
# Location of root django.contrib.admin URL, use {% url 'admin:index' %}
ADMIN_URL = r'^admin/'
# Your common stuff: Below this line define 3rd party library settings
......@@ -334,3 +330,7 @@ MUSICBRAINZ_CACHE_DURATION = env.int(
)
CACHALOT_ENABLED = env.bool('CACHALOT_ENABLED', default=True)
# Custom Admin URL, use {% url 'admin:index' %}
ADMIN_URL = env('DJANGO_ADMIN_URL', default='^api/admin/')
......@@ -54,7 +54,7 @@ SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
# ------------------------------------------------------------------------------
# Hosts/domain names that are valid for this site
# See https://docs.djangoproject.com/en/1.6/ref/settings/#allowed-hosts
ALLOWED_HOSTS = env.list('DJANGO_ALLOWED_HOSTS', default=['funkwhale.io'])
ALLOWED_HOSTS = env.list('DJANGO_ALLOWED_HOSTS')
# END SITE CONFIGURATION
INSTALLED_APPS += ("gunicorn", )
......@@ -65,10 +65,6 @@ INSTALLED_APPS += ("gunicorn", )
# ------------------------
DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage'
# URL that handles the media served from MEDIA_ROOT, used for managing
# stored files.
MEDIA_URL = '/media/'
# Static Assets
# ------------------------
STATICFILES_STORAGE = 'django.contrib.staticfiles.storage.StaticFilesStorage'
......@@ -92,11 +88,6 @@ TEMPLATES[0]['OPTIONS']['loaders'] = [
'django.template.loaders.filesystem.Loader', 'django.template.loaders.app_directories.Loader', ]),
]
# DATABASE CONFIGURATION
# ------------------------------------------------------------------------------
# Raises ImproperlyConfigured exception if DATABASE_URL not in os.environ
DATABASES['default'] = env.db("DATABASE_URL")
# CACHING
# ------------------------------------------------------------------------------
# Heroku URL does not pass the DB number, so we parse it in
......@@ -151,7 +142,5 @@ LOGGING = {
}
}
# Custom Admin URL, use {% url 'admin:index' %}
ADMIN_URL = env('DJANGO_ADMIN_URL')
# Your production stuff: Below this line define 3rd party library settings
......@@ -22,6 +22,9 @@ CACHES = {
'LOCATION': ''
}
}
INSTALLED_APPS += ('kombu.transport.django',)
BROKER_URL = 'django://'
# TESTING
# ------------------------------------------------------------------------------
TEST_RUNNER = 'django.test.runner.DiscoverRunner'
......
import glob
from django.core.management.base import BaseCommand, CommandError
from funkwhale_api.providers.audiofile import importer
from funkwhale_api.providers.audiofile import tasks
class Command(BaseCommand):
......@@ -61,7 +61,7 @@ class Command(BaseCommand):
for path in matching:
self.stdout.write(message.format(path))
try:
importer.from_path(path)
tasks.from_path(path)
except Exception as e:
self.stdout.write('Error: {}'.format(e))
......
......@@ -3,7 +3,7 @@ import datetime
import unittest
from test_plus.test import TestCase
from funkwhale_api.providers.audiofile import importer
from funkwhale_api.providers.audiofile import tasks
DATA_DIR = os.path.dirname(os.path.abspath(__file__))
......@@ -27,7 +27,7 @@ class TestAudioFile(TestCase):
return_value='OggVorbis',
)
with m1, m2:
track_file = importer.from_path(
track_file = tasks.from_path(
os.path.join(DATA_DIR, 'dummy_file.ogg'))
self.assertEqual(
......
......@@ -2,8 +2,11 @@
import os
import sys
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
if __name__ == "__main__":
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local")
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.production")
from django.core.management import execute_from_command_line
......
##basic build dependencies of various Django apps for Ubuntu 14.04
#build-essential metapackage install: make, gcc, g++,
build-essential
#required to translate
gettext
#python-dev
##shared dependencies of:
##Pillow, pylibmc
zlib1g-dev
##Postgresql and psycopg2 dependencies
libjpeg-dev
zlib1g-dev
libpq-dev
postgresql-client
##Pillow dependencies
#libtiff4-dev
#libjpeg8-dev
#libfreetype6-dev
#liblcms1-dev
#libwebp-dev
##django-extensions
#graphviz-dev
##hitch
#python-setuptools
#python3-dev
#python-virtualenv
#python-pip
#firefox
#automake
#libtool
#libreadline6
#libreadline6-dev
#libreadline-dev
libsqlite3-dev
#libxml2
#libxml2-dev
#libssl-dev
#libbz2-dev
#wget
#curl
#llvm
libav-tools
python3-dev
......@@ -4,3 +4,5 @@ test:
command: pytest
volumes:
- .:/app
environment:
- "DATABASE_URL=sqlite://"
# If you're tweaking this file from the template, ensure you edit at lest the
# If you're tweaking this file from the template, ensure you edit at least the
# following variables:
# - DJANGO_SECRET_KEY
# - DJANGO_ALLOWED_HOSTS
# Additionaly, on non-docker setup, you'll also have to tweak/uncomment those
# variables:
# - DATABASE_URL
# - CACHE_URL
# - STATIC_ROOT
# - MEDIA_ROOT
# Docker only
# -----------
# The tag of the image we should use
# (it will be interpolated in docker-compose file)
# You can comment or ignore this if you're not using docker
FUNKWHALE_VERSION=latest
......@@ -17,26 +25,52 @@ FUNKWHALE_VERSION=latest
# Set this variables to bind the API server to another interface/port
# example: FUNKWHALE_API_IP=0.0.0.0
# example: FUNKWHALE_API_PORT=5678
FUNKWHALE_API_IP=
FUNKWHALE_API_PORT=
FUNKWHALE_API_IP=127.0.0.1
FUNKWHALE_API_PORT=5000
# API/Django configuration
# which settings module should django use?
# You don't have to touch this unless you really know what you're doing
DJANGO_SETTINGS_MODULE=config.settings.production
# Database configuration
# Examples:
# DATABASE_URL=postgresql://<user>:<password>@<host>:<port>/<database>
# DATABASE_URL=postgresql://funkwhale:passw0rd@localhost:5432/funkwhale_database
# Use the next one if you followed Debian installation guide
# DATABASE_URL=postgresql://funkwhale@:5432/funkwhale
# Generate one using `openssl rand -base64 45`, for example
DJANGO_SECRET_KEY=
# Cache configuration
# Examples:
# CACHE_URL=redis://<host>:<port>/<database>
# CACHE_URL=redis://localhost:6379/0
# Use the next one if you followed Debian installation guide
# CACHE_URL=redis://127.0.0.1:6379/0
# Where media files (such as album covers or audio tracks) should be stored
# on your system?
# (Ensure this directory actually exists)
# MEDIA_ROOT=/srv/funkwhale/data/media
# You don't have to edit this
DJANGO_ADMIN_URL=^api/admin/
# Where static files (such as API css or icons) should be compiled
# on your system?
# (Ensure this directory actually exists)
# STATIC_ROOT=/srv/funkwhale/data/static
# Update it to match the domain that will be used to reach your funkwhale
# instance
# Example: DJANGO_ALLOWED_HOSTS=funkwhale.yourdomain.com
DJANGO_ALLOWED_HOSTS=yourdomain
# which settings module should django use?
# You don't have to touch this unless you really know what you're doing
DJANGO_SETTINGS_MODULE=config.settings.production
# Generate one using `openssl rand -base64 45`, for example
DJANGO_SECRET_KEY=
# You don't have to edit this, but you can put the admin on another URL if you
# want to
# DJANGO_ADMIN_URL=^api/admin/
# If True, unauthenticated users won't be able to query the API
API_AUTHENTICATION_REQUIRED=True
......
[Unit]
Description=Funkwhale application server
After=redis.service postgresql.service
PartOf=funkwhale.target
[Service]
User=funkwhale
# adapt this depending on the path of your funkwhale installation
WorkingDirectory=/srv/funkwhale/api
EnvironmentFile=/srv/funkwhale/config/.env
ExecStart=/srv/funkwhale/virtualenv/bin/gunicorn config.wsgi:application -b ${FUNKWHALE_API_IP}:${FUNKWHALE_API_PORT}
[Install]
WantedBy=multi-user.target
[Unit]
Description=Funkwhale celery worker
After=redis.service postgresql.service
PartOf=funkwhale.target
[Service]
User=funkwhale
# adapt this depending on the path of your funkwhale installation
WorkingDirectory=/srv/funkwhale/api
EnvironmentFile=/srv/funkwhale/config/.env
ExecStart=/srv/funkwhale/virtualenv/bin/python manage.py celery worker
[Install]
WantedBy=multi-user.target
[Unit]
Description=Funkwhale
Wants=funkwhale-server.service funkwhale-worker.service
# Ensure you update at least the server_name variables to match your own
# domain
upstream funkwhale-api {
# depending on your setup, you may want to udpate this
server localhost:5000;
}
server {
listen 80;
listen [::]:80;
server_name demo.funkwhale.audio;
# useful for Let's Encrypt
location /.well-known/acme-challenge/ { allow all; }
location / { return 301 https://$host$request_uri; }
listen 80;
listen [::]:80;
# update this to match your instance name
server_name demo.funkwhale.audio;
# useful for Let's Encrypt
location /.well-known/acme-challenge/ { allow all; }
location / { return 301 https://$host$request_uri; }
}
server {
listen 443 ssl http2;
listen [::]:443 ssl http2;
# update this to match your instance name
server_name demo.funkwhale.audio;
# TLS
# Feel free to use your own configuration for SSL here or simply remove the
# lines and move the configuration to the previous server block if you
# don't want to run funkwhale behind https (this is not recommanded)
# have a look here for let's encrypt configuration:
# https://certbot.eff.org/all-instructions/#debian-9-stretch-nginx
ssl_protocols TLSv1.2;
ssl_ciphers HIGH:!MEDIUM:!LOW:!aNULL:!NULL:!SHA;
ssl_prefer_server_ciphers on;
ssl_session_cache shared:SSL:10m;
ssl_certificate /etc/letsencrypt/live/example.com/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/example.com/privkey.pem;
ssl_certificate /etc/letsencrypt/live/demo.funkwhale.audio/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/demo.funkwhale.audio/privkey.pem;
# HSTS
add_header Strict-Transport-Security "max-age=31536000";
......
......@@ -34,6 +34,8 @@ services:
command: python manage.py celery worker
environment:
- C_FORCE_ROOT=true
- "DATABASE_URL=postgresql://postgres@postgres/postgres"
- "CACHE_URL=redis://redis:6379/0"
volumes:
- ./api:/app
- ./data/music:/music
......@@ -46,12 +48,14 @@ services:
volumes:
- ./api:/app
- ./data/music:/music
environment:
- "DATABASE_URL=postgresql://postgres@postgres/postgres"
- "CACHE_URL=redis://redis:6379/0"
ports:
- "12081:12081"
links:
- postgres
- redis
- celeryworker
nginx:
env_file: .env.dev
......
......@@ -21,9 +21,7 @@ Changelog
* [feature] can now import artist and releases from youtube and musicbrainz.
This requires a YouTube API key for the search
* [breaking] we now check for user permission before serving audio files, which requires
a specific configuration block in your reverse proxy configuration:
.. code-block::
a specific configuration block in your reverse proxy configuration::
location /_protected/media {
internal;
......
......@@ -17,10 +17,12 @@
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
# import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))
import os
import sys
sys.path.insert(0, os.path.abspath('../api'))
import funkwhale_api # NOQA
# -- General configuration ------------------------------------------------
......@@ -55,9 +57,11 @@ author = 'Eliot Berriot'
# built documents.
#
# The short X.Y version.
version = '0.1'
# version = funkwhale_api.__version__
# @TODO use real version here
version = 'feature/22-debian-installation'
# The full version, including alpha/beta/rc tags.
release = '0.1'
release = version
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
......@@ -152,6 +156,3 @@ texinfo_documents = [
author, 'funkwhale', 'One line description of project.',
'Miscellaneous'),
]
......@@ -5,10 +5,18 @@ From music directory on the server
----------------------------------
You can import music files in funkwhale assuming they are located on the server
and readable by the funkwhale application.
and readable by the funkwhale application. Your music files should contain at
least an ``artist``, ``album`` and ``title`` tags.
Assuming your music is located at ``/music`` and your music files contains at
least an ``artist``, ``album`` and ``title`` tag, you can import those tracks as follows:
You can import those tracks as follows, assuming they are located in
``/srv/funkwhale/data/music``:
.. code-block:: bash
python api/manage.py import_files "/srv/funkwhale/data/music/**/*.ogg" --recursive --noinput
When you use docker, the ``/srv/funkwhale/data/music`` is mounted from the host
to the ``/music`` directory on the container:
.. code-block:: bash
......@@ -17,6 +25,7 @@ least an ``artist``, ``album`` and ``title`` tag, you can import those tracks as
For the best results, we recommand tagging your music collection through
`Picard <http://picard.musicbrainz.org/>`_ in order to have the best quality metadata.
.. note::
This command is idempotent, meaning you can run it multiple times on the same
......@@ -26,6 +35,18 @@ For the best results, we recommand tagging your music collection through
At the moment, only OGG/Vorbis and MP3 files with ID3 tags are supported
.. note::
The --recursive flag will work only on Python 3.5+, which is the default
version When using Docker or Debian 9. If you use an older version of Python,
remove the --recursive flag and use more explicit import patterns instead::
# this will only import ogg files at the second level
"/srv/funkwhale/data/music/*/*.ogg"
# this will only import ogg files in the fiven directory
"/srv/funkwhale/data/music/System-of-a-down/*.ogg"
Getting demo tracks
^^^^^^^^^^^^^^^^^^^
......@@ -34,10 +55,10 @@ If you do not have any music on your server but still want to test the import
process, you can call the following methods do download a few albums licenced
under creative commons (courtesy of Jamendo):
.. code-block:: bash
.. parsed-literal::
curl -L -o download-tracks.sh "https://code.eliotberriot.com/funkwhale/funkwhale/raw/master/demo/download-tracks.sh"
curl -L -o music.txt "https://code.eliotberriot.com/funkwhale/funkwhale/raw/master/demo/music.txt"
curl -L -o download-tracks.sh "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/demo/download-tracks.sh"
curl -L -o music.txt "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/demo/music.txt"
chmod +x download-tracks.sh
./download-tracks.sh music.txt
......
Debian installation
===================
.. note::
this guide targets Debian 9, which is the latest debian, but should work
similarly on Debian 8.
External dependencies
---------------------
The guides will focus on installing funkwhale-specific components and
dependencies. However, funkwhale requires a
:doc:`few external dependencies <./external_dependencies>` for which
documentation is outside of this document scope.
Install utilities
-----------------
You'll need a few utilities during this guide that are not always present by
default on system. You can install them using:
.. code-block:: shell
sudo apt-get update
sudo apt-get install curl python3-venv git unzip
Layout
-------
All funkwhale-related files will be located under ``/srv/funkwhale`` apart
from database files and a few configuration files. We will also have a
dedicated ``funwhale`` user to launch the processes we need and own those files.
You are free to use different values here, just remember to adapt those in the
next steps.
.. _create-funkwhale-user:
Create the user and the directory:
.. code-block:: shell
sudo adduser --system --home /srv/funkwhale funkwhale
cd /srv/funkwhale
Log in as the newly created user from now on:
.. code-block:: shell
sudo -u funkwhale -H bash
Now let's setup our directory layout. Here is how it will look like::
.
├── config # config / environment files
├── api # api code of your instance
├── data # persistent data, such as music files
├── front # frontend files for the web user interface
└── virtualenv # python dependencies for funkwhale
Create the aforementionned directories:
.. code-block:: shell
mkdir -p config api data/static data/media data/music front
The ``virtualenv`` directory is a bit special and will be created separately.
Download latest funkwhale release
----------------------------------
Funkwhale is splitted in two components:
1. The API, which will handle music storage and user accounts
2. The frontend, that will simply connect to the API to interact with its data
Those components are packaged in subsequent releases, such as 0.1, 0.2, etc.
You can browse the :doc:`changelog </changelog>` for a list of available releases
and pick the one you want to install, usually the latest one should be okay.
In this guide, we'll assume you want to install the latest version of funkwhale,
which is |version|:
First, we'll download the latest api release.
.. parsed-literal::
curl -L -o "api-|version|.zip" "https://code.eliotberriot.com/funkwhale/funkwhale/-/jobs/artifacts/|version|/download?job=build_api"
unzip "api-|version|.zip" -d extracted
mv extracted/api api
rmdir extracted
Then we'll download the frontend files:
.. parsed-literal::
curl -L -o "front-|version|.zip" "https://code.eliotberriot.com/funkwhale/funkwhale/-/jobs/artifacts/|version|/download?job=build_front"
unzip "front-|version|.zip" -d extracted
mv extracted/front .
rmdir extracted
You can leave the ZIP archives in the directory, this will help you know
which version you've installed next time you want to upgrade your installation.
System dependencies
-------------------
First, switch to the api directory:
.. code-block:: shell
cd api
A few OS packages are required in order to run funkwhale. The list is available
in ``api/requirements.apt`` or by running
``./install_os_dependencies.sh list``.
.. note::
Ensure you are running the next commands as root or using sudo
(and not as the funkwhale) user.
You can install those packages all at once:
.. code-block:: shell
./install_os_dependencies.sh install
From now on you can switch back to the funkwhale user.
Python dependencies
--------------------
Go back to the base directory:
.. code-block:: shell
cd /srv/funkwhale
To avoid collisions with other software on your system, Python dependencies
will be installed in a dedicated
`virtualenv <https://docs.python.org/3/library/venv.html>`_.
First, create the virtualenv:
.. code-block:: shell
python3 -m venv /srv/funkwhale/virtualenv
This will result in a ``virtualenv`` directory being created in
``/srv/funkwhale/virtualenv``.
In the rest of this guide, we'll need to activate this environment to ensure
dependencies are installed within it, and not directly on your host system.
This is done with the following command:
.. code-block:: shell
source /srv/funkwhale/virtualenv/bin/activate
Finally, install the python dependencies:
.. code-block:: shell
pip install wheel
pip install -r api/requirements.txt
.. important::
further commands involving python should always be run after you activated
the virtualenv, as described earlier, otherwise those commands will raise
errors
Environment file
----------------
You can now start to configure funkwhale. The main way to achieve that is by
adding an environment file that will host settings that are relevant to your
installation.
Download the sample environment file:
.. parsed-literal::
curl -L -o config/.env "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/deploy/env.prod.sample"
You can then edit it: the file is heavily commented, and the most relevant
configuration options are mentionned at the top of the file.