Merge branch 'feature/22-debian-installation' into 'develop'

Initial documentation on debian

See merge request funkwhale/funkwhale!15

.gitlab-ci.yml

@@ -18,6 +18,8 @@ test_api:
     - pip install -r requirements/test.txt
   script:
     - pytest
+  variables:
+    DATABASE_URL: "sqlite://"
 
   tags:
     - docker

api/compose/django/entrypoint.sh

@@ -4,7 +4,7 @@ set -e
 # Since docker-compose relies heavily on environment variables itself for configuration, we'd have to define multiple
 # environment variables just to support cookiecutter out of the box. That makes no sense, so this little entrypoint
 # does all this for us.
-export REDIS_URL=redis://redis:6379/0
+export CACHE_URL=redis://redis:6379/0
 
 # the official postgres image uses 'postgres' as default user if not set explictly.
 if [ -z "$POSTGRES_ENV_POSTGRES_USER" ]; then
@@ -13,7 +13,7 @@ fi
 
 export DATABASE_URL=postgres://$POSTGRES_ENV_POSTGRES_USER:$POSTGRES_ENV_POSTGRES_PASSWORD@postgres:5432/$POSTGRES_ENV_POSTGRES_USER
 
-export CELERY_BROKER_URL=$REDIS_URL
+export CELERY_BROKER_URL=$CACHE_URL
 
 # we copy the frontend files, if any so we can serve them from the outside
 if [ -d "frontend" ]; then

api/config/settings/common.py

@@ -124,7 +124,7 @@ MANAGERS = ADMINS
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#databases
 DATABASES = {
     # Raises ImproperlyConfigured exception if DATABASE_URL not in os.environ
-    'default': env.db("DATABASE_URL", default="postgresql://postgres@postgres/postgres"),
+    'default': env.db("DATABASE_URL"),
 }
 DATABASES['default']['ATOMIC_REQUESTS'] = True
 #
@@ -199,7 +199,7 @@ CRISPY_TEMPLATE_PACK = 'bootstrap3'
 # STATIC FILE CONFIGURATION
 # ------------------------------------------------------------------------------
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#static-root
-STATIC_ROOT = str(ROOT_DIR('staticfiles'))
+STATIC_ROOT = env("STATIC_ROOT", default=str(ROOT_DIR('staticfiles')))
 
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#static-url
 STATIC_URL = env("STATIC_URL", default='/staticfiles/')
@@ -218,12 +218,10 @@ STATICFILES_FINDERS = (
 # MEDIA CONFIGURATION
 # ------------------------------------------------------------------------------
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#media-root
-MEDIA_ROOT = str(APPS_DIR('media'))
-
-
+MEDIA_ROOT = env("MEDIA_ROOT", default=str(APPS_DIR('media')))
 
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#media-url
-MEDIA_URL = '/media/'
+MEDIA_URL = env("MEDIA_URL", default='/media/')
 
 # URL Configuration
 # ------------------------------------------------------------------------------
@@ -253,26 +251,24 @@ LOGIN_URL = 'account_login'
 # SLUGLIFIER
 AUTOSLUG_SLUGIFY_FUNCTION = 'slugify.slugify'
 
-########## CELERY
-INSTALLED_APPS += ('funkwhale_api.taskapp.celery.CeleryConfig',)
-# if you are not using the django database broker (e.g. rabbitmq, redis, memcached), you can remove the next line.
-INSTALLED_APPS += ('kombu.transport.django',)
-BROKER_URL = env("CELERY_BROKER_URL", default='django://')
-########## END CELERY
-
-
+CACHE_DEFAULT = "redis://127.0.0.1:6379/0"
 CACHES = {
-    "default": {
-        "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": "{0}/{1}".format(env.cache_url('REDIS_URL', default="redis://127.0.0.1:6379"), 0),
-        "OPTIONS": {
-            "CLIENT_CLASS": "django_redis.client.DefaultClient",
-            "IGNORE_EXCEPTIONS": True,  # mimics memcache behavior.
-                                        # http://niwinz.github.io/django-redis/latest/#_memcached_exceptions_behavior
-        }
-    }
+    "default": env.cache_url('CACHE_URL', default=CACHE_DEFAULT)
 }
 
+CACHES["default"]["BACKEND"] = "django_redis.cache.RedisCache"
+CACHES["default"]["OPTIONS"] = {
+    "CLIENT_CLASS": "django_redis.client.DefaultClient",
+    "IGNORE_EXCEPTIONS": True,  # mimics memcache behavior.
+                                # http://niwinz.github.io/django-redis/latest/#_memcached_exceptions_behavior
+}
+
+
+########## CELERY
+INSTALLED_APPS += ('funkwhale_api.taskapp.celery.CeleryConfig',)
+BROKER_URL = env(
+    "CELERY_BROKER_URL", default=env('CACHE_URL', default=CACHE_DEFAULT))
+########## END CELERY
 # Location of root django.contrib.admin URL, use {% url 'admin:index' %}
 ADMIN_URL = r'^admin/'
 # Your common stuff: Below this line define 3rd party library settings
@@ -336,3 +332,7 @@ MUSICBRAINZ_CACHE_DURATION = env.int(
 )
 
 CACHALOT_ENABLED = env.bool('CACHALOT_ENABLED', default=True)
+
+
+# Custom Admin URL, use {% url 'admin:index' %}
+ADMIN_URL = env('DJANGO_ADMIN_URL', default='^api/admin/')

api/config/settings/production.py

@@ -54,7 +54,7 @@ SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
 # ------------------------------------------------------------------------------
 # Hosts/domain names that are valid for this site
 # See https://docs.djangoproject.com/en/1.6/ref/settings/#allowed-hosts
-ALLOWED_HOSTS = env.list('DJANGO_ALLOWED_HOSTS', default=['funkwhale.io'])
+ALLOWED_HOSTS = env.list('DJANGO_ALLOWED_HOSTS')
 # END SITE CONFIGURATION
 
 INSTALLED_APPS += ("gunicorn", )
@@ -65,10 +65,6 @@ INSTALLED_APPS += ("gunicorn", )
 # ------------------------
 DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage'
 
-# URL that handles the media served from MEDIA_ROOT, used for managing
-# stored files.
-MEDIA_URL = '/media/'
-
 # Static Assets
 # ------------------------
 STATICFILES_STORAGE = 'django.contrib.staticfiles.storage.StaticFilesStorage'
@@ -92,11 +88,6 @@ TEMPLATES[0]['OPTIONS']['loaders'] = [
         'django.template.loaders.filesystem.Loader', 'django.template.loaders.app_directories.Loader', ]),
 ]
 
-# DATABASE CONFIGURATION
-# ------------------------------------------------------------------------------
-# Raises ImproperlyConfigured exception if DATABASE_URL not in os.environ
-DATABASES['default'] = env.db("DATABASE_URL")
-
 # CACHING
 # ------------------------------------------------------------------------------
 # Heroku URL does not pass the DB number, so we parse it in
@@ -151,7 +142,5 @@ LOGGING = {
     }
 }
 
-# Custom Admin URL, use {% url 'admin:index' %}
-ADMIN_URL = env('DJANGO_ADMIN_URL')
 
 # Your production stuff: Below this line define 3rd party library settings

api/config/settings/test.py

@@ -22,6 +22,9 @@ CACHES = {
         'LOCATION': ''
     }
 }
+INSTALLED_APPS += ('kombu.transport.django',)
+BROKER_URL = 'django://'
+
 # TESTING
 # ------------------------------------------------------------------------------
 TEST_RUNNER = 'django.test.runner.DiscoverRunner'

api/funkwhale_api/providers/audiofile/management/commands/import_files.py

 import glob
 from django.core.management.base import BaseCommand, CommandError
-from funkwhale_api.providers.audiofile import importer
+from funkwhale_api.providers.audiofile import tasks
 
 
 class Command(BaseCommand):
@@ -61,7 +61,7 @@ class Command(BaseCommand):
         for path in matching:
             self.stdout.write(message.format(path))
             try:
-                importer.from_path(path)
+                tasks.from_path(path)
             except Exception as e:
                 self.stdout.write('Error: {}'.format(e))
 

api/funkwhale_api/providers/audiofile/tests/test_disk_import.py

@@ -3,7 +3,7 @@ import datetime
 import unittest
 from test_plus.test import TestCase
 
-from funkwhale_api.providers.audiofile import importer
+from funkwhale_api.providers.audiofile import tasks
 
 DATA_DIR = os.path.dirname(os.path.abspath(__file__))
 
@@ -27,7 +27,7 @@ class TestAudioFile(TestCase):
             return_value='OggVorbis',
         )
         with m1, m2:
-            track_file = importer.from_path(
+            track_file = tasks.from_path(
                 os.path.join(DATA_DIR, 'dummy_file.ogg'))
 
         self.assertEqual(

api/manage.py

@@ -2,8 +2,11 @@
 import os
 import sys
 
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+
 if __name__ == "__main__":
-    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local")
+
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.production")
 
     from django.core.management import execute_from_command_line
 

api/requirements.apt

-##basic build dependencies of various Django apps for Ubuntu 14.04
-#build-essential metapackage install: make, gcc, g++,
 build-essential
-#required to translate
 gettext
-#python-dev
-
-##shared dependencies of:
-##Pillow, pylibmc
 zlib1g-dev
-
-##Postgresql and psycopg2 dependencies
+libjpeg-dev
+zlib1g-dev
 libpq-dev
 postgresql-client
-##Pillow dependencies
-#libtiff4-dev
-#libjpeg8-dev
-#libfreetype6-dev
-#liblcms1-dev
-#libwebp-dev
-
-
-##django-extensions
-#graphviz-dev
-
-##hitch
-#python-setuptools
-#python3-dev
-#python-virtualenv
-#python-pip
-#firefox
-#automake
-#libtool
-#libreadline6
-#libreadline6-dev
-#libreadline-dev
-libsqlite3-dev
-#libxml2
-#libxml2-dev
-#libssl-dev
-#libbz2-dev
-#wget
-#curl
-#llvm
-
 libav-tools
+python3-dev

api/test.yml

@@ -4,3 +4,5 @@ test:
   command: pytest
   volumes:
     - .:/app
+  environment:
+    - "DATABASE_URL=sqlite://"

deploy/env.prod.sample

-# If you're tweaking this file from the template, ensure you edit at lest the
+# If you're tweaking this file from the template, ensure you edit at least the
 # following variables:
 # - DJANGO_SECRET_KEY
 # - DJANGO_ALLOWED_HOSTS
 
+# Additionaly, on non-docker setup, you'll also have to tweak/uncomment those
+# variables:
+# - DATABASE_URL
+# - CACHE_URL
+# - STATIC_ROOT
+# - MEDIA_ROOT
+
 # Docker only
 # -----------
 
 # The tag of the image we should use
 # (it will be interpolated in docker-compose file)
+# You can comment or ignore this if you're not using docker
 FUNKWHALE_VERSION=latest
 
 
@@ -17,26 +25,52 @@ FUNKWHALE_VERSION=latest
 # Set this variables to bind the API server to another interface/port
 # example: FUNKWHALE_API_IP=0.0.0.0
 # example: FUNKWHALE_API_PORT=5678
-FUNKWHALE_API_IP=
-FUNKWHALE_API_PORT=
+FUNKWHALE_API_IP=127.0.0.1
+FUNKWHALE_API_PORT=5000
+
 
 # API/Django configuration
 
-# which settings module should django use?
-# You don't have to touch this unless you really know what you're doing
-DJANGO_SETTINGS_MODULE=config.settings.production
+# Database configuration
+# Examples:
+#  DATABASE_URL=postgresql://<user>:<password>@<host>:<port>/<database>
+#  DATABASE_URL=postgresql://funkwhale:passw0rd@localhost:5432/funkwhale_database
+# Use the next one if you followed Debian installation guide
+# DATABASE_URL=postgresql://funkwhale@:5432/funkwhale
 
-# Generate one using `openssl rand -base64 45`, for example
-DJANGO_SECRET_KEY=
+# Cache configuration
+# Examples:
+#  CACHE_URL=redis://<host>:<port>/<database>
+#  CACHE_URL=redis://localhost:6379/0
+# Use the next one if you followed Debian installation guide
+# CACHE_URL=redis://127.0.0.1:6379/0
+
+# Where media files (such as album covers or audio tracks) should be stored
+# on your system?
+# (Ensure this directory actually exists)
+# MEDIA_ROOT=/srv/funkwhale/data/media
 
-# You don't have to edit this
-DJANGO_ADMIN_URL=^api/admin/
+# Where static files (such as API css or icons) should be compiled
+# on your system?
+# (Ensure this directory actually exists)
+# STATIC_ROOT=/srv/funkwhale/data/static
 
 # Update it to match the domain that will be used to reach your funkwhale
 # instance
 # Example: DJANGO_ALLOWED_HOSTS=funkwhale.yourdomain.com
 DJANGO_ALLOWED_HOSTS=yourdomain
 
+# which settings module should django use?
+# You don't have to touch this unless you really know what you're doing
+DJANGO_SETTINGS_MODULE=config.settings.production
+
+# Generate one using `openssl rand -base64 45`, for example
+DJANGO_SECRET_KEY=
+
+# You don't have to edit this, but you can put the admin on another URL if you
+# want to
+# DJANGO_ADMIN_URL=^api/admin/
+
 # If True, unauthenticated users won't be able to query the API
 API_AUTHENTICATION_REQUIRED=True
 

deploy/funkwhale-server.service

+[Unit]
+Description=Funkwhale application server
+After=redis.service postgresql.service
+PartOf=funkwhale.target
+
+[Service]
+User=funkwhale
+# adapt this depending on the path of your funkwhale installation
+WorkingDirectory=/srv/funkwhale/api
+EnvironmentFile=/srv/funkwhale/config/.env
+ExecStart=/srv/funkwhale/virtualenv/bin/gunicorn config.wsgi:application -b ${FUNKWHALE_API_IP}:${FUNKWHALE_API_PORT}
+
+[Install]
+WantedBy=multi-user.target

deploy/funkwhale-worker.service

+[Unit]
+Description=Funkwhale celery worker
+After=redis.service postgresql.service
+PartOf=funkwhale.target
+
+[Service]
+User=funkwhale
+# adapt this depending on the path of your funkwhale installation
+WorkingDirectory=/srv/funkwhale/api
+EnvironmentFile=/srv/funkwhale/config/.env
+ExecStart=/srv/funkwhale/virtualenv/bin/python manage.py celery worker
+
+[Install]
+WantedBy=multi-user.target

deploy/funkwhale.target

+[Unit]
+Description=Funkwhale
+Wants=funkwhale-server.service funkwhale-worker.service

deploy/nginx.conf

+# Ensure you update at least the server_name variables to match your own
+# domain
+
 upstream funkwhale-api {
     # depending on your setup, you may want to udpate this
     server localhost:5000;
 }
 
 server {
-  listen 80;
-  listen [::]:80;
-  server_name demo.funkwhale.audio;
-  # useful for Let's Encrypt
-  location /.well-known/acme-challenge/ { allow all; }
-  location / { return 301 https://$host$request_uri; }
+    listen 80;
+    listen [::]:80;
+    # update this to match your instance name
+    server_name demo.funkwhale.audio;
+    # useful for Let's Encrypt
+    location /.well-known/acme-challenge/ { allow all; }
+    location / { return 301 https://$host$request_uri; }
 }
 
 server {
     listen      443 ssl http2;
     listen [::]:443 ssl http2;
+    # update this to match your instance name
     server_name demo.funkwhale.audio;
 
     # TLS
+    # Feel free to use your own configuration for SSL here or simply remove the
+    # lines and move the configuration to the previous server block if you
+    # don't want to run funkwhale behind https (this is not recommanded)
+    # have a look here for let's encrypt configuration:
+    # https://certbot.eff.org/all-instructions/#debian-9-stretch-nginx
     ssl_protocols TLSv1.2;
     ssl_ciphers HIGH:!MEDIUM:!LOW:!aNULL:!NULL:!SHA;
     ssl_prefer_server_ciphers on;
     ssl_session_cache shared:SSL:10m;
-    ssl_certificate     /etc/letsencrypt/live/example.com/fullchain.pem;
-    ssl_certificate_key /etc/letsencrypt/live/example.com/privkey.pem;
+    ssl_certificate     /etc/letsencrypt/live/demo.funkwhale.audio/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/demo.funkwhale.audio/privkey.pem;
     # HSTS
     add_header Strict-Transport-Security "max-age=31536000";
 

dev.yml

@@ -34,6 +34,8 @@ services:
     command: python manage.py celery worker
     environment:
         - C_FORCE_ROOT=true
+        - "DATABASE_URL=postgresql://postgres@postgres/postgres"
+        - "CACHE_URL=redis://redis:6379/0"
     volumes:
       - ./api:/app
       - ./data/music:/music
@@ -46,12 +48,14 @@ services:
     volumes:
       - ./api:/app
       - ./data/music:/music
+    environment:
+      - "DATABASE_URL=postgresql://postgres@postgres/postgres"
+      - "CACHE_URL=redis://redis:6379/0"
     ports:
       - "12081:12081"
     links:
       - postgres
       - redis
-      - celeryworker
 
   nginx:
     env_file: .env.dev

docs/changelog.rst

@@ -21,9 +21,7 @@ Changelog
 * [feature] can now import artist and releases from youtube and musicbrainz.
   This requires a YouTube API key for the search
 * [breaking] we now check for user permission before serving audio files, which requires
-a specific configuration block in your reverse proxy configuration:
-
-.. code-block::
+  a specific configuration block in your reverse proxy configuration::
 
     location /_protected/media {
         internal;

docs/conf.py

@@ -17,10 +17,12 @@
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
-# import os
-# import sys
-# sys.path.insert(0, os.path.abspath('.'))
+import os
+import sys
 
+sys.path.insert(0, os.path.abspath('../api'))
+
+import funkwhale_api  # NOQA
 
 # -- General configuration ------------------------------------------------
 
@@ -55,9 +57,11 @@ author = 'Eliot Berriot'
 # built documents.
 #
 # The short X.Y version.
-version = '0.1'
+# version = funkwhale_api.__version__
+# @TODO use real version here
+version = 'feature/22-debian-installation'
 # The full version, including alpha/beta/rc tags.
-release = '0.1'
+release = version
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -152,6 +156,3 @@ texinfo_documents = [
      author, 'funkwhale', 'One line description of project.',
      'Miscellaneous'),
 ]
-
-
-

docs/importing-music.rst

@@ -5,10 +5,18 @@ From music directory on the server
 ----------------------------------
 
 You can import music files in funkwhale assuming they are located on the server
-and readable by the funkwhale application.
+and readable by the funkwhale application. Your music files should contain at
+least an ``artist``, ``album`` and ``title`` tags.
 
-Assuming your music is located at ``/music`` and your music files contains at
-least an ``artist``, ``album`` and ``title`` tag, you can import those tracks as follows:
+You can import those tracks as follows, assuming they are located in
+``/srv/funkwhale/data/music``:
+
+.. code-block:: bash
+
+    python api/manage.py import_files "/srv/funkwhale/data/music/**/*.ogg" --recursive --noinput
+
+When you use docker, the ``/srv/funkwhale/data/music`` is mounted from the host
+to the ``/music`` directory on the container:
 
 .. code-block:: bash
 
@@ -17,6 +25,7 @@ least an ``artist``, ``album`` and ``title`` tag, you can import those tracks as
 For the best results, we recommand tagging your music collection through
 `Picard <http://picard.musicbrainz.org/>`_ in order to have the best quality metadata.
 
+
 .. note::
 
     This command is idempotent, meaning you can run it multiple times on the same
@@ -26,6 +35,18 @@ For the best results, we recommand tagging your music collection through
 
     At the moment, only OGG/Vorbis and MP3 files with ID3 tags are supported
 
+.. note::
+
+    The --recursive flag will work only on Python 3.5+, which is the default
+    version When using Docker or Debian 9. If you use an older version of Python,
+    remove the --recursive flag and use more explicit import patterns instead::
+
+        # this will only import ogg files at the second level
+        "/srv/funkwhale/data/music/*/*.ogg"
+        # this will only import ogg files in the fiven directory
+        "/srv/funkwhale/data/music/System-of-a-down/*.ogg"
+
+
 
 Getting demo tracks
 ^^^^^^^^^^^^^^^^^^^
@@ -34,10 +55,10 @@ If you do not have any music on your server but still want to test the import
 process, you can call the following methods do download a few albums licenced
 under creative commons (courtesy of Jamendo):
 
-.. code-block:: bash
+.. parsed-literal::
 
-    curl -L -o download-tracks.sh "https://code.eliotberriot.com/funkwhale/funkwhale/raw/master/demo/download-tracks.sh"
-    curl -L -o music.txt "https://code.eliotberriot.com/funkwhale/funkwhale/raw/master/demo/music.txt"
+    curl -L -o download-tracks.sh "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/demo/download-tracks.sh"
+    curl -L -o music.txt "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/demo/music.txt"
     chmod +x download-tracks.sh
     ./download-tracks.sh music.txt