Merge branch 'dockerized-nginx' into 'develop'

Fix #358: dockerized nginx to make deployment easier

Closes #358

See merge request funkwhale/funkwhale!389

changes/changelog.d/358.enhancement

+Simplified and less error-prone nginx setup (#358)
+
+
+Simplified nginx setup [Docker: Manual action required]
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+We've received a lot of user feedback regarding our installation process,
+and it seems the proxy part is the one which is the most confusing and difficult.
+Unfortunately, this is also the one where errors and mistakes can completely break
+the application.
+
+To make things easier for everyone, we now offer a simplified deployment
+process for the reverse proxy part. This will make upgrade of the proxy configuration
+significantly easier on docker deployments.
+
+On non-docker instances, you have nothing to do.
+
+If you have a dockerized instance, here is the upgrade path.
+
+First, tweak your .env file::
+
+    # remove the FUNKWHALE_URL variable
+    # and add the next variables
+    FUNKWHALE_HOSTNAME=yourdomain.funkwhale
+    FUNKWHALE_PROTOCOL=https
+
+    # add the following variable, matching the path your app is deployed
+    # leaving the default should work fine if you deployed using the same
+    # paths as the documentation
+    FUNKWHALE_FRONTEND_PATH=/srv/funkwhale/front/dist
+
+Then, add the following block at the end of your docker-compose.yml file::
+
+    # existing services
+    api:
+        ...
+    celeryworker:
+        ...
+
+    # new service
+    nginx:
+      image: nginx
+      env_file:
+        - .env
+      environment:
+        # Override those variables in your .env file if needed
+        - "NGINX_MAX_BODY_SIZE=${NGINX_MAX_BODY_SIZE-30M}"
+      volumes:
+        - "./nginx/funkwhale.template:/etc/nginx/conf.d/funkwhale.template:ro"
+        - "./nginx/funkwhale_proxy.conf:/etc/nginx/funkwhale_proxy.conf:ro"
+        - "${MUSIC_DIRECTORY_SERVE_PATH-/srv/funkwhale/data/music}:${MUSIC_DIRECTORY_SERVE_PATH-/srv/funkwhale/data/music}:ro"
+        - "${MEDIA_ROOT}:${MEDIA_ROOT}:ro"
+        - "${STATIC_ROOT}:${STATIC_ROOT}:ro"
+        - "${FUNKWHALE_FRONTEND_PATH}:/frontend:ro"
+      ports:
+        # override those variables in your .env file if needed
+        - "${FUNKWHALE_API_IP}:${FUNKWHALE_API_PORT}:80"
+      command: >
+          sh -c "envsubst \"`env | awk -F = '{printf \" $$%s\", $$1}'`\"
+          < /etc/nginx/conf.d/funkwhale.template
+          > /etc/nginx/conf.d/default.conf
+          && cat /etc/nginx/conf.d/default.conf
+          && nginx -g 'daemon off;'"
+      links:
+        - api
+
+By doing that, you'll enable a dockerized nginx that will automatically be
+configured to serve your Funkwhale instance.
+
+Download the required configuration files for the nginx container:
+
+.. parsed-literal::
+
+    cd /srv/funkwhale
+    mkdir nginx
+    curl -L -o nginx/funkwhale.template "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/deploy/docker.nginx.template"
+    curl -L -o nginx/funkwhale_proxy.conf "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/deploy/funkwhale_proxy.conf"
+
+Update the funkwhale.conf configuration of your server's reverse-proxy::
+
+    # the file should match something like that, upgrade all variables
+    # between ${} to match the ones in your .env file,
+    # and your SSL configuration if you're not using let's encrypt
+    # The important thing is that you only have a single location block
+    # that proxies everything to your dockerized nginx.
+
+    sudo nano /etc/nginx/sites-enabled/funkwhale.conf
+    upstream fw {
+        # depending on your setup, you may want to udpate this
+        server ${FUNKWHALE_API_IP}:${FUNKWHALE_API_PORT};
+    }
+    map $http_upgrade $connection_upgrade {
+        default upgrade;
+        ''      close;
+    }
+
+    server {
+        listen 80;
+        listen [::]:80;
+        server_name ${FUNKWHALE_HOSTNAME};
+        location / { return 301 https://$host$request_uri; }
+    }
+    server {
+        listen      443 ssl;
+        listen [::]:443 ssl;
+        server_name ${FUNKWHALE_HOSTNAME};
+
+        # TLS
+        ssl_protocols TLSv1.2;
+        ssl_ciphers HIGH:!MEDIUM:!LOW:!aNULL:!NULL:!SHA;
+        ssl_prefer_server_ciphers on;
+        ssl_session_cache shared:SSL:10m;
+        ssl_certificate     /etc/letsencrypt/live/${FUNKWHALE_HOSTNAME}/fullchain.pem;
+        ssl_certificate_key /etc/letsencrypt/live/${FUNKWHALE_HOSTNAME}/privkey.pem;
+
+        # HSTS
+        add_header Strict-Transport-Security "max-age=31536000";
+
+        location / {
+            include /etc/nginx/funkwhale_proxy.conf;
+            proxy_pass   http://fw/;
+        }
+    }
+
+Check that your configuration is valid then reload:
+
+    sudo nginx -t
+    sudo systemctl reload nginx

demo/setup.sh

@@ -12,6 +12,9 @@ mkdir -p $demo_path
 echo 'Downloading demo files...'
 curl -L -o docker-compose.yml "https://code.eliotberriot.com/funkwhale/funkwhale/raw/$version/deploy/docker-compose.yml"
 curl -L -o .env "https://code.eliotberriot.com/funkwhale/funkwhale/raw/$version/deploy/env.prod.sample"
+mkdir nginx
+curl -L -o nginx/funkwhale.template "https://code.eliotberriot.com/funkwhale/funkwhale/raw/$version/deploy/docker.nginx.template"
+curl -L -o nginx/funkwhale_proxy.conf "https://code.eliotberriot.com/funkwhale/funkwhale/raw/$version/deploy/funkwhale_proxy.conf"
 
 mkdir data/
 curl -L -o front.zip "https://code.eliotberriot.com/funkwhale/funkwhale/-/jobs/artifacts/$version/download?job=build_front"
@@ -23,6 +26,7 @@ echo "MUSIC_DIRECTORY_SERVE_PATH=$music_path" >> .env
 echo "MUSIC_DIRECTORY_PATH=$music_path" >> .env
 echo "MEDIA_ROOT=$demo_path/data/media/" >> .env
 echo "STATIC_ROOT=$demo_path/data/static/" >> .env
+echo "FUNKWHALE_FRONTEND_PATH=$demo_path/front/dist/" >> .env
 
 # /usr/local/bin/docker-compose pull
 /usr/local/bin/docker-compose up -d postgres redis

deploy/docker-compose.yml

-version: '3'
+version: "3"
 
 services:
-
   postgres:
     restart: unless-stopped
     env_file: .env
@@ -55,25 +54,35 @@ services:
       - "${MUSIC_DIRECTORY_SERVE_PATH-/srv/funkwhale/data/music}:${MUSIC_DIRECTORY_PATH-/music}:ro"
       - "${MEDIA_ROOT}:${MEDIA_ROOT}"
       - "${STATIC_ROOT}:${STATIC_ROOT}"
-      - ./front/dist:/frontend
+      - "${FUNKWHALE_FRONTEND_PATH}:/frontend"
     ports:
-      - "${FUNKWHALE_API_IP:-127.0.0.1}:${FUNKWHALE_API_PORT:-5000}:5000"
+      - "5000"
     links:
       - postgres
       - redis
 
-  # If you want to have the nginx proxy managed by docker for some reason
-  # (i.e. if you use apache as a proxy on your host),
-  # you can uncomment the following lines.
-  # nginx:
-  #   image: nginx
-  #   links:
-  #     - api
-  #   volumes:
-  #     - ./nginx.conf:/etc/nginx/conf.d/funkwhale.conf:ro
-  #     - ./funkwhale_proxy.conf:/etc/nginx/funkwhale_proxy.conf:ro
-  #     - ./data/media:/srv/funkwhale/data/media:ro
-  #     - ./front/dist:/srv/funkwhale/front/dist:ro
-  #     - ./data/static:/srv/funkwhale/data/static/:ro
-  #   ports:
-  #     - "127.0.0.1:5001:80"
+  nginx:
+    image: nginx
+    env_file:
+      - .env
+    environment:
+      # Override those variables in your .env file if needed
+      - "NGINX_MAX_BODY_SIZE=${NGINX_MAX_BODY_SIZE-30M}"
+    volumes:
+      - "./nginx/funkwhale.template:/etc/nginx/conf.d/funkwhale.template:ro"
+      - "./nginx/funkwhale_proxy.conf:/etc/nginx/funkwhale_proxy.conf:ro"
+      - "${MUSIC_DIRECTORY_SERVE_PATH-/srv/funkwhale/data/music}:${MUSIC_DIRECTORY_SERVE_PATH-/srv/funkwhale/data/music}:ro"
+      - "${MEDIA_ROOT}:${MEDIA_ROOT}:ro"
+      - "${STATIC_ROOT}:${STATIC_ROOT}:ro"
+      - "${FUNKWHALE_FRONTEND_PATH}:/frontend:ro"
+    ports:
+      # override those variables in your .env file if needed
+      - "${FUNKWHALE_API_IP}:${FUNKWHALE_API_PORT}:80"
+    command: >
+        sh -c "envsubst \"`env | awk -F = '{printf \" $$%s\", $$1}'`\"
+        < /etc/nginx/conf.d/funkwhale.template
+        > /etc/nginx/conf.d/default.conf
+        && cat /etc/nginx/conf.d/default.conf
+        && nginx -g 'daemon off;'"
+    links:
+      - api

deploy/docker.funkwhale_proxy.conf

+# use this one if you put the nginx container behind another proxy
+# you will have to set some headers on this proxy as well to ensure
+# everything works correctly, you can use the ones from the funkwhale_proxy.conf file
+# at https://code.eliotberriot.com/funkwhale/funkwhale/blob/develop/deploy/funkwhale_proxy.conf
+# your proxy will also need to support websockets
+
+real_ip_header X-Forwarded-For;
+set_real_ip_from 172.17.0.0/16;
+
+proxy_set_header Host $http_x_forwarded_host;
+proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
+proxy_set_header X-Forwarded-Host $http_x_forwarded_host;
+proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
+proxy_redirect off;
+
+# websocket support
+proxy_http_version 1.1;
+proxy_set_header Upgrade $http_upgrade;
+proxy_set_header Connection $connection_upgrade;

deploy/docker.nginx.template

+upstream funkwhale-api {
+    # depending on your setup, you may want to udpate this
+    server api:5000;
+}
+
+
+# required for websocket support
+map $http_upgrade $connection_upgrade {
+    default upgrade;
+    ''      close;
+}
+
+server {
+    listen 80;
+    server_name ${FUNKWHALE_HOSTNAME};
+
+    # TLS
+    # Feel free to use your own configuration for SSL here or simply remove the
+    # lines and move the configuration to the previous server block if you
+    # don't want to run funkwhale behind https (this is not recommended)
+    # have a look here for let's encrypt configuration:
+    # https://certbot.eff.org/all-instructions/#debian-9-stretch-nginx
+
+    root /frontend;
+
+    location / {
+        try_files $uri $uri/ @rewrites;
+    }
+
+    location @rewrites {
+        rewrite ^(.+)$ /index.html last;
+    }
+    location /api/ {
+        include /etc/nginx/funkwhale_proxy.conf;
+        # this is needed if you have file import via upload enabled
+        client_max_body_size ${NGINX_MAX_BODY_SIZE};
+        proxy_pass   http://funkwhale-api/api/;
+    }
+
+    location /federation/ {
+        include /etc/nginx/funkwhale_proxy.conf;
+        proxy_pass   http://funkwhale-api/federation/;
+    }
+
+    # You can comment this if you do not plan to use the Subsonic API
+    location /rest/ {
+        include /etc/nginx/funkwhale_proxy.conf;
+        proxy_pass   http://funkwhale-api/api/subsonic/rest/;
+    }
+
+    location /.well-known/ {
+        include /etc/nginx/funkwhale_proxy.conf;
+        proxy_pass   http://funkwhale-api/.well-known/;
+    }
+
+    location /media/ {
+        alias ${MEDIA_ROOT}/;
+    }
+
+    location /_protected/media {
+        # this is an internal location that is used to serve
+        # audio files once correct permission / authentication
+        # has been checked on API side
+        internal;
+        alias   ${MEDIA_ROOT};
+    }
+
+    location /_protected/music {
+        # this is an internal location that is used to serve
+        # audio files once correct permission / authentication
+        # has been checked on API side
+        # Set this to the same value as your MUSIC_DIRECTORY_PATH setting
+        internal;
+        alias   ${MUSIC_DIRECTORY_PATH};
+    }
+
+    location /staticfiles/ {
+        # django static files
+        alias ${STATIC_ROOT}/;
+    }
+}

deploy/docker.proxy.template

+upstream fw {
+    # depending on your setup, you may want to udpate this
+    server ${FUNKWHALE_API_IP}:${FUNKWHALE_API_PORT};
+}
+map $http_upgrade $connection_upgrade {
+    default upgrade;
+    ''      close;
+}
+
+server {
+    listen 80;
+    listen [::]:80;
+    server_name ${FUNKWHALE_HOSTNAME};
+    location / { return 301 https://$host$request_uri; }
+}
+server {
+    listen      443 ssl;
+    listen [::]:443 ssl;
+    server_name ${FUNKWHALE_HOSTNAME};
+
+    # TLS
+    ssl_protocols TLSv1.2;
+    ssl_ciphers HIGH:!MEDIUM:!LOW:!aNULL:!NULL:!SHA;
+    ssl_prefer_server_ciphers on;
+    ssl_session_cache shared:SSL:10m;
+    ssl_certificate     /etc/letsencrypt/live/${FUNKWHALE_HOSTNAME}/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/${FUNKWHALE_HOSTNAME}/privkey.pem;
+
+    # HSTS
+    add_header Strict-Transport-Security "max-age=31536000";
+
+    location / {
+        include /etc/nginx/funkwhale_proxy.conf;
+        proxy_pass   http://fw/;
+    }
+}

deploy/env.prod.sample

@@ -5,7 +5,7 @@
 # following variables:
 # - DJANGO_SECRET_KEY
 # - DJANGO_ALLOWED_HOSTS
-# - FUNKWHALE_URL
+# - FUNKWHALE_HOSTNAME
 # - EMAIL_CONFIG and DEFAULT_FROM_EMAIL if you plan to send emails)
 # On non-docker setup **only**, you'll also have to tweak/uncomment those variables:
 # - DATABASE_URL
@@ -38,7 +38,8 @@ FUNKWHALE_API_PORT=5000
 
 # Replace this by the definitive, public domain you will use for
 # your instance
-FUNKWHALE_URL=https://yourdomain.funwhale
+FUNKWHALE_HOSTNAME=yourdomain.funkwhale
+FUNKWHALE_PROTOCOL=https
 
 # Configure email sending using this variale
 # By default, funkwhale will output emails sent to stdout
@@ -117,6 +118,8 @@ RAVEN_DSN=https://44332e9fdd3d42879c7d35bf8562c6a4:0062dc16a22b41679cd5765e5342f
 #   MUSIC_DIRECTORY_PATH=/srv/funkwhale/data/music
 #   # MUSIC_DIRECTORY_SERVE_PATH= # stays commented, not needed
 
+MUSIC_DIRECTORY_PATH=/srv/funkwhale/data/music
+MUSIC_DIRECTORY_SERVE_PATH=/srv/funkwhale/data/music
 
 # LDAP settings
 # Use the following options to allow authentication on your Funkwhale instance
@@ -131,3 +134,8 @@ RAVEN_DSN=https://44332e9fdd3d42879c7d35bf8562c6a4:0062dc16a22b41679cd5765e5342f
 # LDAP_SEARCH_FILTER=(|(cn={0})(mail={0}))
 # LDAP_START_TLS=False
 # LDAP_ROOT_DN=dc=domain,dc=com
+
+FUNKWHALE_FRONTEND_PATH=/srv/funkwhale/front/dist
+
+# Nginx related configuration
+NGINX_MAX_BODY_SIZE=30M

deploy/nginx.conf → deploy/nginx.template

-# Ensure you update at least the server_name variables to match your own
+# This file was generated from funkwhale.template
 
-# domain
 upstream funkwhale-api {
     # depending on your setup, you may want to udpate this
-    server localhost:5000;
+    server ${FUNKWHALE_API_IP}:${FUNKWHALE_API_PORT};
 }
 
 server {
     listen 80;
     listen [::]:80;
     # update this to match your instance name
-    server_name demo.funkwhale.audio;
+    server_name ${FUNKWHALE_HOSTNAME};
     # useful for Let's Encrypt
     location /.well-known/acme-challenge/ { allow all; }
     location / { return 301 https://$host$request_uri; }
@@ -25,8 +24,7 @@ map $http_upgrade $connection_upgrade {
 server {
     listen      443 ssl http2;
     listen [::]:443 ssl http2;
-    # update this to match your instance name
-    server_name demo.funkwhale.audio;
+    server_name ${FUNKWHALE_HOSTNAME};
 
     # TLS
     # Feel free to use your own configuration for SSL here or simply remove the
@@ -38,12 +36,12 @@ server {
     ssl_ciphers HIGH:!MEDIUM:!LOW:!aNULL:!NULL:!SHA;
     ssl_prefer_server_ciphers on;
     ssl_session_cache shared:SSL:10m;
-    ssl_certificate     /etc/letsencrypt/live/demo.funkwhale.audio/fullchain.pem;
-    ssl_certificate_key /etc/letsencrypt/live/demo.funkwhale.audio/privkey.pem;
+    ssl_certificate     /etc/letsencrypt/live/${FUNKWHALE_HOSTNAME}/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/${FUNKWHALE_HOSTNAME}/privkey.pem;
     # HSTS
     add_header Strict-Transport-Security "max-age=31536000";
 
-    root /srv/funkwhale/front/dist;
+    root ${FUNKWHALE_FRONTEND_PATH};
 
     location / {
         try_files $uri $uri/ @rewrites;
@@ -55,7 +53,7 @@ server {
     location /api/ {
         include /etc/nginx/funkwhale_proxy.conf;
         # this is needed if you have file import via upload enabled
-        client_max_body_size 30M;
+        client_max_body_size ${NGINX_MAX_BODY_SIZE};
         proxy_pass   http://funkwhale-api/api/;
     }
 
@@ -76,7 +74,7 @@ server {
     }
 
     location /media/ {
-        alias /srv/funkwhale/data/media/;
+        alias ${MEDIA_ROOT}/;
     }
 
     location /_protected/media {
@@ -84,7 +82,7 @@ server {
         # audio files once correct permission / authentication
         # has been checked on API side
         internal;
-        alias   /srv/funkwhale/data/media;
+        alias   ${MEDIA_ROOT};
     }
 
     location /_protected/music {
@@ -93,11 +91,11 @@ server {
         # has been checked on API side
         # Set this to the same value as your MUSIC_DIRECTORY_PATH setting
         internal;
-        alias   /srv/funkwhale/data/music;
+        alias   ${MUSIC_DIRECTORY_SERVE_PATH};
     }
 
     location /staticfiles/ {
         # django static files
-        alias /srv/funkwhale/data/static/;
+        alias ${STATIC_ROOT}/;
     }
 }

docs/installation/docker.rst

@@ -9,8 +9,10 @@ Download the sample docker-compose file:
 
 .. parsed-literal::
 
-    mkdir -p /srv/funkwhale
     cd /srv/funkwhale
+    mkdir nginx
+    curl -L -o nginx/funkwhale.template "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/deploy/docker.nginx.template"
+    curl -L -o nginx/funkwhale_proxy.conf "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/deploy/funkwhale_proxy.conf"
     curl -L -o docker-compose.yml "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/deploy/docker-compose.yml"
 
 Create your env file:

docs/installation/index.rst

@@ -114,16 +114,57 @@ On Arch Linux and its derivatives:
 
     sudo pacman -S nginx
 
-Then, download our sample virtualhost file and proxy conf:
+To avoid configuration errors at this level, we will generate an nginx configuration
+using your .env file. This will ensure your reverse-proxy configuration always
+match the application configuration and make upgrade/maintenance easier.
+
+On docker deployments, run the following commands::
 
 .. parsed-literal::
 
-    curl -L -o /etc/nginx/funkwhale_proxy.conf "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/deploy/funkwhale_proxy.conf"
-    curl -L -o /etc/nginx/sites-available/funkwhale.conf "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/deploy/nginx.conf"
+    # download the needed files
+    curl -L -o /etc/nginx/funkwhale_proxy.conf "https://code.eliotberriot.com/funkwhale/funkwhale/raw/develop/deploy/funkwhale_proxy.conf"
+    curl -L -o /etc/nginx/sites-available/funkwhale.template "https://code.eliotberriot.com/funkwhale/funkwhale/raw/develop/deploy/docker.proxy.template"
+
+    # create a final nginx configuration using the template based on your environment
+    set -a && source /srv/funkwhale/.env && set +a
+    envsubst "`env | awk -F = '{printf \" $%s\", $$1}'`" \
+        < /etc/nginx/sites-available/funkwhale.template \
+        > /etc/nginx/sites-available/funkwhale.conf
+
+On non-docker deployments, run the following commands::
+
+.. parsed-literal::
+
+    # download the needed files
+    curl -L -o /etc/nginx/funkwhale_proxy.conf "https://code.eliotberriot.com/funkwhale/funkwhale/raw/develop/deploy/funkwhale_proxy.conf"
+    curl -L -o /etc/nginx/sites-available/funkwhale.template "https://code.eliotberriot.com/funkwhale/funkwhale/raw/develop/deploy/docker.proxy.template"
+
+    # create a final nginx configuration using the template based on your environment
+    set -a && source /srv/funkwhale/config/.env && set +a
+    envsubst "`env | awk -F = '{printf \" $%s\", $$1}'`" \
+        < /etc/nginx/sites-available/funkwhale.template \
+        > /etc/nginx/sites-available/funkwhale.conf
+
+.. note::
+
+    The resulting file should not contain any variable such as ``${FUNKWHALE_HOSTNAME}``.
+    You can check that using this command::
+
+        grep '${' /etc/nginx/sites-available/funkwhale.conf
+
+.. note::
+
+    You can freely adapt the resulting file to your own needs, as we cannot
+    cover every use case with a single template, especially when it's related
+    to SSL configuration.
+
+Finally, enable the resulting configuration:
+
+.. code-block:: bash
     ln -s /etc/nginx/sites-available/funkwhale.conf /etc/nginx/sites-enabled/
 
-Ensure static assets and proxy pass match your configuration, and check the configuration is valid with ``nginx -t``.
-If everything is fine, you can restart your nginx server with ``service nginx restart``.
+Check the configuration is valid with ``nginx -t`` then reload your nginx server with ``systemctl restart nginx``.
 
 .. warning::
 

docs/upgrading.rst

@@ -30,6 +30,11 @@ easy:
     # hardcode the targeted version your env file
     # (look for the FUNKWHALE_VERSION variable)
     nano .env
+    # Load your environment variables
+    source .env
+    # Download newest nginx configuration file
+    curl -L -o nginx/funkwhale.template "https://code.eliotberriot.com/funkwhale/funkwhale/raw/develop/deploy/docker.nginx.template"
+    curl -L -o nginx/funkwhale_proxy.conf "https://code.eliotberriot.com/funkwhale/funkwhale/raw/develop/deploy/funkwhale_proxy.conf"
     # Pull the new version containers
     docker-compose pull
     # Apply the database migrations