diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 684e3233a4c860470d21521299e4b40952bb84dc..ff6da8edf0b810772260bb53d1786fc12b0030a2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -63,12 +63,15 @@ review_docs: BUILD_PATH: "../public" before_script: - cd docs + - apt-get update + - apt-get install -y graphviz + - pip install sphinx + cache: key: "$CI_PROJECT_ID__sphinx" paths: - "$PIP_CACHE_DIR" script: - - pip install sphinx - ./build_docs.sh - mkdir -p /static/docs/$CI_BUILD_REF_SLUG - cp -r $CI_PROJECT_DIR/public/* /static/docs/$CI_BUILD_REF_SLUG @@ -205,8 +208,10 @@ pages: BUILD_PATH: "../public" before_script: - cd docs - script: + - apt-get update + - apt-get install -y graphviz - pip install sphinx + script: - ./build_docs.sh cache: key: "$CI_PROJECT_ID__sphinx" diff --git a/CONTRIBUTING b/CONTRIBUTING index 9fbda15372619b4c9c4a4b90701390e832e1dd1b..24a3a78ff7358050858e9ccdda0066b7a7b1c9a4 100644 --- a/CONTRIBUTING +++ b/CONTRIBUTING @@ -1,5 +1,5 @@ Contribute to Funkwhale development -================================== +=================================== First of all, thank you for your interest in the project! We really appreciate the fact that you're about to take some time to read this @@ -111,7 +111,7 @@ Create it like this:: Create docker network -^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^ Create the federation network:: diff --git a/changes/changelog.d/256.doc b/changes/changelog.d/256.doc new file mode 100644 index 0000000000000000000000000000000000000000..34608e4c3ffcb770aa106cda8c02db25dffc8b51 --- /dev/null +++ b/changes/changelog.d/256.doc @@ -0,0 +1 @@ +Added troubleshotting and technical overview documentation (#256) diff --git a/docs/Dockerfile b/docs/Dockerfile index 1067eb8be427bcb66cb4ad824993ef300ba04cb6..76147bb6d0f75d08ed9d26106425ab0175187c76 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -1,4 +1,5 @@ -FROM python:3.6-alpine +FROM python:3.6 +RUN apt-get update && apt-get install -y graphviz RUN pip install sphinx livereload WORKDIR /app/docs diff --git a/docs/architecture.rst b/docs/architecture.rst new file mode 100644 index 0000000000000000000000000000000000000000..ef52b3b38d693d2a59be857e24532f82f27fd998 --- /dev/null +++ b/docs/architecture.rst @@ -0,0 +1,118 @@ +Architecture +============ + +Funkwhale is made of several components, each of them fulfilling a specific mission: + +.. graphviz:: + + digraph { + node [shape=record]; + rankdir=TB + concentrate=true + user [group="frontend" label="User" fontsize="9"] + webui [group="frontend" label="Web interface (VueJS SPA)" fontsize="9"] + subapps [group="frontend" label="Subsonic-compatible apps (DSub, Clementine)" fontsize="9"] + proxy [label="Reverse proxy (Nginx/Apache)" fontsize="9"] + api [label="API Server (Django)" fontsize="9"] + db [label="Database (PostgreSQL)" fontsize="9"] + cache [label="Cache and message queue (Redis)" fontsize="9"] + worker [label="Worker (Celery)" fontsize="9"] + scheduler [label="Task scheduler (Celery Beat)" fontsize="9"] + + user -> subapps -> proxy + user -> webui -> proxy + cache -> worker + proxy -> api + api -> cache + api -> db + scheduler -> cache + worker -> cache + worker -> db + } + +This graph may looks a bit scary, so we'll detail the role of each component below. + +The user +-------- + +Funkwhale users can interact with your instance using: + +- The official web interface +- Third-party apps + +The web interface +----------------- + +This refers to Funkwhale's built-in web interface, which is a Single Page application +written in Vue JS. This application will interact with Funkwhale's API to retrieve +or persist data. + +Third-party apps +---------------- + +Since Funkwhale implements a subset of the Subsonic API, it's compatible with existing apps such +as DSub, Ultrasonic or Clementine that support this API. Those apps can be used as a replacement +or in conjunction of the web interface, but the underlying data is the same. + +The reverse proxy +----------------- + +Funkwhale's API server should never be exposed directly to the internet, as we require +a reverse proxy (Apache or Nginx) for performance and security reasons. The reverse proxy +will receive client HTTP requests, and: + +- Proxy them to the API server +- Serve requested static files (Audio files, stylesheets, javascript, fonts...) + +The API server +-------------- + +Funkwhale's API server is the central piece of the project. This component is responsible +for answering and processing user requests, manipulate data from the database, send long-running +tasks to workers, etc. + +It's a Python/Django application. + +The database +------------ + +Most of the data such as user accounts, favorites, music metadata or playlist is stored +in a PostgreSQL database. + +The cache/message queue +----------------------- + +Fetching data from the database is sometimes slow or resource hungry. To reduce +the load, Redis act as a cache for data that is considerably faster than a database. + +It is also a message queue that will deliver tasks to the worker. + +The worker +---------- + +Some operations are too long to live in the HTTP request/response cycle. Typically, +importing a bunch of uploaded tracks could take a minute or two. + +To keep the API response time as fast as possible, we offload long-running tasks +to a background process, also known as the Celery worker. + +Typical tasks include: + +- Handling music imports +- Handling federation/ActivityPub messages +- Scanning other instances libraries + +This worker is also able to retry failed tasks, or spawn automatically +more process when the number of received tasks increase. + +The scheduler +------------- + +Some long-running tasks are not triggered by user or external input, but on a recurring +basis instead. The scheduler is responsible for triggering those tasks and put the corresponding +messages in the message queue so the worker can process them. + +Recurring tasks include: + +- Cache cleaning +- Music metadata refreshing diff --git a/docs/conf.py b/docs/conf.py index 01da9bc05b473b280e53c6d29101a01b6027d6ab..8b1a3d41aab24e2d2e51cf594ed8be904ca46c26 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,7 +20,7 @@ import os import sys -sys.path.insert(0, os.path.abspath('../api')) +sys.path.insert(0, os.path.abspath("../api")) import funkwhale_api # NOQA @@ -33,24 +33,24 @@ import funkwhale_api # NOQA # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = [] +extensions = ["sphinx.ext.graphviz"] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = 'funkwhale' -copyright = '2017, Eliot Berriot' -author = 'Eliot Berriot' +project = "funkwhale" +copyright = "2017, Eliot Berriot" +author = "Eliot Berriot" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -71,10 +71,10 @@ language = None # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False @@ -85,7 +85,7 @@ todo_include_todos = False # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +html_theme = "alabaster" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -96,13 +96,13 @@ html_theme = 'alabaster' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # -- Options for HTMLHelp output ------------------------------------------ # Output file base name for HTML help builder. -htmlhelp_basename = 'funkwhaledoc' +htmlhelp_basename = "funkwhaledoc" # -- Options for LaTeX output --------------------------------------------- @@ -111,15 +111,12 @@ latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # # 'preamble': '', - # Latex figure (float) alignment # # 'figure_align': 'htbp', @@ -129,8 +126,7 @@ latex_elements = { # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'funkwhale.tex', 'funkwhale Documentation', - 'Eliot Berriot', 'manual'), + (master_doc, "funkwhale.tex", "funkwhale Documentation", "Eliot Berriot", "manual") ] @@ -138,10 +134,7 @@ latex_documents = [ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'funkwhale', 'funkwhale Documentation', - [author], 1) -] +man_pages = [(master_doc, "funkwhale", "funkwhale Documentation", [author], 1)] # -- Options for Texinfo output ------------------------------------------- @@ -150,7 +143,13 @@ man_pages = [ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'funkwhale', 'funkwhale Documentation', - author, 'funkwhale', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "funkwhale", + "funkwhale Documentation", + author, + "funkwhale", + "One line description of project.", + "Miscellaneous", + ) ] diff --git a/docs/configuration.rst b/docs/configuration.rst index 7b7751fc61ee9827fb2f1267060e114682f32907..6ec138e4cc61944f943d02e0db478b6b346a65f6 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -124,6 +124,16 @@ On non-docker setup, you don't need to configure this setting. .. note:: This path should not include any trailing slash +.. _setting-REVERSE_PROXY_TYPE: + +``REVERSE_PROXY_TYPE`` +^^^^^^^^^^^^^^^^^^^^^^ + +Default: ``nginx`` + +The type of reverse-proxy behind which Funkwhale is served. Either ``apache2`` +or ``nginx``. This is only used if you are using in-place import. + User permissions ---------------- diff --git a/docs/index.rst b/docs/index.rst index f177a42403895c07c300acb74cfec11aeb2b50d7..7d200da54643c3b04124669691f6c3e3cf30eecd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -13,9 +13,11 @@ Funkwhale is a self-hosted, modern free and open-source music server, heavily in users/index features + architecture installation/index upgrading configuration + troubleshooting importing-music federation api diff --git a/docs/troubleshooting.rst b/docs/troubleshooting.rst new file mode 100644 index 0000000000000000000000000000000000000000..61fa4c68aec28e9518ea7d3b57d478e529f7d27f --- /dev/null +++ b/docs/troubleshooting.rst @@ -0,0 +1,212 @@ +Troubleshooting +=============== + +Various errors and issues can arise on your Funkwhale instance, caused by configuration errors, +deployment/environment specific issues, or bugs in the software itself. + +On this document, you'll find: + +- Tools and commands you can use to better understand the issues +- A list of common pitfalls and errors and how to solve them +- A collection of links and advice to get help from the community and report new issues + +Diagnose problems +^^^^^^^^^^^^^^^^^ + +Funkwhale is made of several components, each one being a potential cause for failure. Having an even basic overview +of Funkwhale's technical architecture can help you understand what is going on. You can refer to :doc:`the technical architecture </architecture>` for that. + +Problems usually fall into one of those categories: + +- **Frontend**: Funkwhale's interface is not loading, not behaving as expected, music is not playing +- **API**: the interface do not display any data or show errors +- **Import**: uploaded/imported tracks are not imported correctly or at all +- **Federation**: you cannot contact other Funkwhale servers, access their library, play federated tracks +- **Everything else** + +Each category comes with its own set of diagnose tools and/or commands we will detail below. We'll also give you simple +steps for each type of problem. Please try those to see if it fix your issues. If none of those works, please report your issue on our +issue tracker. + +Frontend issues +^^^^^^^^^^^^^^^ + +Diagnostic tools: + +- Javascript and network logs from your browser console (see instructions on how to open it in `Chrome <https://developers.google.com/web/tools/chrome-devtools/console/>`_ and `Firefox <https://developer.mozilla.org/en-US/docs/Tools/Web_Console/Opening_the_Web_Console>`_ +- Proxy and API access and error logs (see :ref:`access-logs`) +- The same operation works from a different browser + +Common problems +*************** + +The front-end is completely blank +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You are visiting Funkwhale, but you don't see anything. + +- Try from a different browser +- Check network errors in your browser console. If you see responses with 40X or 50X statuses, there is probably an issue with the webserver configuration +- If you don't see anything wrong in the network console, check the Javascript console +- Disable your browser extensions (like adblockers) + +Music is not playing +~~~~~~~~~~~~~~~~~~~~ + +You have some tracks in your queue that don't play, or the queue is jumping from one track to the next until +there is no more track available: + +- Try with other tracks. If it works with some tracks but not other tracks, this may means that the failing tracks are not probably imported + or that your browser does not support a specific audio format +- Check the network and javascript console for potential errors + +Tracks are not appending to the queue +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When clicking on "Play", "Play all albums" or "Play all" buttons, some tracks are not appended to the queue. This is +actually a feature of Funkwhale: those tracks have no file associated with them, so we cannot play them. + +Specific pages are loading forever or blank +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When viewing a given page, the page load never ends (you continue to see the spinner), or nothing seems to appear at all: + +- Ensure your internet connection is up and running +- Ensure your instance is up and running +- Check the network and javascript console for potential errors + + +Backend issues +^^^^^^^^^^^^^^ + +Diagnostic tools: + +- Reverse proxy logs: + - Apache logs should be available at :file:`/var/log/apache/access.log` and :file:`/var/log/apache/error.log` + - Nginx logs should be available at :file:`/var/log/nginx/access.log` and :file:`/var/log/nginx/error.log` +- API logs: + - Docker setup: ``docker-compose logs -f --tail=50 api`` (remove the ``--tail`` flag to get the full logs) + - Non-docker setup: ``journalctl -xn -u funkwhale-server`` + +.. note:: + + If you edit your .env file to test a new configuration, you have to restart your services to pick up the changes: + + - Docker setup: ``docker-compose up -d`` + - Non-docker setup: ``systemctl restart funkwhale.target`` + +Common problems +*************** + +Instance work properly, but audio files are not served (404 error) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- If you're using docker, ensure the ``MEDIA_ROOT`` variable is commented in your env file +- Ensure the ``_protected/media`` block points toward the path where media files are stored (``/srv/funkwhale/data/media``, by default) +- If you're using in-place import, ensure :ref:`setting-MUSIC_DIRECTORY_PATH`, :ref:`setting-MUSIC_DIRECTORY_SERVE_PATH` and :ref:`setting-REVERSE_PROXY_TYPE` are configured properly, and that the files are readable by the webserver + +Weakref error when running ``python manage.py <command>`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +On Python <3.6, you may see this kind of errors when running commands like ``python manage.py migrate``:: + + Exception ignored in: <function WeakValueDictionary.__init__.<locals>.remove at 0x107e7a6a8> + Traceback (most recent call last): + File "/srv/funkwhale/venv/lib/python3.5/weakref.py", line 117, in remove + TypeError: 'NoneType' object is not callable + +This is caused by a bug in Python (cf https://github.com/celery/celery/issues/3818), and is not affecting in any way +the command you execute. You can safely ignore this error. + +``Your models have changes that are not yet reflected in a migration`` warning +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When running ``python manage.py migrate`` (both in docker or non-docker), you may end-up with this:: + + Operations to perform: + Apply all migrations: account, admin, auth, authtoken, common, contenttypes, dynamic_preferences, favorites, federation, history, music, playlists, radios, requests, sessions, sites, socialaccount, taggit, users + Running migrations: + No migrations to apply. + + Your models have changes that are not yet reflected in a migration, and so won't be applied. + Run 'manage.py makemigrations' to make new migrations, and then re-run 'manage.py migrate' to apply them. + +This warning can be safely ignored. You should not run the suggested ``manage.py makemigrations`` command. + +File import issues +^^^^^^^^^^^^^^^^^^ + +Unless you are using the CLI to import files, imports are send as tasks in a queue to a celery worker that will process them. + +Diagnostic tools: + +- Celery worker logs: + - Docker setup: ``docker-compose logs -f --tail=50 celeryworker`` (remove the ``--tail`` flag to get the full logs) + - Non-docker setup: ``journalctl -xn -u funkwhale-worker`` + +Federation issues +^^^^^^^^^^^^^^^^^ + +Received federations messages are sent to a dedicated task queue and processed asynchronously by a celery worker. + +Diagnostic tools: + +- API logs: + - Docker setup: ``docker-compose logs -f --tail=50 api`` (remove the ``--tail`` flag to get the full logs) + - Non-docker setup: ``journalctl -xn -u funkwhale-server`` +- Celery worker logs: + - Docker setup: ``docker-compose logs -f --tail=50 celeryworker`` (remove the ``--tail`` flag to get the full logs) + - Non-docker setup: ``journalctl -xn -u funkwhale-worker`` + +Common problems +*************** + +I have no access to another instance library +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Check if it works with the demo library (library@demo.funkwhale.audio) +- Check if the remote library received your follow request and approved it +- Trigger a scan via the interface +- Have a look in the celery logs for potential errors during the scan + +Other problems +^^^^^^^^^^^^^^ + +It's a bit hard to give targeted advice about problems that do not fit in the previous categories. However, we can recommend to: + +- Try to identify the scope of the issue and reproduce it reliably +- Ensure your instance is configured as detailed in the installation documentation, and if you did not use the default + values, to check what you changed +- To read the .env file carefuly, as most of the options are described in the comments + + +Report an issue or get help +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Well be more than happy to help you to debug installation and configuration issues. The main channel +for receiving support about your Funkwhale installation is the `#funkwhale-troubleshooting:matrix.org <https://riot.im/app/#/room/#funkwhale-troubleshooting:matrix.org>`_ Matrix channel. + +Before asking for help, we'd really appreciate if you took the time to go through this document and try to diagnose the problem yourself. But if you don't find +anything relevant or don't have the time, we'll be there for you! + +Here are a few recommendations on how to structure and what to include in your help requests: + +- Give us as much context as possible about your installation (OS, version, Docker/non-docker, reverse-proxy type, relevant logs and errors, etc.) +- Including screenshots or small gifs or videos can help us considerably when debugging front-end issues + +You can also open issues on our `issue tracker <https://code.eliotberriot.com/funkwhale/funkwhale/issues>`_. Please have a quick look for +similar issues before doing that, and use the issue tracker only to report bugs, suggest enhancements (both in the software and the documentation) or new features. + +.. warning:: + + If you ever need to share screenshots or urls with someone else, ensure those do not include your personnal token. + This token is binded to your account and can be used to connect and use your account. + + Urls that includes your token looks like: ``https://your.instance/api/v1/trackfiles/42/serve/?jwt=yoursecrettoken`` + +Improving this documentation +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you feel like something should be improved in this document (and in the documentation in general), feel free to `edit +it <https://code.eliotberriot.com/funkwhale/funkwhale/tree/develop/docs>`_ and open a Merge Request. If you lack time or skills +to do that, you can open an issue to discuss that, and someone else will do it. diff --git a/docs/upgrading.rst b/docs/upgrading.rst index 85a2b50577efbc5fc49659c0bb31ad870aecd656..437613b438247c347e1c0dc9d7ac0c0b25af25b3 100644 --- a/docs/upgrading.rst +++ b/docs/upgrading.rst @@ -72,7 +72,7 @@ Upgrading the API ^^^^^^^^^^^^^^^^^ On non-docker, upgrade involves a few more commands. We assume your setup -match what is described in :doc:`debian`: +match what is described in :doc:`/installation/debian`: .. parsed-literal::