Merge branch 'release/0.18'

.env.dev

@@ -12,3 +12,9 @@ MUSIC_DIRECTORY_PATH=/music
 BROWSABLE_API_ENABLED=True
 FORWARDED_PROTO=http
 LDAP_ENABLED=False
+
+# Uncomment this if you're using traefik/https
+# FORCE_HTTPS_URLS=True
+
+# Customize to your needs
+POSTGRES_VERSION=11

.gitignore

@@ -93,4 +93,6 @@ po/*.po
 docs/swagger
 _build
 front/src/translations.json
+front/src/translations/*.json
 front/locales/en_US/LC_MESSAGES/app.po
+*.prof

.gitlab-ci.yml

 variables:
   IMAGE_NAME: funkwhale/funkwhale
   IMAGE: $IMAGE_NAME:$CI_COMMIT_REF_NAME
-  IMAGE_LATEST: $IMAGE_NAME:latest
+  ALL_IN_ONE_IMAGE_NAME: funkwhale/all-in-one
+  ALL_IN_ONE_IMAGE: $ALL_IN_ONE_IMAGE_NAME:$CI_COMMIT_REF_NAME
   PIP_CACHE_DIR: "$CI_PROJECT_DIR/pip-cache"
   PYTHONDONTWRITEBYTECODE: "true"
   REVIEW_DOMAIN: preview.funkwhale.audio
@@ -131,16 +132,15 @@ flake8:
 
 test_api:
   services:
-    - postgres:9.4
+    - postgres:11
     - redis:3
   stage: test
-  image: funkwhale/funkwhale:latest
+  image: funkwhale/funkwhale:develop
   cache:
     key: "$CI_PROJECT_ID__pip_cache"
     paths:
       - "$PIP_CACHE_DIR"
   variables:
-    DJANGO_ALLOWED_HOSTS: "localhost"
     DATABASE_URL: "postgresql://postgres@postgres/postgres"
     FUNKWHALE_URL: "https://funkwhale.ci"
     DJANGO_SETTINGS_MODULE: config.settings.local
@@ -148,11 +148,10 @@ test_api:
     - branches
   before_script:
     - cd api
-    - apt-get update
-    - grep "^[^#;]" requirements.apt | grep -Fv "python3-dev" | xargs apt-get install -y --no-install-recommends
-    - pip install -r requirements/base.txt
-    - pip install -r requirements/local.txt
-    - pip install -r requirements/test.txt
+    - sed -i '/Pillow/d' requirements/base.txt
+    - pip3 install -r requirements/base.txt
+    - pip3 install -r requirements/local.txt
+    - pip3 install -r requirements/test.txt
   script:
     - pytest --cov=funkwhale_api tests/
   tags:
@@ -166,7 +165,7 @@ test_front:
   only:
     - branches
   script:
-    - yarn install
+    - yarn install --check-files
     - yarn test:unit
   cache:
     key: "funkwhale__front_dependencies"
@@ -194,11 +193,6 @@ build_front:
     # cf https://dev.funkwhale.audio/funkwhale/funkwhale/issues/169
     - yarn build | tee /dev/stderr | (! grep -i 'ERROR in')
     - chmod -R 755 dist
-  cache:
-    key: "funkwhale__front_dependencies"
-    paths:
-      - front/node_modules
-      - front/yarn.lock
   artifacts:
     name: "front_${CI_COMMIT_REF_NAME}"
     paths:
@@ -207,6 +201,7 @@ build_front:
     - tags@funkwhale/funkwhale
     - master@funkwhale/funkwhale
     - develop@funkwhale/funkwhale
+
   tags:
     - docker
 
@@ -236,9 +231,11 @@ pages:
 
 docker_release:
   stage: deploy
+  image: bash
   before_script:
     - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
     - cp -r front/dist api/frontend
+    - (if [ "$CI_COMMIT_REF_NAME" == "develop" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
     - cd api
   script:
     - docker build -t $IMAGE .
@@ -249,15 +246,42 @@ docker_release:
   tags:
     - docker-build
 
+docker_all_in_one_release:
+  stage: deploy
+  image: bash
+  variables:
+    ALL_IN_ONE_REF: master
+    ALL_IN_ONE_ARTIFACT_URL: https://github.com/thetarkus/docker-funkwhale/archive/$ALL_IN_ONE_REF.zip
+    BUILD_PATH: all_in_one
+  before_script:
+    - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
+    - (if [ "$CI_COMMIT_REF_NAME" == "develop" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
+  script:
+    - wget $ALL_IN_ONE_ARTIFACT_URL -O all_in_one.zip
+    - unzip -o all_in_one.zip -d tmpdir
+    - mv tmpdir/docker-funkwhale-$ALL_IN_ONE_REF $BUILD_PATH && rmdir tmpdir
+    - cp -r api $BUILD_PATH/src/api
+    - cp -r front $BUILD_PATH/src/front
+    - cd $BUILD_PATH
+    - ./scripts/download-nginx-template.sh src/ $CI_COMMIT_REF_NAME
+    - docker build -t $ALL_IN_ONE_IMAGE .
+    - docker push $ALL_IN_ONE_IMAGE
+  only:
+    - develop@funkwhale/funkwhale
+    - tags@funkwhale/funkwhale
+  tags:
+    - docker-build
+
 build_api:
   # Simply publish a zip containing api/ directory
   stage: deploy
-  image: busybox
+  image: bash
   artifacts:
     name: "api_${CI_COMMIT_REF_NAME}"
     paths:
       - api
   script:
+    - (if [ "$CI_COMMIT_REF_NAME" == "develop" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
     - chmod -R 750 api
     - echo Done!
   only:

.gitlab/issue_templates/Bug.md

@@ -34,6 +34,12 @@ Describe the expected behaviour.
 
 ## Context
 
+<!--
+The version of your instance can be found on the footer : Source code (x.y)
+-->
+
+**Funkwhale version(s) affected**: x.y
+
 <!--
 If relevant, share additional context here like:
 

CHANGELOG

@@ -10,6 +10,297 @@ This changelog is viewable on the web at https://docs.funkwhale.audio/changelog.
 
 .. towncrier
 
+0.18 "Naomi" (2019-01-22)
+-------------------------
+
+This release is dedicated to Naomi, an early contributor and beta tester of Funkwhale.
+Her positivity, love and support have been incredibly helpful and helped shape the project
+as you can enjoy it today. Thank you so much Naomi <3
+
+Upgrade instructions are available at
+https://docs.funkwhale.audio/index.html, ensure you also execute the intructions
+marked with ``[manual action required]`` and ``[manual action suggested]``.
+
+See ``Full changelog`` below for an exhaustive list of changes!
+
+Audio transcoding is back!
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+After removal of our first, buggy transcoding implementation, we're proud to announce
+that this feature is back. It is enabled by default, and can be configured/disabled
+in your instance settings!
+
+This feature works in the browser, with federated/non-federated tracks and using Subsonic clients.
+Transcoded tracks are generated on the fly, and cached for a configurable amount of time,
+to reduce the load on the server.
+
+
+Licensing and copyright information
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Funkwhale is now able to parse copyright and license data from file and store
+this information. Apart from displaying it on each track detail page,
+no additional behaviour is currently implemented to use this new data, but this
+will change in future releases.
+
+License and copyright data is also broadcasted over federation.
+
+License matching is done on the content of the ``License`` tag in the files,
+with a fallback on the ``Copyright`` tag.
+
+Funkwhale will successfully extract licensing data for the following licenses:
+
+- Creative Commons 0 (Public Domain)
+- Creative Commons 1.0 (All declinations)
+- Creative Commons 2.0 (All declinations)
+- Creative Commons 2.5 (All declinations and countries)
+- Creative Commons 3.0 (All declinations and countries)
+- Creative Commons 4.0 (All declinations)
+
+Support for other licenses such as Art Libre or WTFPL will be added in future releases.
+
+
+Instance-level moderation tools
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This release includes a first set of moderation tools that will give more control
+to admins about the way their instance federate with other instance and accounts on the network.
+Using these tools, it's now possible to:
+
+- Browse known accounts and domains, and associated data (storage size, software version, etc.)
+- Purge data belonging to given accounts and domains
+- Block or partially restrict interactions with any account or domain
+
+All those features are usable using a brand new "moderation" permission, meaning
+you can appoints one or nultiple moderators to help with this task.
+
+I'd like to thank all Mastodon contributors, because some of the these tools are heavily
+inspired from what's being done in Mastodon. Thank you so much!
+
+
+Iframe widget to embed public tracks and albums [manual action required]
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Funkwhale now support embedding a lightweight audio player on external websites
+for album and tracks that are available in public libraries. Important pages,
+such as artist, album and track pages also include OpenGraph tags that will
+enable previews on compatible apps (like sharing a Funkwhale track link on Mastodon
+or Twitter).
+
+To achieve that, we had to tweak the way Funkwhale front-end is served. You'll have
+to modify your nginx configuration when upgrading to keep your instance working.
+
+**On docker setups**, edit your ``/srv/funkwhale/nginx/funkwhale.template`` and replace
+the ``location /api/`` and `location /` blocks by the following snippets::
+
+    location / {
+        include /etc/nginx/funkwhale_proxy.conf;
+        # this is needed if you have file import via upload enabled
+        client_max_body_size ${NGINX_MAX_BODY_SIZE};
+        proxy_pass   http://funkwhale-api/;
+    }
+
+    location /front/ {
+        alias /frontend/;
+    }
+
+The change of configuration will be picked when restarting your nginx container.
+
+**On non-docker setups**, edit your ``/etc/nginx/sites-available/funkwhale.conf`` file,
+and replace the ``location /api/`` and `location /` blocks by the following snippets::
+
+
+    location / {
+        include /etc/nginx/funkwhale_proxy.conf;
+        # this is needed if you have file import via upload enabled
+        client_max_body_size ${NGINX_MAX_BODY_SIZE};
+        proxy_pass   http://funkwhale-api/;
+    }
+
+    location /front/ {
+        alias ${FUNKWHALE_FRONTEND_PATH}/;
+    }
+
+Replace ``${FUNKWHALE_FRONTEND_PATH}`` by the corresponding variable from your .env file,
+which should be ``/srv/funkwhale/front/dist`` by default, then reload your nginx process with
+``sudo systemctl reload nginx``.
+
+
+Alternative docker deployment method
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Thanks to the awesome done by @thetarkus at https://github.com/thetarkus/docker-funkwhale,
+we're now able to provide an alternative and easier Docker deployment method!
+
+In contrast with our current, multi-container offer, this method integrates
+all Funkwhale processes and services (database, redis, etc.) into a single, easier to deploy container.
+
+Both method will coexist in parallel, as each one has pros and cons. You can learn more
+about this exciting new deployment option by visiting https://docs.funkwhale.audio/installation/docker.html!
+
+Automatically load .env file
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+On non-docker deployments, earlier versions required you to source
+the config/.env file before launching any Funkwhale command, with ``export $(cat config/.env | grep -v ^# | xargs)``
+This led to more complex and error prode deployment / setup.
+
+This is not the case anymore, and Funkwhale will automatically load this file if it's available.
+
+
+Delete pre 0.17 federated tracks [manual action suggested]
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you were using Funkwhale before the 0.17 release and federated with other instances,
+it's possible that you still have some unplayable federated files in the database.
+
+To purge the database of those entries, you can run the following command:
+
+On docker setups::
+
+    docker-compose run --rm api python manage.py script delete_pre_017_federated_uploads --no-input
+
+On non-docker setups::
+
+    python manage.py script delete_pre_017_federated_uploads --no-input
+
+
+Enable gzip compression [manual action suggested]
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Gzip compression will be enabled on new instances by default
+and will reduce the amount of bandwidth consumed by your instance.
+
+If you with to benefit from gzip compression on your instance,
+edit your reverse proxy virtualhost file (located at ``/etc/nginx/sites-available/funkwhale.conf``) and add the following snippet
+in the server block, then reload your nginx server::
+
+    server {
+        # ... exiting configuration
+
+        # compression settings
+        gzip on;
+        gzip_comp_level    5;
+        gzip_min_length    256;
+        gzip_proxied       any;
+        gzip_vary          on;
+
+        gzip_types
+            application/atom+xml
+            application/javascript
+            application/json
+            application/ld+json
+            application/activity+json
+            application/manifest+json
+            application/rss+xml
+            application/vnd.geo+json
+            application/vnd.ms-fontobject
+            application/x-font-ttf
+            application/x-web-app-manifest+json
+            application/xhtml+xml
+            application/xml
+            font/opentype
+            image/bmp
+            image/svg+xml
+            image/x-icon
+            text/cache-manifest
+            text/css
+            text/plain
+            text/vcard
+            text/vnd.rim.location.xloc
+            text/vtt
+            text/x-component
+            text/x-cross-domain-policy;
+        # end of compression settings
+    }
+
+Full changelog
+^^^^^^^^^^^^^^
+
+Features:
+
+- Allow embedding of albums and tracks available in public libraries via an <iframe> (#578)
+- Audio transcoding is back! (#272)
+- First set of instance level moderation tools (#580, !521)
+- Store licensing and copyright information from file metadata, if available (#308)
+
+
+Enhancements:
+
+- Add UI elements for multi-disc albums (#631)
+- Added alternative funkwhale/all-in-one docker image (#614)
+- Broadcast library updates (name, description, visibility) over federation
+- Based Docker image on alpine to have a smaller (and faster to build) image
+- Improved front-end performance by stripping unused dependencies, reducing bundle size
+  and enabling gzip compression
+- Improved accessibility by using main/section/nav tags and aria-labels in most critical places (#612)
+- The progress bar in the player now display loading state / buffer loading (#586)
+- Added "type: funkwhale" and "funkwhale-version" in Subsonic responses (#573)
+- Documented keyboard shortcuts, list is now available by pressing "h" or in the footer (#611)
+- Documented which Subsonic endpoints are implemented (#575)
+- Hide invitation code field during signup when it's not required (#410)
+- Importer will now pick embedded images in files with OTHER type if no COVER_FRONT is present
+- Improved keyboard accessibility on player, queue and various controls (#576)
+- Improved performance when listing playable tracks, albums and artists
+- Increased default upload limit from 30 to 100MB (#654)
+- Load env file in config/.env automatically to avoid sourcing it by hand (#626)
+- More resilient date parsing during audio import, will not crash anymore on
+  invalid dates (#622)
+- Now start radios immediatly, skipping any existing tracks in queue (#585)
+- Officially support connecting to a password protected redis server, with
+  the redis://:password@localhost:6379/0 scheme (#640)
+- Performance improvement when fetching favorites, down to a single, small http request
+- Removed "Activity" page, since all the data is available on the "Browse" page (#600)
+- Removed the need to specify the DJANGO_ALLOWED_HOSTS variable
+- Restructured the footer, added useful links and removed unused content
+- Show short entries first in search results to improve UX
+- Store disc number and order tracks by disc number / position) (#507)
+- Strip EXIF metadata from uploaded avatars to avoid leaking private data (#374)
+- Support blind key rotation in HTTP Signatures (#658)
+- Support setting a server URL in settings.json (#650)
+- Updated default docker postgres version from 9.4 to 11 (#656)
+- Updated lots of dependencies (especially django 2.0->2.1), and removed unused dependencies (#657)
+- Improved test suite speed by reducing / disabling expensive operations (#648)
+
+
+Bugfixes:
+
+- Fixed parsing of embedded file cover for ogg files tagged with MusicBrainz (#469)
+- Upgraded core dependencies to fix websocket/messaging issues and possible memory leaks (#643)
+- Fix ".None" extension when downloading Flac file (#473)
+- Fixed None extension when downloading an in-place imported file (#621)
+- Added a script to prune pre 0.17 federated tracks (#564)
+- Advertise public libraries properly in ActivityPub representations (#553)
+- Allow opus file upload (#598)
+- Do not display "view on MusicBrainz" button if we miss the mbid (#422)
+- Do not try to create unaccent extension if it's already present (#663)
+- Ensure admin links in sidebar are displayed for users with relavant permissions, and only them (#597)
+- Fix broken websocket connexion under Chrome browser (#589)
+- Fix play button not starting playback with empty queue (#632)
+- Fixed a styling inconsistency on about page when instance description was missing (#659)
+- Fixed a UI discrepency in playlist tracks count (#647)
+- Fixed greyed tracks in radio builder and detail page (#637)
+- Fixed inconsistencies in subsonic error responses (#616)
+- Fixed incorrect icon for "next track" in player control (#613)
+- Fixed malformed search string when redirecting to LyricsWiki (#608)
+- Fixed missing track count on various library cards (#581)
+- Fixed skipped track when appending multiple tracks to the queue under certain conditions (#209)
+- Fixed wrong album/track count on artist page (#599)
+- Hide unplayable/emtpy playlists in "Browse playlist" pages (#424)
+- Initial UI render using correct language from browser (#644)
+- Invalid URI for reverse proxy websocket with apache (#617)
+- Properly encode Wikipedia and lyrics search urls (#470)
+- Refresh profile after user settings update to avoid cache issues (#606)
+- Use role=button instead of empty links for player controls (#610)
+
+
+Documentation:
+
+- Deploy documentation from the master branch instead of the develop branch to avoid inconsistencies (#642)
+- Document how to find and use library id when importing files in CLI (#562)
+- Fix documentation typos (#645)
+
+
 0.17 (2018-10-07)
 -----------------
 
@@ -120,7 +411,7 @@ Then, add the following block at the end of your docker-compose.yml file::
         - .env
       environment:
         # Override those variables in your .env file if needed
-        - "NGINX_MAX_BODY_SIZE=${NGINX_MAX_BODY_SIZE-30M}"
+        - "NGINX_MAX_BODY_SIZE=${NGINX_MAX_BODY_SIZE-100M}"
       volumes:
         - "./nginx/funkwhale.template:/etc/nginx/conf.d/funkwhale.template:ro"
         - "./nginx/funkwhale_proxy.conf:/etc/nginx/funkwhale_proxy.conf:ro"

CONTRIBUTING.rst

@@ -35,18 +35,20 @@ Setup front-end only development environment
     cd funkwhale
     cd front
 
-2. Install [nodejs](https://nodejs.org/en/download/package-manager/) and [yarn](https://yarnpkg.com/lang/en/docs/install/#debian-stable)
+2. Install `nodejs <https://nodejs.org/en/download/package-manager/>`_ and `yarn <https://yarnpkg.com/lang/en/docs/install/#debian-stable>`_
+
 3. Install the dependencies::
 
     yarn install
 
 4. Launch the development server::
 
-    # this will serve the front-end on http://localhost:8000
+    # this will serve the front-end on http://localhost:8000/front/
     VUE_PORT=8000 yarn serve
 
 5. Make the front-end talk with an existing server (like https://demo.funkwhale.audio),
    by clicking on the corresponding link in the footer
+
 6. Start hacking!
 
 Setup your development environment
@@ -307,7 +309,7 @@ A typical fragment looks like that:
     Fixed broken audio player on Chrome 42 for ogg files (#567)
 
 If the work fixes one or more issues, the issue number should be included at the
-end of the fragment (``(#567)`` is the issue number in the previous example.
+end of the fragment (``(#567)`` is the issue number in the previous example).
 
 If your work is not related to a specific issue, use the merge request
 identifier instead, like this:
@@ -389,7 +391,7 @@ This is regular pytest, so you can use any arguments/options that pytest usually
     # Stop on first failure
     docker-compose -f dev.yml run --rm api pytest -x
     # Run a specific test file
-    docker-compose -f dev.yml run --rm api pytest tests/test_acoustid.py
+    docker-compose -f dev.yml run --rm api pytest tests/music/test_models.py
 
 Writing tests
 ^^^^^^^^^^^^^
@@ -507,7 +509,7 @@ useful when testing components that depend on each other:
         # here, we ensure no email was sent
         mocked_notify.assert_not_called()
 
-Views: you can find some readable views tests in :file:`tests/users/test_views.py`
+Views: you can find some readable views tests in file: ``api/tests/users/test_views.py``
 
 .. note::
 

api/Dockerfile

-FROM python:3.6
+FROM alpine:3.8
 
-ENV PYTHONUNBUFFERED 1
+RUN \
+    echo 'installing dependencies' && \
+    apk add                \
+    bash             \
+    git             \
+    gettext            \
+    musl-dev           \
+    gcc \
+    postgresql-dev \
+    python3-dev        \
+    py3-psycopg2       \
+    py3-pillow         \
+    libldap            \
+    ffmpeg             \
+    libpq              \
+    libmagic           \
+    libffi-dev         \
+    zlib-dev           \
+    openldap-dev && \
+    \
+    \
+    ln -s /usr/bin/python3 /usr/bin/python
 
-# Requirements have to be pulled and installed here, otherwise caching won't work
-RUN echo 'deb http://httpredir.debian.org/debian/ jessie-backports main' > /etc/apt/sources.list.d/ffmpeg.list
-COPY ./requirements.apt /requirements.apt
-RUN apt-get update; \
-    grep "^[^#;]" requirements.apt | \
-    grep -Fv "python3-dev" | \
-    xargs apt-get install -y --no-install-recommends; \
-    rm -rf /usr/share/doc/* /usr/share/locale/*
-RUN curl -L https://github.com/acoustid/chromaprint/releases/download/v1.4.2/chromaprint-fpcalc-1.4.2-linux-x86_64.tar.gz | tar -xz -C /usr/local/bin --strip 1
+RUN mkdir /requirements
 COPY ./requirements/base.txt /requirements/base.txt
-RUN pip install -r /requirements/base.txt
-COPY ./requirements/production.txt /requirements/production.txt
-RUN pip install -r /requirements/production.txt
+RUN \
+    echo 'fixing requirements file for alpine' && \
+    sed -i '/Pillow/d' /requirements/base.txt && \
+    \
+    \
+    echo 'installing pip requirements' && \
+    pip3 install --no-cache-dir --upgrade pip && \
+    pip3 install --no-cache-dir setuptools wheel && \
+    pip3 install --no-cache-dir -r /requirements/base.txt
 
-COPY . /app
-
-# Since youtube-dl code is updated fairly often, we split it here
-RUN pip install --upgrade youtube-dl
-
-WORKDIR /app
+ARG install_dev_deps=0
+COPY ./requirements/*.txt /requirements/
+RUN \
+    if [ "$install_dev_deps" = "1" ] ; then echo "Installing dev dependencies" && pip3 install --no-cache-dir -r /requirements/local.txt -r /requirements/test.txt ; else echo "Skipping dev deps installation" ; fi
 
 ENTRYPOINT ["./compose/django/entrypoint.sh"]
 CMD ["./compose/django/daphne.sh"]
+
+COPY . /app
+WORKDIR /app

api/compose/django/daphne.sh

 #!/bin/bash -eux
 python /app/manage.py collectstatic --noinput
-/usr/local/bin/daphne -b 0.0.0.0 -p 5000 config.asgi:application --proxy-headers
+daphne -b 0.0.0.0 -p 5000 config.asgi:application --proxy-headers

api/compose/django/dev-entrypoint.sh

-#!/bin/bash
+#!/bin/sh
 set -e
 exec "$@"

api/compose/django/entrypoint.sh

-#!/bin/bash
+#!/bin/sh
 set -e
 # This entrypoint is used to play nicely with the current cookiecutter configuration.
 # Since docker-compose relies heavily on environment variables itself for configuration, we'd have to define multiple

api/config/api_urls.py

@@ -10,7 +10,7 @@ from funkwhale_api.playlists import views as playlists_views
 from funkwhale_api.subsonic.views import SubsonicViewSet
 
 router = routers.SimpleRouter()
-router.register(r"settings", GlobalPreferencesViewSet, base_name="settings")
+router.register(r"settings", GlobalPreferencesViewSet, basename="settings")
 router.register(r"activity", activity_views.ActivityViewSet, "activity")
 router.register(r"tags", views.TagViewSet, "tags")
 router.register(r"tracks", views.TrackViewSet, "tracks")
@@ -19,6 +19,7 @@ router.register(r"libraries", views.LibraryViewSet, "libraries")
 router.register(r"listen", views.ListenViewSet, "listen")
 router.register(r"artists", views.ArtistViewSet, "artists")
 router.register(r"albums", views.AlbumViewSet, "albums")
+router.register(r"licenses", views.LicenseViewSet, "licenses")
 router.register(r"playlists", playlists_views.PlaylistViewSet, "playlists")
 router.register(
     r"playlist-tracks", playlists_views.PlaylistTrackViewSet, "playlist-tracks"
@@ -26,10 +27,11 @@ router.register(
 v1_patterns = router.urls
 
 subsonic_router = routers.SimpleRouter(trailing_slash=False)
-subsonic_router.register(r"subsonic/rest", SubsonicViewSet, base_name="subsonic")
+subsonic_router.register(r"subsonic/rest", SubsonicViewSet, basename="subsonic")
 
 
 v1_patterns += [
+    url(r"^oembed/$", views.OembedView.as_view(), name="oembed"),
     url(
         r"^instance/",
         include(("funkwhale_api.instance.urls", "instance"), namespace="instance"),

api/config/settings/common.py

@@ -13,7 +13,7 @@ from __future__ import absolute_import, unicode_literals
 import datetime
 import logging
 
-from urllib.parse import urlparse, urlsplit
+from urllib.parse import urlsplit
 
 import environ
 from celery.schedules import crontab
@@ -69,12 +69,23 @@ else:
         FUNKWHALE_HOSTNAME = _parsed.netloc
         FUNKWHALE_PROTOCOL = _parsed.scheme
 
+FUNKWHALE_PROTOCOL = FUNKWHALE_PROTOCOL.lower()
+FUNKWHALE_HOSTNAME = FUNKWHALE_HOSTNAME.lower()
 FUNKWHALE_URL = "{}://{}".format(FUNKWHALE_PROTOCOL, FUNKWHALE_HOSTNAME)
-
+FUNKWHALE_SPA_HTML_ROOT = env(
+    "FUNKWHALE_SPA_HTML_ROOT", default=FUNKWHALE_URL + "/front/"
+)
+FUNKWHALE_SPA_HTML_CACHE_DURATION = env.int(
+    "FUNKWHALE_SPA_HTML_CACHE_DURATION", default=60 * 15
+)
+FUNKWHALE_EMBED_URL = env(
+    "FUNKWHALE_EMBED_URL", default=FUNKWHALE_SPA_HTML_ROOT + "embed.html"
+)
+APP_NAME = "Funkwhale"
 
 # XXX: deprecated, see #186
 FEDERATION_ENABLED = env.bool("FEDERATION_ENABLED", default=True)
-FEDERATION_HOSTNAME = env("FEDERATION_HOSTNAME", default=FUNKWHALE_HOSTNAME)
+FEDERATION_HOSTNAME = env("FEDERATION_HOSTNAME", default=FUNKWHALE_HOSTNAME).lower()
 # XXX: deprecated, see #186
 FEDERATION_COLLECTION_PAGE_SIZE = env.int("FEDERATION_COLLECTION_PAGE_SIZE", default=50)
 # XXX: deprecated, see #186
@@ -83,7 +94,7 @@ FEDERATION_MUSIC_NEEDS_APPROVAL = env.bool(
 )
 # XXX: deprecated, see #186
 FEDERATION_ACTOR_FETCH_DELAY = env.int("FEDERATION_ACTOR_FETCH_DELAY", default=60 * 12)
-ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS")
+ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=[]) + [FUNKWHALE_HOSTNAME]
 
 # APP CONFIGURATION
 # ------------------------------------------------------------------------------
@@ -145,10 +156,10 @@ LOCAL_APPS = (
     "funkwhale_api.requests",
     "funkwhale_api.favorites",
     "funkwhale_api.federation",
+    "funkwhale_api.moderation",
     "funkwhale_api.radios",
     "funkwhale_api.history",
     "funkwhale_api.playlists",
-    "funkwhale_api.providers.acoustid",
     "funkwhale_api.subsonic",
 )
 
@@ -159,7 +170,7 @@ INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS
 # MIDDLEWARE CONFIGURATION
 # ------------------------------------------------------------------------------
 MIDDLEWARE = (
-    # Make sure djangosecure.middleware.SecurityMiddleware is listed first
+    "funkwhale_api.common.middleware.SPAFallbackMiddleware",
     "django.contrib.sessions.middleware.SessionMiddleware",
     "corsheaders.middleware.CorsMiddleware",
     "django.middleware.common.CommonMiddleware",
@@ -305,8 +316,7 @@ FILE_UPLOAD_PERMISSIONS = 0o644
 # URL Configuration
 # ------------------------------------------------------------------------------
 ROOT_URLCONF = "config.urls"
-# See: https://docs.djangoproject.com/en/dev/ref/settings/#wsgi-application
-WSGI_APPLICATION = "config.wsgi.application"
+SPA_URLCONF = "config.spa_urls"
 ASGI_APPLICATION = "config.routing.application"
 
 # This ensures that Django will be able to detect a secure connection
@@ -315,7 +325,7 @@ SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
 # AUTHENTICATION CONFIGURATION
 # ------------------------------------------------------------------------------
 AUTHENTICATION_BACKENDS = (
-    "django.contrib.auth.backends.ModelBackend",
+    "funkwhale_api.users.auth_backends.ModelBackend",
     "allauth.account.auth_backends.AuthenticationBackend",
 )
 SESSION_COOKIE_HTTPONLY = False
@@ -400,15 +410,20 @@ if AUTH_LDAP_ENABLED:
 AUTOSLUG_SLUGIFY_FUNCTION = "slugify.slugify"
 
 CACHE_DEFAULT = "redis://127.0.0.1:6379/0"
-CACHES = {"default": env.cache_url("CACHE_URL", default=CACHE_DEFAULT)}
+CACHES = {
+    "default": env.cache_url("CACHE_URL", default=CACHE_DEFAULT),
+    "local": {
+        "BACKEND": "django.core.cache.backends.locmem.LocMemCache",
+        "LOCATION": "local-cache",
+    },
+}
 
 CACHES["default"]["BACKEND"] = "django_redis.cache.RedisCache"
 
-cache_url = urlparse(CACHES["default"]["LOCATION"])
 CHANNEL_LAYERS = {
     "default": {
         "BACKEND": "channels_redis.core.RedisChannelLayer",
-        "CONFIG": {"hosts": [(cache_url.hostname, cache_url.port)]},
+        "CONFIG": {"hosts": [CACHES["default"]["LOCATION"]]},
     }
 }
 
@@ -435,7 +450,12 @@ CELERY_BEAT_SCHEDULE = {
         "task": "federation.clean_music_cache",
         "schedule": crontab(hour="*/2"),
         "options": {"expires": 60 * 2},
-    }
+    },
+    "music.clean_transcoding_cache": {
+        "task": "music.clean_transcoding_cache",
+        "schedule": crontab(hour="*"),
+        "options": {"expires": 60 * 2},
+    },
 }
 
 JWT_AUTH = {
@@ -516,6 +536,7 @@ MUSICBRAINZ_HOSTNAME = env("MUSICBRAINZ_HOSTNAME", default="musicbrainz.org")
 # Custom Admin URL, use {% url 'admin:index' %}
 ADMIN_URL = env("DJANGO_ADMIN_URL", default="^api/admin/")
 CSRF_USE_SESSIONS = True
+SESSION_ENGINE = "django.contrib.sessions.backends.cache"
 
 # Playlist settings
 # XXX: deprecated, see #186
@@ -570,3 +591,9 @@ VERSATILEIMAGEFIELD_RENDITION_KEY_SETS = {
     ]
 }
 VERSATILEIMAGEFIELD_SETTINGS = {"create_images_on_demand": False}
+RSA_KEY_SIZE = 2048
+# for performance gain in tests, since we don't need to actually create the
+# thumbnails
+CREATE_IMAGE_THUMBNAILS = env.bool("CREATE_IMAGE_THUMBNAILS", default=True)
+# we rotate actor keys at most every two days by default
+ACTOR_KEY_ROTATION_DELAY = env.int("ACTOR_KEY_ROTATION_DELAY", default=3600 * 48)

api/config/settings/local.py

@@ -14,6 +14,7 @@ from .common import *  # noqa
 # DEBUG
 # ------------------------------------------------------------------------------
 DEBUG = env.bool("DJANGO_DEBUG", default=True)
+FORCE_HTTPS_URLS = env.bool("FORCE_HTTPS_URLS", default=False)
 TEMPLATES[0]["OPTIONS"]["debug"] = DEBUG
 
 # SECRET CONFIGURATION
@@ -31,7 +32,6 @@ EMAIL_PORT = 1025
 
 # django-debug-toolbar
 # ------------------------------------------------------------------------------
-MIDDLEWARE += ("debug_toolbar.middleware.DebugToolbarMiddleware",)
 
 # INTERNAL_IPS = ('127.0.0.1', '10.0.2.2',)
 
@@ -39,20 +39,24 @@ DEBUG_TOOLBAR_CONFIG = {
     "DISABLE_PANELS": ["debug_toolbar.panels.redirects.RedirectsPanel"],
     "SHOW_TEMPLATE_CONTEXT": True,
     "SHOW_TOOLBAR_CALLBACK": lambda request: True,
-    "JQUERY_URL": "",
+    "JQUERY_URL": "/staticfiles/admin/js/vendor/jquery/jquery.js",
 }
 
 # django-extensions
 # ------------------------------------------------------------------------------
 # INSTALLED_APPS += ('django_extensions', )
-INSTALLED_APPS += ("debug_toolbar",)
+
+# Debug toolbar is slow, we disable it for tests
+DEBUG_TOOLBAR_ENABLED = env.bool("DEBUG_TOOLBAR_ENABLED", default=DEBUG)
+if DEBUG_TOOLBAR_ENABLED:
+    MIDDLEWARE += ("debug_toolbar.middleware.DebugToolbarMiddleware",)
+    INSTALLED_APPS += ("debug_toolbar",)
 
 # TESTING
 # ------------------------------------------------------------------------------
 TEST_RUNNER = "django.test.runner.DiscoverRunner"
 
 # CELERY
-# In development, all tasks will be executed locally by blocking until the task returns
 CELERY_TASK_ALWAYS_EAGER = False
 # END CELERY
 
@@ -72,3 +76,10 @@ LOGGING = {
     },
 }
 CSRF_TRUSTED_ORIGINS = [o for o in ALLOWED_HOSTS]
+
+
+if env.bool("WEAK_PASSWORDS", default=False):
+    # Faster during tests
+    PASSWORD_HASHERS = ("django.contrib.auth.hashers.MD5PasswordHasher",)
+
+MIDDLEWARE = ("funkwhale_api.common.middleware.DevHttpsMiddleware",) + MIDDLEWARE

api/config/spa_urls.py

+from django import urls
+
+from funkwhale_api.music import spa_views
+
+
+urlpatterns = [
+    urls.re_path(
+        r"^library/tracks/(?P<pk>\d+)/?$", spa_views.library_track, name="library_track"
+    ),
+    urls.re_path(
+        r"^library/albums/(?P<pk>\d+)/?$", spa_views.library_album, name="library_album"
+    ),
+    urls.re_path(
+        r"^library/artists/(?P<pk>\d+)/?$",
+        spa_views.library_artist,
+        name="library_artist",
+    ),
+]

api/config/wsgi.py

-"""
-WSGI config for funkwhale_api project.
-
-This module contains the WSGI application used by Django's development server
-and any production WSGI deployments. It should expose a module-level variable
-named ``application``. Django's ``runserver`` and ``runfcgi`` commands discover
-this application via the ``WSGI_APPLICATION`` setting.
-
-Usually you will have the standard Django WSGI application here, but it also
-might make sense to replace the whole Django WSGI application with a custom one
-that later delegates to the Django one. For example, you could introduce WSGI
-middleware here, or combine a Django application with an application of another
-framework.
-
-"""
-import os
-
-from django.core.wsgi import get_wsgi_application
-from whitenoise.django import DjangoWhiteNoise
-
-# We defer to a DJANGO_SETTINGS_MODULE already in the environment. This breaks
-# if running multiple sites in the same mod_wsgi process. To fix this, use
-# mod_wsgi daemon mode with each site in its own daemon process, or use
-# os.environ["DJANGO_SETTINGS_MODULE"] = "config.settings.production"
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.production")
-
-# This application object is used by any WSGI server configured to use this
-# file. This includes Django's development server, if the WSGI_APPLICATION
-# setting points here.
-application = get_wsgi_application()
-
-# Use Whitenoise to serve static files
-# See: https://whitenoise.readthedocs.org/
-application = DjangoWhiteNoise(application)
-
-
-# Apply WSGI middleware here.
-# from helloworld.wsgi import HelloWorldApplication
-# application = HelloWorldApplication(application)

api/docker/Dockerfile.test

-FROM python:3.6
-
-ENV PYTHONUNBUFFERED 1
-
-# Requirements have to be pulled and installed here, otherwise caching won't work
-RUN echo 'deb http://httpredir.debian.org/debian/ jessie-backports main' > /etc/apt/sources.list.d/ffmpeg.list
-COPY ./requirements.apt /requirements.apt
-RUN apt-get update; \
-    grep "^[^#;]" requirements.apt | \
-    grep -Fv "python3-dev" | \
-    xargs apt-get install -y --no-install-recommends; \
-    rm -rf /usr/share/doc/* /usr/share/locale/*
-
-RUN curl -L https://github.com/acoustid/chromaprint/releases/download/v1.4.2/chromaprint-fpcalc-1.4.2-linux-x86_64.tar.gz | tar -xz -C /usr/local/bin --strip 1
-
-RUN mkdir /requirements
-COPY ./requirements/base.txt /requirements/base.txt
-RUN pip install -r /requirements/base.txt
-COPY ./requirements/local.txt /requirements/local.txt
-RUN pip install -r /requirements/local.txt
-COPY ./requirements/test.txt /requirements/test.txt
-RUN pip install -r /requirements/test.txt
-
-COPY . /app
-WORKDIR /app
-ENTRYPOINT ["compose/django/dev-entrypoint.sh"]

api/funkwhale_api/__init__.py

 # -*- coding: utf-8 -*-
-__version__ = "0.17"
+__version__ = "0.18"
 __version_info__ = tuple(
     [
         int(num) if num.isdigit() else num

api/funkwhale_api/common/channels.py

@@ -5,7 +5,7 @@ from asgiref.sync import async_to_sync
 from channels.layers import get_channel_layer
 from django.core.serializers.json import DjangoJSONEncoder
 
-logger = logging.getLogger(__file__)
+logger = logging.getLogger(__name__)
 channel_layer = get_channel_layer()
 group_add = async_to_sync(channel_layer.group_add)
 

api/funkwhale_api/common/decorators.py

+from rest_framework import response
+from rest_framework import decorators
+
+
+def action_route(serializer_class):
+    @decorators.action(methods=["post"], detail=False)
+    def action(self, request, *args, **kwargs):
+        queryset = self.get_queryset()
+        serializer = serializer_class(request.data, queryset=queryset)
+        serializer.is_valid(raise_exception=True)
+        result = serializer.save()
+        return response.Response(result, status=200)
+
+    return action

api/funkwhale_api/common/middleware.py

+import html
+import requests
+
+from django import http
+from django.conf import settings
+from django.core.cache import caches
+from django import urls
+
+from . import preferences
+from . import utils
+
+EXCLUDED_PATHS = ["/api", "/federation", "/.well-known"]
+
+
+def should_fallback_to_spa(path):
+    if path == "/":
+        return True
+    return not any([path.startswith(m) for m in EXCLUDED_PATHS])
+
+
+def serve_spa(request):
+    html = get_spa_html(settings.FUNKWHALE_SPA_HTML_ROOT)
+    head, tail = html.split("</head>", 1)
+    if not preferences.get("common__api_authentication_required"):
+        try:
+            request_tags = get_request_head_tags(request) or []
+        except urls.exceptions.Resolver404:
+            # we don't have any custom tags for this route
+            request_tags = []
+    else:
+        # API is not open, we don't expose any custom data
+        request_tags = []
+    default_tags = get_default_head_tags(request.path)
+    unique_attributes = ["name", "property"]
+
+    final_tags = request_tags
+    skip = []
+
+    for t in final_tags:
+        for attr in unique_attributes:
+            if attr in t:
+                skip.append(t[attr])
+    for t in default_tags:
+        existing = False
+        for attr in unique_attributes:
+            if t.get(attr) in skip:
+                existing = True
+                break
+        if not existing:
+            final_tags.append(t)
+
+    # let's inject our meta tags in the HTML
+    head += "\n" + "\n".join(render_tags(final_tags)) + "\n</head>"
+
+    return http.HttpResponse(head + tail)
+
+
+def get_spa_html(spa_url):
+    cache_key = "spa-html:{}".format(spa_url)
+    cached = caches["local"].get(cache_key)
+    if cached:
+        return cached
+
+    response = requests.get(
+        utils.join_url(spa_url, "index.html"),
+        verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
+    )
+    response.raise_for_status()
+    content = response.text
+    caches["local"].set(cache_key, content, settings.FUNKWHALE_SPA_HTML_CACHE_DURATION)
+    return content
+
+
+def get_default_head_tags(path):
+    instance_name = preferences.get("instance__name")
+    short_description = preferences.get("instance__short_description")
+    app_name = settings.APP_NAME
+
+    parts = [instance_name, app_name]
+
+    return [
+        {"tag": "meta", "property": "og:type", "content": "website"},
+        {
+            "tag": "meta",
+            "property": "og:site_name",
+            "content": " - ".join([p for p in parts if p]),
+        },
+        {"tag": "meta", "property": "og:description", "content": short_description},
+        {
+            "tag": "meta",
+            "property": "og:image",
+            "content": utils.join_url(settings.FUNKWHALE_URL, "/front/favicon.png"),
+        },
+        {
+            "tag": "meta",
+            "property": "og:url",
+            "content": utils.join_url(settings.FUNKWHALE_URL, path),
+        },
+    ]
+
+
+def render_tags(tags):
+    """
+    Given a dict like {'tag': 'meta', 'hello': 'world'}
+    return a html ready tag like
+    <meta hello="world" />
+    """
+    for tag in tags:
+
+        yield "<{tag} {attrs} />".format(
+            tag=tag.pop("tag"),
+            attrs=" ".join(
+                [
+                    '{}="{}"'.format(a, html.escape(str(v)))
+                    for a, v in sorted(tag.items())
+                    if v
+                ]
+            ),
+        )
+
+
+def get_request_head_tags(request):
+    match = urls.resolve(request.path, urlconf=settings.SPA_URLCONF)
+    return match.func(request, *match.args, **match.kwargs)
+
+
+class SPAFallbackMiddleware:
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        response = self.get_response(request)
+
+        if response.status_code == 404 and should_fallback_to_spa(request.path):
+            return serve_spa(request)
+
+        return response
+
+
+class DevHttpsMiddleware:
+    """
+    In development, it's sometimes difficult to have django use HTTPS
+    when we have django behind nginx behind traefix.
+
+    We thus use a simple setting (in dev ONLY) to control that.
+    """
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        if settings.FORCE_HTTPS_URLS:
+            setattr(request.__class__, "scheme", "https")
+            setattr(
+                request,
+                "get_host",
+                lambda: request.__class__.get_host(request).replace(":80", ":443"),
+            )
+        return self.get_response(request)