Merge branch 'release/0.6.1'

.env.dev

 API_AUTHENTICATION_REQUIRED=True
-CACHALOT_ENABLED=False
 RAVEN_ENABLED=false
 RAVEN_DSN=https://44332e9fdd3d42879c7d35bf8562c6a4:0062dc16a22b41679cd5765e5342f716@sentry.eliotberriot.com/5

.gitignore

@@ -84,3 +84,5 @@ front/test/unit/coverage
 front/test/e2e/reports
 front/selenium-debug.log
 docs/_build
+
+data/

CHANGELOG

@@ -3,6 +3,27 @@ Changelog
 
 .. towncrier
 
+0.6.1 (unreleased)
+------------------
+
+Features:
+
+- Can now skip acoustid on file import with the --no-acoustid flag (#111)
+
+
+Bugfixes:
+
+- Added missing batch id in output during import (#112)
+- Added some feedback on the play button (#100)
+- Smarter pagination which takes a fixed size (#84)
+
+
+Other:
+
+- Completely removed django-cachalot from the codebase (#110). You can safely
+  remove the CACHALOT_ENABLED setting from your .env file
+
+
 0.6 (2018-03-04)
 ----------------
 

api/compose/django/daphne.sh

 #!/bin/bash -eux
 python /app/manage.py collectstatic --noinput
-/usr/local/bin/daphne --root-path=/app -b 0.0.0.0 -p 5000 config.asgi:application
+/usr/local/bin/daphne -b 0.0.0.0 -p 5000 config.asgi:application

api/compose/django/entrypoint.sh

@@ -4,16 +4,19 @@ set -e
 # Since docker-compose relies heavily on environment variables itself for configuration, we'd have to define multiple
 # environment variables just to support cookiecutter out of the box. That makes no sense, so this little entrypoint
 # does all this for us.
-export CACHE_URL=redis://redis:6379/0
+export CACHE_URL=${CACHE_URL:="redis://redis:6379/0"}
 
-# the official postgres image uses 'postgres' as default user if not set explictly.
-if [ -z "$POSTGRES_ENV_POSTGRES_USER" ]; then
+if [ -z "$DATABASE_URL" ]; then
+  # the official postgres image uses 'postgres' as default user if not set explictly.
+  if [ -z "$POSTGRES_ENV_POSTGRES_USER" ]; then
     export POSTGRES_ENV_POSTGRES_USER=postgres
+  fi
+  export DATABASE_URL=postgres://$POSTGRES_ENV_POSTGRES_USER:$POSTGRES_ENV_POSTGRES_PASSWORD@postgres:5432/$POSTGRES_ENV_POSTGRES_USER
 fi
 
-export DATABASE_URL=postgres://$POSTGRES_ENV_POSTGRES_USER:$POSTGRES_ENV_POSTGRES_PASSWORD@postgres:5432/$POSTGRES_ENV_POSTGRES_USER
-
-export CELERY_BROKER_URL=$CACHE_URL
+if [ -z "$CELERY_BROKER_URL" ]; then
+  export CELERY_BROKER_URL=$CACHE_URL
+fi
 
 # we copy the frontend files, if any so we can serve them from the outside
 if [ -d "frontend" ]; then

api/config/settings/common.py

@@ -55,7 +55,6 @@ THIRD_PARTY_APPS = (
     'rest_framework',
     'rest_framework.authtoken',
     'taggit',
-    'cachalot',
     'rest_auth',
     'rest_auth.registration',
     'mptt',
@@ -310,7 +309,7 @@ CELERY_BROKER_URL = env(
     "CELERY_BROKER_URL", default=env('CACHE_URL', default=CACHE_DEFAULT))
 ########## END CELERY
 # Location of root django.contrib.admin URL, use {% url 'admin:index' %}
-ADMIN_URL = r'^admin/'
+
 # Your common stuff: Below this line define 3rd party library settings
 CELERY_TASK_DEFAULT_RATE_LIMIT = 1
 CELERY_TASK_TIME_LIMIT = 300
@@ -371,9 +370,6 @@ MUSICBRAINZ_CACHE_DURATION = env.int(
     default=300
 )
 
-CACHALOT_ENABLED = env.bool('CACHALOT_ENABLED', default=True)
-
-
 # Custom Admin URL, use {% url 'admin:index' %}
 ADMIN_URL = env('DJANGO_ADMIN_URL', default='^api/admin/')
 CSRF_USE_SESSIONS = True

api/demo/load-demo-data.sh

@@ -6,8 +6,8 @@ python manage.py migrate --noinput
 
 echo "Creating demo user..."
 
-cat demo/demo-user.py | python manage.py shell --plain
+cat demo/demo-user.py | python manage.py shell -i python
 
 echo "Importing demo tracks..."
 
-python manage.py import_files "/music/**/*.ogg" --recursive --noinput
+python manage.py import_files "/music/**/*.ogg" --recursive --noinput --username demo

api/funkwhale_api/__init__.py

 # -*- coding: utf-8 -*-
-__version__ = '0.6'
+__version__ = '0.6.1'
 __version_info__ = tuple([int(num) if num.isdigit() else num for num in __version__.replace('-', '.', 1).split('.')])

api/funkwhale_api/common/auth.py

@@ -9,6 +9,7 @@ from rest_framework import exceptions
 from rest_framework_jwt.settings import api_settings
 from rest_framework_jwt.authentication import BaseJSONWebTokenAuthentication
 
+from funkwhale_api.users.models import User
 
 
 class TokenHeaderAuth(BaseJSONWebTokenAuthentication):
@@ -40,7 +41,7 @@ class TokenAuthMiddleware:
         auth = TokenHeaderAuth()
         try:
             user, token = auth.authenticate(scope)
-        except exceptions.AuthenticationFailed:
+        except (User.DoesNotExist, exceptions.AuthenticationFailed):
             user = AnonymousUser()
 
         scope['user'] = user

api/funkwhale_api/music/tasks.py

 from django.core.files.base import ContentFile
 
+from dynamic_preferences.registries import global_preferences_registry
+
 from funkwhale_api.taskapp import celery
 from funkwhale_api.providers.acoustid import get_acoustid_client
 from funkwhale_api.providers.audiofile.tasks import import_track_data_from_path
@@ -23,21 +25,22 @@ def set_acoustid_on_track_file(track_file):
         return update(result['id'])
 
 
-def _do_import(import_job, replace):
+def _do_import(import_job, replace, use_acoustid=True):
     from_file = bool(import_job.audio_file)
     mbid = import_job.mbid
     acoustid_track_id = None
     duration = None
     track = None
-    if not mbid and from_file:
+    manager = global_preferences_registry.manager()
+    use_acoustid = use_acoustid and manager['providers_acoustid__api_key']
+    if not mbid and use_acoustid and from_file:
         # we try to deduce mbid from acoustid
         client = get_acoustid_client()
         match = client.get_best_match(import_job.audio_file.path)
-        if not match:
-            raise ValueError('Cannot get match')
-        duration = match['recordings'][0]['duration']
-        mbid = match['recordings'][0]['id']
-        acoustid_track_id = match['id']
+        if match:
+            duration = match['recordings'][0]['duration']
+            mbid = match['recordings'][0]['id']
+            acoustid_track_id = match['id']
     if mbid:
         track, _ = models.Track.get_or_create_from_api(mbid=mbid)
     else:
@@ -77,13 +80,13 @@ def _do_import(import_job, replace):
     models.ImportJob.objects.filter(
         status__in=['pending', 'errored']),
     'import_job')
-def import_job_run(self, import_job, replace=False):
+def import_job_run(self, import_job, replace=False, use_acoustid=True):
     def mark_errored():
         import_job.status = 'errored'
-        import_job.save()
+        import_job.save(update_fields=['status'])
 
     try:
-        return _do_import(import_job, replace)
+        return _do_import(import_job, replace, use_acoustid=use_acoustid)
     except Exception as exc:
         if not settings.DEBUG:
             try:

api/funkwhale_api/providers/audiofile/management/commands/import_files.py

@@ -34,6 +34,13 @@ class Command(BaseCommand):
             default=False,
             help='Will launch celery tasks for each file to import instead of doing it synchronously and block the CLI',
         )
+        parser.add_argument(
+            '--no-acoustid',
+            action='store_true',
+            dest='no_acoustid',
+            default=False,
+            help='Use this flag to completely bypass acoustid completely',
+        )
         parser.add_argument(
             '--noinput', '--no-input', action='store_false', dest='interactive',
             help="Do NOT prompt the user for input of any kind.",
@@ -81,13 +88,12 @@ class Command(BaseCommand):
                 raise CommandError("Import cancelled.")
 
         batch = self.do_import(matching, user=user, options=options)
-
         message = 'Successfully imported {} tracks'
         if options['async']:
             message = 'Successfully launched import for {} tracks'
         self.stdout.write(message.format(len(matching)))
         self.stdout.write(
-            "For details, please refer to import batch #".format(batch.pk))
+            "For details, please refer to import batch #{}".format(batch.pk))
 
     @transaction.atomic
     def do_import(self, matching, user, options):
@@ -109,7 +115,10 @@ class Command(BaseCommand):
 
             job.save()
             try:
-                utils.on_commit(import_handler, import_job_id=job.pk)
+                utils.on_commit(
+                    import_handler,
+                    import_job_id=job.pk,
+                    use_acoustid=not options['no_acoustid'])
             except Exception as e:
                 self.stdout.write('Error: {}'.format(e))
 

api/funkwhale_api/templates/404.html

-{% extends "base.html" %}
-
-{% block title %}Page Not found{% endblock %}
-
-{% block content %}
-<h1>Page Not found</h1>
-
-<p>This is not the page you were looking for.</p>
-{% endblock content %}

api/funkwhale_api/templates/500.html

-{% extends "base.html" %}
-
-{% block title %}Server Error{% endblock %}
-
-{% block content %}
-<h1>Ooops!!! 500</h1>
-
-<h3>Looks like something went wrong!</h3>
-
-<p>We track these errors automatically, but if the problem persists feel free to contact us. In the meantime, try refreshing.</p>
-{% endblock content %}
-
-

api/funkwhale_api/templates/base.html

-{% load staticfiles i18n %}<!DOCTYPE html>
-<html lang="en" ng-app>
-  <head>
-    <meta charset="utf-8">
-    <meta http-equiv="x-ua-compatible" content="ie=edge">
-    <title>{% block title %}funkwhale_api{% endblock title %}</title>
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <meta name="description" content="">
-    <meta name="author" content="">
-
-    <!-- HTML5 shim, for IE6-8 support of HTML5 elements -->
-    <!--[if lt IE 9]>
-      <script src="https://html5shim.googlecode.com/svn/trunk/html5.js"></script>
-    <![endif]-->
-
-    {% block css %}
-    <!-- Latest compiled and minified CSS -->
-    <link rel="stylesheet" href="https://cdn.rawgit.com/twbs/bootstrap/v4-dev/dist/css/bootstrap.css">
-
-    <!-- Your stuff: Third-party css libraries go here -->
-
-    <!-- This file store project specific CSS -->
-    <link href="{% static 'css/project.css' %}" rel="stylesheet">
-    {% endblock %}
-
-    {% block angular %}
-    <script src="https://ajax.googleapis.com/ajax/libs/angularjs/1.4.8/angular.min.js"></script>
-    {% endblock %}
-
-  </head>
-
-  <body>
-
-    <div class="m-b">
-      <nav class="navbar navbar-dark navbar-static-top bg-inverse">
-        <div class="container">
-          <a class="navbar-brand" href="/">funkwhale_api</a>
-          <button type="button" class="navbar-toggler hidden-sm-up pull-right" data-toggle="collapse" data-target="#bs-navbar-collapse-1">
-            &#9776;
-          </button>
-
-          <!-- Collect the nav links, forms, and other content for toggling -->
-          <div class="collapse navbar-toggleable-xs" id="bs-navbar-collapse-1">
-            <ul class="nav navbar-nav">
-              <li class="nav-item">
-                <a class="nav-link" href="">Home</a>
-              </li>
-              <li class="nav-item">
-                <a class="nav-link" href="">About</a>
-              </li>
-            </ul>
-
-            <ul class="nav navbar-nav pull-right">
-              {% if request.user.is_authenticated %}
-                <li class="nav-item">
-                  <a class="nav-link" href="{% url 'users:detail' request.user.username  %}">{% trans "My Profile" %}</a>
-                </li>
-                <li class="nav-item">
-                  <a class="nav-link" href="{% url 'account_logout' %}">{% trans "Logout" %}</a>
-                </li>
-              {% else %}
-                <li class="nav-item">
-                  <a id="sign-up-link" class="nav-link" href="{% url 'account_signup' %}">{% trans "Sign Up" %}</a>
-                </li>
-                <li class="nav-item">
-                  <a id="log-in-link" class="nav-link" href="{% url 'account_login' %}">{% trans "Log In" %}</a>
-                </li>
-              {% endif %}
-            </ul>
-          </div>
-        </div>
-      </nav>
-    </div>
-
-    <div class="container">
-
-      {% if messages %}
-          {% for message in messages %}
-              <div class="alert {% if message.tags %}alert-{{ message.tags }}{% endif %}">{{ message }}</div>
-          {% endfor %}
-      {% endif %}
-
-      {% block content %}
-        <p>Use this document as a way to quick start any new project.</p>
-      {% endblock content %}
-
-    </div> <!-- /container -->
-
-    {% block modal %}{% endblock modal %}
-
-    <!-- Le javascript
-    ================================================== -->
-    <!-- Placed at the end of the document so the pages load faster -->
-    {% block javascript %}
-      <!-- Latest JQuery -->
-      <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.4/jquery.min.js"></script>
-
-      <!-- Latest compiled and minified JavaScript -->
-      <script src="https://cdn.rawgit.com/twbs/bootstrap/v4-dev/dist/js/bootstrap.js"></script>
-
-      <!-- Your stuff: Third-party javascript libraries go here -->
-
-      <!-- place project specific Javascript in this file -->
-      <script src="{% static 'js/project.js' %}"></script>
-    {% endblock javascript %}
-  </body>
-</html>

api/funkwhale_api/templates/pages/about.html

-{% extends "base.html" %}
\ No newline at end of file

api/funkwhale_api/templates/pages/home.html

-{% extends "base.html" %}
\ No newline at end of file

api/requirements/base.txt

@@ -50,9 +50,6 @@ mutagen>=1.39,<1.40
 django-taggit>=0.22,<0.23
 # Until this is merged
 git+https://github.com/EliotBerriot/PyMemoize.git@django
-# Until this is merged
-#django-cachalot==1.5.0
-git+https://github.com/EliotBerriot/django-cachalot.git@django-2
 
 django-dynamic-preferences>=1.5,<1.6
 pyacoustid>=1.1.5,<1.2

api/tests/music/test_tasks.py

@@ -9,7 +9,8 @@ from . import data as api_data
 DATA_DIR = os.path.dirname(os.path.abspath(__file__))
 
 
-def test_set_acoustid_on_track_file(factories, mocker):
+def test_set_acoustid_on_track_file(factories, mocker, preferences):
+    preferences['providers_acoustid__api_key'] = 'test'
     track_file = factories['music.TrackFile'](acoustid_track_id=None)
     id = 'e475bf79-c1ce-4441-bed7-1e33f226c0a2'
     payload = {
@@ -31,7 +32,7 @@ def test_set_acoustid_on_track_file(factories, mocker):
 
     assert str(track_file.acoustid_track_id) == id
     assert r == id
-    m.assert_called_once_with('', track_file.audio_file.path, parse=False)
+    m.assert_called_once_with('test', track_file.audio_file.path, parse=False)
 
 
 def test_set_acoustid_on_track_file_required_high_score(factories, mocker):
@@ -48,7 +49,9 @@ def test_set_acoustid_on_track_file_required_high_score(factories, mocker):
     assert track_file.acoustid_track_id is None
 
 
-def test_import_job_can_run_with_file_and_acoustid(factories, mocker):
+def test_import_job_can_run_with_file_and_acoustid(
+        preferences, factories, mocker):
+    preferences['providers_acoustid__api_key'] = 'test'
     path = os.path.join(DATA_DIR, 'test.ogg')
     mbid = '9968a9d6-8d92-4051-8f76-674e157b6eed'
     acoustid_payload = {
@@ -88,7 +91,46 @@ def test_import_job_can_run_with_file_and_acoustid(factories, mocker):
     assert job.source == 'file://'
 
 
-def test_import_job_can_be_skipped(factories, mocker):
+def test_run_import_skipping_accoustid(factories, mocker):
+    m = mocker.patch('funkwhale_api.music.tasks._do_import')
+    path = os.path.join(DATA_DIR, 'test.ogg')
+    job = factories['music.FileImportJob'](audio_file__path=path)
+    tasks.import_job_run(import_job_id=job.pk, use_acoustid=False)
+    m.assert_called_once_with(job, False, use_acoustid=False)
+
+
+def test__do_import_skipping_accoustid(factories, mocker):
+    t = factories['music.Track']()
+    m = mocker.patch(
+        'funkwhale_api.music.tasks.import_track_data_from_path',
+        return_value=t)
+    path = os.path.join(DATA_DIR, 'test.ogg')
+    job = factories['music.FileImportJob'](
+        mbid=None,
+        audio_file__path=path)
+    p = job.audio_file.path
+    tasks._do_import(job, replace=False, use_acoustid=False)
+    m.assert_called_once_with(p)
+
+
+def test__do_import_skipping_accoustid_if_no_key(
+        factories, mocker, preferences):
+    preferences['providers_acoustid__api_key'] = ''
+    t = factories['music.Track']()
+    m = mocker.patch(
+        'funkwhale_api.music.tasks.import_track_data_from_path',
+        return_value=t)
+    path = os.path.join(DATA_DIR, 'test.ogg')
+    job = factories['music.FileImportJob'](
+        mbid=None,
+        audio_file__path=path)
+    p = job.audio_file.path
+    tasks._do_import(job, replace=False, use_acoustid=False)
+    m.assert_called_once_with(p)
+
+
+def test_import_job_can_be_skipped(factories, mocker, preferences):
+    preferences['providers_acoustid__api_key'] = 'test'
     path = os.path.join(DATA_DIR, 'test.ogg')
     mbid = '9968a9d6-8d92-4051-8f76-674e157b6eed'
     track_file = factories['music.TrackFile'](track__mbid=mbid)
@@ -124,7 +166,8 @@ def test_import_job_can_be_skipped(factories, mocker):
     assert job.status == 'skipped'
 
 
-def test_import_job_can_be_errored(factories, mocker):
+def test_import_job_can_be_errored(factories, mocker, preferences):
+    preferences['providers_acoustid__api_key'] = 'test'
     path = os.path.join(DATA_DIR, 'test.ogg')
     mbid = '9968a9d6-8d92-4051-8f76-674e157b6eed'
     track_file = factories['music.TrackFile'](track__mbid=mbid)

api/tests/test_import_audio_file.py

@@ -54,7 +54,7 @@ def test_management_command_requires_a_valid_username(factories, mocker):
 
 
 def test_import_files_creates_a_batch_and_job(factories, mocker):
-    m = m = mocker.patch('funkwhale_api.common.utils.on_commit')
+    m = mocker.patch('funkwhale_api.common.utils.on_commit')
     user = factories['users.User'](username='me')
     path = os.path.join(DATA_DIR, 'dummy_file.ogg')
     call_command(
@@ -77,4 +77,24 @@ def test_import_files_creates_a_batch_and_job(factories, mocker):
     assert job.source == 'file://' + path
     m.assert_called_once_with(
         music_tasks.import_job_run.delay,
-        import_job_id=job.pk)
+        import_job_id=job.pk,
+        use_acoustid=True)
+
+
+def test_import_files_skip_acoustid(factories, mocker):
+    m = mocker.patch('funkwhale_api.common.utils.on_commit')
+    user = factories['users.User'](username='me')
+    path = os.path.join(DATA_DIR, 'dummy_file.ogg')
+    call_command(
+        'import_files',
+        path,
+        username='me',
+        async=True,
+        no_acoustid=True,
+        interactive=False)
+    batch = user.imports.latest('id')
+    job = batch.jobs.first()
+    m.assert_called_once_with(
+        music_tasks.import_job_run.delay,
+        import_job_id=job.pk,
+        use_acoustid=False)

demo/setup.sh

+#!/bin/bash -eux
+version="develop"
+music_path="/usr/share/music"
+demo_path="/srv/funkwhale-demo/demo"
+
+echo 'Cleaning everything...'
+cd $demo_path
+docker-compose down -v || echo 'Nothing to stop'
+rm -rf /srv/funkwhale-demo/demo/*
+mkdir -p $demo_path
+echo 'Downloading demo files...'
+curl -L -o docker-compose.yml "https://code.eliotberriot.com/funkwhale/funkwhale/raw/$version/deploy/docker-compose.yml"
+curl -L -o .env "https://code.eliotberriot.com/funkwhale/funkwhale/raw/$version/deploy/env.prod.sample"
+
+mkdir data/
+cp -r $music_path data/music
+
+curl -L -o front.zip "https://code.eliotberriot.com/funkwhale/funkwhale/-/jobs/artifacts/$version/download?job=build_front"
+unzip front.zip
+
+echo "FUNKWHALE_URL=https://demo.funkwhale.audio/" >> .env
+echo "DJANGO_SECRET_KEY=demo" >> .env
+echo "DJANGO_ALLOWED_HOSTS=demo.funkwhale.audio" >> .env
+echo "FUNKWHALE_VERSION=$version" >> .env
+echo "FUNKWHALE_API_PORT=5001" >> .env
+
+docker-compose pull
+docker-compose up -d postgres redis
+sleep 5
+docker-compose run --rm api demo/load-demo-data.sh
+docker-compose up -d