Commit 30d6195e authored by Agate's avatar Agate 💬

Merge branch 'release/0.2'

parents e45edadc 0b01bf30
Pipeline #134 passed with stages
in 3 minutes and 1 second
BACKEND_URL=http://localhost:12081
YOUTUBE_API_KEY=
API_AUTHENTICATION_REQUIRED=False
BACKEND_URL=http://localhost:6001
API_AUTHENTICATION_REQUIRED=True
CACHALOT_ENABLED=False
......@@ -70,6 +70,7 @@ docker_develop:
stage: deploy
before_script:
- docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
- cp -r front/dist api/frontend
- cd api
script:
- docker build -t $IMAGE .
......@@ -83,6 +84,7 @@ docker_release:
stage: deploy
before_script:
- docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
- cp -r front/dist api/frontend
- cd api
script:
- docker build -t $IMAGE -t $IMAGE_LATEST .
......
......@@ -9,10 +9,15 @@ export REDIS_URL=redis://redis:6379/0
# the official postgres image uses 'postgres' as default user if not set explictly.
if [ -z "$POSTGRES_ENV_POSTGRES_USER" ]; then
export POSTGRES_ENV_POSTGRES_USER=postgres
fi
fi
export DATABASE_URL=postgres://$POSTGRES_ENV_POSTGRES_USER:$POSTGRES_ENV_POSTGRES_PASSWORD@postgres:5432/$POSTGRES_ENV_POSTGRES_USER
export CELERY_BROKER_URL=$REDIS_URL
exec "$@"
\ No newline at end of file
# we copy the frontend files, if any so we can serve them from the outside
if [ -d "frontend" ]; then
mkdir -p /frontend
cp -r frontend/* /frontend/
fi
exec "$@"
FROM nginx:latest
ADD nginx.conf /etc/nginx/nginx.conf
\ No newline at end of file
......@@ -4,26 +4,40 @@ from funkwhale_api.music import views
from funkwhale_api.playlists import views as playlists_views
from rest_framework_jwt import views as jwt_views
from dynamic_preferences.api.viewsets import GlobalPreferencesViewSet
from dynamic_preferences.users.viewsets import UserPreferencesViewSet
router = routers.SimpleRouter()
router.register(r'settings', GlobalPreferencesViewSet, base_name='settings')
router.register(r'tags', views.TagViewSet, 'tags')
router.register(r'tracks', views.TrackViewSet, 'tracks')
router.register(r'trackfiles', views.TrackFileViewSet, 'trackfiles')
router.register(r'artists', views.ArtistViewSet, 'artists')
router.register(r'albums', views.AlbumViewSet, 'albums')
router.register(r'import-batches', views.ImportBatchViewSet, 'import-batches')
router.register(r'submit', views.SubmitViewSet, 'submit')
router.register(r'playlists', playlists_views.PlaylistViewSet, 'playlists')
router.register(r'playlist-tracks', playlists_views.PlaylistTrackViewSet, 'playlist-tracks')
router.register(
r'playlist-tracks',
playlists_views.PlaylistTrackViewSet,
'playlist-tracks')
v1_patterns = router.urls
v1_patterns += [
url(r'^providers/', include('funkwhale_api.providers.urls', namespace='providers')),
url(r'^favorites/', include('funkwhale_api.favorites.urls', namespace='favorites')),
url(r'^search$', views.Search.as_view(), name='search'),
url(r'^radios/', include('funkwhale_api.radios.urls', namespace='radios')),
url(r'^history/', include('funkwhale_api.history.urls', namespace='history')),
url(r'^users/', include('funkwhale_api.users.api_urls', namespace='users')),
url(r'^token/', jwt_views.obtain_jwt_token),
url(r'^providers/',
include('funkwhale_api.providers.urls', namespace='providers')),
url(r'^favorites/',
include('funkwhale_api.favorites.urls', namespace='favorites')),
url(r'^search$',
views.Search.as_view(), name='search'),
url(r'^radios/',
include('funkwhale_api.radios.urls', namespace='radios')),
url(r'^history/',
include('funkwhale_api.history.urls', namespace='history')),
url(r'^users/',
include('funkwhale_api.users.api_urls', namespace='users')),
url(r'^token/',
jwt_views.obtain_jwt_token),
url(r'^token/refresh/', jwt_views.refresh_jwt_token),
]
......
......@@ -53,6 +53,7 @@ THIRD_PARTY_APPS = (
'rest_auth',
'rest_auth.registration',
'mptt',
'dynamic_preferences',
)
# Apps specific for this project go here.
......@@ -65,6 +66,7 @@ LOCAL_APPS = (
'funkwhale_api.history',
'funkwhale_api.playlists',
'funkwhale_api.providers.audiofile',
'funkwhale_api.providers.youtube',
)
# See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
......@@ -217,7 +219,6 @@ STATICFILES_FINDERS = (
# See: https://docs.djangoproject.com/en/dev/ref/settings/#media-root
MEDIA_ROOT = str(APPS_DIR('media'))
USE_SAMPLE_TRACK = env.bool("USE_SAMPLE_TRACK", False)
# See: https://docs.djangoproject.com/en/dev/ref/settings/#media-url
......@@ -261,7 +262,6 @@ BROKER_URL = env("CELERY_BROKER_URL", default='django://')
# Location of root django.contrib.admin URL, use {% url 'admin:index' %}
ADMIN_URL = r'^admin/'
SESSION_SAVE_EVERY_REQUEST = True
# Your common stuff: Below this line define 3rd party library settings
CELERY_DEFAULT_RATE_LIMIT = 1
CELERYD_TASK_TIME_LIMIT = 300
......@@ -290,6 +290,7 @@ REST_FRAMEWORK = {
'PAGE_SIZE': 25,
'DEFAULT_AUTHENTICATION_CLASSES': (
'funkwhale_api.common.authentication.JSONWebTokenAuthenticationQS',
'rest_framework_jwt.authentication.JSONWebTokenAuthentication',
'rest_framework.authentication.SessionAuthentication',
'rest_framework.authentication.BasicAuthentication',
......@@ -299,9 +300,24 @@ REST_FRAMEWORK = {
)
}
FUNKWHALE_PROVIDERS = {
'youtube': {
'api_key': env('YOUTUBE_API_KEY', default='REPLACE_ME')
}
}
ATOMIC_REQUESTS = False
# Wether we should check user permission before serving audio files (meaning
# return an obfuscated url)
# This require a special configuration on the reverse proxy side
# See https://wellfire.co/learn/nginx-django-x-accel-redirects/ for example
PROTECT_AUDIO_FILES = env.bool('PROTECT_AUDIO_FILES', default=True)
# Which path will be used to process the internal redirection
# **DO NOT** put a slash at the end
PROTECT_FILES_PATH = env('PROTECT_FILES_PATH', default='/_protected')
# use this setting to tweak for how long you want to cache
# musicbrainz results. (value is in seconds)
MUSICBRAINZ_CACHE_DURATION = env.int(
'MUSICBRAINZ_CACHE_DURATION',
default=300
)
CACHALOT_ENABLED = env.bool('CACHALOT_ENABLED', default=True)
# -*- coding: utf-8 -*-
__version__ = '0.1.0'
__version__ = '0.2.0'
__version_info__ = tuple([int(num) if num.isdigit() else num for num in __version__.replace('-', '.', 1).split('.')])
from rest_framework import exceptions
from rest_framework_jwt import authentication
from rest_framework_jwt.settings import api_settings
class JSONWebTokenAuthenticationQS(
authentication.BaseJSONWebTokenAuthentication):
www_authenticate_realm = 'api'
def get_jwt_value(self, request):
token = request.query_params.get('jwt')
if 'jwt' in request.query_params and not token:
msg = _('Invalid Authorization header. No credentials provided.')
raise exceptions.AuthenticationFailed(msg)
return token
def authenticate_header(self, request):
return '{0} realm="{1}"'.format(
api_settings.JWT_AUTH_HEADER_PREFIX, self.www_authenticate_realm)
from test_plus.test import TestCase
from rest_framework_jwt.settings import api_settings
from funkwhale_api.users.models import User
jwt_payload_handler = api_settings.JWT_PAYLOAD_HANDLER
jwt_encode_handler = api_settings.JWT_ENCODE_HANDLER
class TestJWTQueryString(TestCase):
www_authenticate_realm = 'api'
def test_can_authenticate_using_token_param_in_url(self):
user = User.objects.create_superuser(
username='test', email='test@test.com', password='test')
url = self.reverse('api:v1:tracks-list')
with self.settings(API_AUTHENTICATION_REQUIRED=True):
response = self.client.get(url)
self.assertEqual(response.status_code, 401)
payload = jwt_payload_handler(user)
token = jwt_encode_handler(payload)
print(payload, token)
with self.settings(API_AUTHENTICATION_REQUIRED=True):
response = self.client.get(url, data={
'jwt': token
})
self.assertEqual(response.status_code, 200)
......@@ -37,13 +37,26 @@ def get_mp3_recording_id(f, k):
except IndexError:
raise TagNotFound(k)
def convert_track_number(v):
try:
return int(v)
except ValueError:
# maybe the position is of the form "1/4"
pass
try:
return int(v.split('/')[0])
except (ValueError, AttributeError, IndexError):
pass
CONF = {
'OggVorbis': {
'getter': lambda f, k: f[k][0],
'fields': {
'track_number': {
'field': 'TRACKNUMBER',
'to_application': int
'to_application': convert_track_number
},
'title': {
'field': 'title'
......@@ -74,7 +87,7 @@ CONF = {
'fields': {
'track_number': {
'field': 'TPOS',
'to_application': lambda v: int(v.split('/')[0])
'to_application': convert_track_number
},
'title': {
'field': 'TIT2'
......
......@@ -8,7 +8,6 @@ import markdown
from django.conf import settings
from django.db import models
from django.contrib.staticfiles.templatetags.staticfiles import static
from django.core.files.base import ContentFile
from django.core.files import File
from django.core.urlresolvers import reverse
......@@ -354,10 +353,12 @@ class TrackFile(models.Model):
@property
def path(self):
if settings.USE_SAMPLE_TRACK:
return static('music/sample1.ogg')
if settings.PROTECT_AUDIO_FILES:
return reverse(
'api:v1:trackfiles-serve', kwargs={'pk': self.pk})
return self.audio_file.url
class ImportBatch(models.Model):
creation_date = models.DateTimeField(default=timezone.now)
submitted_by = models.ForeignKey('users.User', related_name='imports')
......
import factory
class ArtistFactory(factory.django.DjangoModelFactory):
name = factory.Sequence(lambda n: 'artist-{0}'.format(n))
mbid = factory.Faker('uuid4')
class Meta:
model = 'music.Artist'
class AlbumFactory(factory.django.DjangoModelFactory):
title = factory.Sequence(lambda n: 'album-{0}'.format(n))
mbid = factory.Faker('uuid4')
release_date = factory.Faker('date')
cover = factory.django.ImageField()
artist = factory.SubFactory(ArtistFactory)
class Meta:
model = 'music.Album'
class TrackFactory(factory.django.DjangoModelFactory):
title = factory.Sequence(lambda n: 'track-{0}'.format(n))
mbid = factory.Faker('uuid4')
album = factory.SubFactory(AlbumFactory)
artist = factory.SelfAttribute('album.artist')
position = 1
class Meta:
model = 'music.Track'
class TrackFileFactory(factory.django.DjangoModelFactory):
track = factory.SubFactory(TrackFactory)
audio_file = factory.django.FileField()
class Meta:
model = 'music.TrackFile'
......@@ -10,6 +10,8 @@ from funkwhale_api.music import serializers
from funkwhale_api.users.models import User
from . import data as api_data
from . import factories
class TestAPI(TMPDirTestCaseMixin, TestCase):
......@@ -214,3 +216,26 @@ class TestAPI(TMPDirTestCaseMixin, TestCase):
with self.settings(API_AUTHENTICATION_REQUIRED=False):
response = getattr(self.client, method)(url)
self.assertEqual(response.status_code, 200)
def test_track_file_url_is_restricted_to_authenticated_users(self):
f = factories.TrackFileFactory()
self.assertNotEqual(f.audio_file, None)
url = f.path
with self.settings(API_AUTHENTICATION_REQUIRED=True):
response = self.client.get(url)
self.assertEqual(response.status_code, 401)
user = User.objects.create_superuser(
username='test', email='test@test.com', password='test')
self.client.login(username=user.username, password='test')
with self.settings(API_AUTHENTICATION_REQUIRED=True):
response = self.client.get(url)
self.assertEqual(response.status_code, 200)
self.assertEqual(
response['X-Accel-Redirect'],
'/_protected{}'.format(f.audio_file.url)
)
import os
import json
import unicodedata
import urllib
from django.core.urlresolvers import reverse
from django.db import models, transaction
from django.db.models.functions import Length
from django.conf import settings
from rest_framework import viewsets, views
from rest_framework.decorators import detail_route, list_route
from rest_framework.response import Response
......@@ -51,6 +54,7 @@ class ArtistViewSet(SearchMixin, viewsets.ReadOnlyModelViewSet):
search_fields = ['name']
ordering_fields = ('creation_date',)
class AlbumViewSet(SearchMixin, viewsets.ReadOnlyModelViewSet):
queryset = (
models.Album.objects.all()
......@@ -63,6 +67,7 @@ class AlbumViewSet(SearchMixin, viewsets.ReadOnlyModelViewSet):
search_fields = ['title']
ordering_fields = ('creation_date',)
class ImportBatchViewSet(viewsets.ReadOnlyModelViewSet):
queryset = models.ImportBatch.objects.all().order_by('-creation_date')
serializer_class = serializers.ImportBatchSerializer
......@@ -70,6 +75,7 @@ class ImportBatchViewSet(viewsets.ReadOnlyModelViewSet):
def get_queryset(self):
return super().get_queryset().filter(submitted_by=self.request.user)
class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet):
"""
A simple ViewSet for viewing and editing accounts.
......@@ -120,6 +126,29 @@ class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet):
return Response(serializer.data)
class TrackFileViewSet(viewsets.ReadOnlyModelViewSet):
queryset = (models.TrackFile.objects.all().order_by('-id'))
serializer_class = serializers.TrackFileSerializer
permission_classes = [ConditionalAuthentication]
@detail_route(methods=['get'])
def serve(self, request, *args, **kwargs):
try:
f = models.TrackFile.objects.get(pk=kwargs['pk'])
except models.TrackFile.DoesNotExist:
return Response(status=404)
response = Response()
filename = "filename*=UTF-8''{}{}".format(
urllib.parse.quote(f.track.full_name),
os.path.splitext(f.audio_file.name)[-1])
response["Content-Disposition"] = "attachment; {}".format(filename)
response['X-Accel-Redirect'] = "{}{}".format(
settings.PROTECT_FILES_PATH,
f.audio_file.url)
return response
class TagViewSet(viewsets.ReadOnlyModelViewSet):
queryset = Tag.objects.all().order_by('name')
serializer_class = serializers.TagSerializer
......
import musicbrainzngs
import memoize.djangocache
from django.conf import settings
from funkwhale_api import __version__
_api = musicbrainzngs
_api.set_useragent('funkwhale', str(__version__), 'contact@eliotberriot.com')
store = memoize.djangocache.Cache('default')
memo = memoize.Memoizer(store, namespace='memoize:musicbrainz')
def clean_artist_search(query, **kwargs):
cleaned_kwargs = {}
if kwargs.get('name'):
......@@ -17,30 +23,55 @@ class API(object):
_api = _api
class artists(object):
search = clean_artist_search
get = _api.get_artist_by_id
search = memo(
clean_artist_search, max_age=settings.MUSICBRAINZ_CACHE_DURATION)
get = memo(
_api.get_artist_by_id,
max_age=settings.MUSICBRAINZ_CACHE_DURATION)
class images(object):
get_front = _api.get_image_front
get_front = memo(
_api.get_image_front,
max_age=settings.MUSICBRAINZ_CACHE_DURATION)
class recordings(object):
search = _api.search_recordings
get = _api.get_recording_by_id
search = memo(
_api.search_recordings,
max_age=settings.MUSICBRAINZ_CACHE_DURATION)
get = memo(
_api.get_recording_by_id,
max_age=settings.MUSICBRAINZ_CACHE_DURATION)
class works(object):
search = _api.search_works
get = _api.get_work_by_id
search = memo(
_api.search_works,
max_age=settings.MUSICBRAINZ_CACHE_DURATION)
get = memo(
_api.get_work_by_id,
max_age=settings.MUSICBRAINZ_CACHE_DURATION)
class releases(object):
search = _api.search_releases
get = _api.get_release_by_id
browse = _api.browse_releases
search = memo(
_api.search_releases,
max_age=settings.MUSICBRAINZ_CACHE_DURATION)
get = memo(
_api.get_release_by_id,
max_age=settings.MUSICBRAINZ_CACHE_DURATION)
browse = memo(
_api.browse_releases,
max_age=settings.MUSICBRAINZ_CACHE_DURATION)
# get_image_front = _api.get_image_front
class release_groups(object):
search = _api.search_release_groups
get = _api.get_release_group_by_id
browse = _api.browse_release_groups
search = memo(
_api.search_release_groups,
max_age=settings.MUSICBRAINZ_CACHE_DURATION)
get = memo(
_api.get_release_group_by_id,
max_age=settings.MUSICBRAINZ_CACHE_DURATION)
browse = memo(
_api.browse_release_groups,
max_age=settings.MUSICBRAINZ_CACHE_DURATION)
# get_image_front = _api.get_image_front
api = API()
import unittest
from test_plus.test import TestCase
from funkwhale_api.musicbrainz import client
class TestAPI(TestCase):
def test_can_search_recording_in_musicbrainz_api(self, *mocks):
r = {'hello': 'world'}
mocked = 'funkwhale_api.musicbrainz.client._api.search_artists'
with unittest.mock.patch(mocked, return_value=r) as m:
self.assertEqual(client.api.artists.search('test'), r)
# now call from cache
self.assertEqual(client.api.artists.search('test'), r)
self.assertEqual(client.api.artists.search('test'), r)
self.assertEqual(m.call_count, 1)
......@@ -2,6 +2,7 @@ from rest_framework import viewsets
from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework.decorators import list_route
import musicbrainzngs
from funkwhale_api.common.permissions import ConditionalAuthentication
......@@ -44,7 +45,7 @@ class ReleaseBrowse(APIView):
def get(self, request, *args, **kwargs):
result = api.releases.browse(
release_group=kwargs['release_group_uuid'],
includes=['recordings'])
includes=['recordings', 'artist-credits'])
return Response(result)
......@@ -54,17 +55,18 @@ class SearchViewSet(viewsets.ViewSet):
@list_route(methods=['get'])
def recordings(self, request, *args, **kwargs):
query = request.GET['query']
results = api.recordings.search(query, artist=query)
results = api.recordings.search(query)
return Response(results)
@list_route(methods=['get'])
def releases(self, request, *args, **kwargs):
query = request.GET['query']
results = api.releases.search(query, artist=query)
results = api.releases.search(query)
return Response(results)
@list_route(methods=['get'])
def artists(self, request, *args, **kwargs):
query = request.GET['query']
results = api.artists.search(query)
# results = musicbrainzngs.search_artists(query)
return Response(results)
......@@ -4,21 +4,20 @@ from apiclient.discovery import build
from apiclient.errors import HttpError
from oauth2client.tools import argparser
from django.conf import settings
from dynamic_preferences.registries import (
global_preferences_registry as registry)
# Set DEVELOPER_KEY to the API key value from the APIs & auth > Registered apps
# tab of
# https://cloud.google.com/console
# Please ensure that you have enabled the YouTube Data API for your project.
DEVELOPER_KEY = settings.FUNKWHALE_PROVIDERS['youtube']['api_key']
YOUTUBE_API_SERVICE_NAME = "youtube"
YOUTUBE_API_VERSION = "v3"
VIDEO_BASE_URL = 'https://www.youtube.com/watch?v={0}'
def _do_search(query):
youtube = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION,
developerKey=DEVELOPER_KEY)
manager = registry.manager()
youtube = build(
YOUTUBE_API_SERVICE_NAME,
YOUTUBE_API_VERSION,
developerKey=manager['providers_youtube__api_key'])
return youtube.search().list(
q=query,
......@@ -55,4 +54,33 @@ class Client(object):
return results
def to_funkwhale(self, result):
"""
We convert youtube results to something more generic.
{
"id": "video id",
"type": "youtube#video",
"url": "https://www.youtube.com/watch?v=id",
"description": "description",
"channelId": "Channel id",
"title": "Title",
"channelTitle": "channel Title",
"publishedAt": "2012-08-22T18:41:03.000Z",