From d3dc770e7a0c4fc18d8d1fa9ddadf0af5191ebc4 Mon Sep 17 00:00:00 2001 From: Eliot Berriot <contact@eliotberriot.com> Date: Sun, 25 Dec 2016 21:00:13 +0100 Subject: [PATCH] Fixed #2: can now import whole artist albums using API --- funkwhale_api/music/tests/test_api.py | 66 ++- funkwhale_api/music/views.py | 31 +- funkwhale_api/musicbrainz/client.py | 12 +- funkwhale_api/musicbrainz/tests/data.py | 478 ++++++++++++++++++ funkwhale_api/musicbrainz/tests/test_api.py | 81 ++- funkwhale_api/musicbrainz/urls.py | 16 +- funkwhale_api/musicbrainz/views.py | 41 +- funkwhale_api/providers/youtube/client.py | 58 ++- funkwhale_api/providers/youtube/tests/data.py | 162 ++++++ .../providers/youtube/tests/test_youtube.py | 55 +- funkwhale_api/providers/youtube/urls.py | 5 +- funkwhale_api/providers/youtube/views.py | 9 + 12 files changed, 962 insertions(+), 52 deletions(-) create mode 100644 funkwhale_api/musicbrainz/tests/data.py create mode 100644 funkwhale_api/providers/youtube/tests/data.py diff --git a/funkwhale_api/music/tests/test_api.py b/funkwhale_api/music/tests/test_api.py index 36c1fa6..ff3deed 100644 --- a/funkwhale_api/music/tests/test_api.py +++ b/funkwhale_api/music/tests/test_api.py @@ -21,7 +21,7 @@ class TestAPI(TMPDirTestCaseMixin, TestCase): mbid = '9968a9d6-8d92-4051-8f76-674e157b6eed' video_id = 'tPEE9ZwTmy0' url = reverse('api:submit-single') - user = User.objects.create_user(username='test', email='test@test.com', password='test') + user = User.objects.create_superuser(username='test', email='test@test.com', password='test') self.client.login(username=user.username, password='test') response = self.client.post(url, {'import_url': 'https://www.youtube.com/watch?v={0}'.format(video_id), 'mbid': mbid}) track = models.Track.objects.get(mbid=mbid) @@ -30,7 +30,7 @@ class TestAPI(TMPDirTestCaseMixin, TestCase): # self.assertIn(video_id, track.files.first().audio_file.name) def test_import_creates_an_import_with_correct_data(self): - user = User.objects.create_user(username='test', email='test@test.com', password='test') + user = User.objects.create_superuser(username='test', email='test@test.com', password='test') mbid = '9968a9d6-8d92-4051-8f76-674e157b6eed' video_id = 'tPEE9ZwTmy0' url = reverse('api:submit-single') @@ -51,7 +51,7 @@ class TestAPI(TMPDirTestCaseMixin, TestCase): @unittest.mock.patch('funkwhale_api.musicbrainz.api.images.get_front', return_value=b'') @unittest.mock.patch('funkwhale_api.musicbrainz.api.releases.get', return_value=api_data.albums['get_with_includes']['hypnotize']) def test_can_import_whole_album(self, *mocks): - user = User.objects.create_user(username='test', email='test@test.com', password='test') + user = User.objects.create_superuser(username='test', email='test@test.com', password='test') payload = { 'releaseId': '47ae093f-1607-49a3-be11-a15d335ccc94', 'tracks': [ @@ -69,7 +69,6 @@ class TestAPI(TMPDirTestCaseMixin, TestCase): }, ] } - video_id = 'tPEE9ZwTmy0' url = reverse('api:submit-album') self.client.login(username=user.username, password='test') with self.settings(CELERY_ALWAYS_EAGER=False): @@ -97,10 +96,63 @@ class TestAPI(TMPDirTestCaseMixin, TestCase): self.assertEqual(job.status, 'pending') self.assertEqual(job.source, row['source']) + @unittest.mock.patch('funkwhale_api.musicbrainz.api.artists.get', return_value=api_data.artists['get']['soad']) + @unittest.mock.patch('funkwhale_api.musicbrainz.api.images.get_front', return_value=b'') + @unittest.mock.patch('funkwhale_api.musicbrainz.api.releases.get', return_value=api_data.albums['get_with_includes']['hypnotize']) + def test_can_import_whole_artist(self, *mocks): + user = User.objects.create_superuser(username='test', email='test@test.com', password='test') + payload = { + 'artistId': 'mbid', + 'albums': [ + { + 'releaseId': '47ae093f-1607-49a3-be11-a15d335ccc94', + 'tracks': [ + { + 'mbid': '1968a9d6-8d92-4051-8f76-674e157b6eed', + 'source': 'https://www.youtube.com/watch?v=1111111111', + }, + { + 'mbid': '2968a9d6-8d92-4051-8f76-674e157b6eed', + 'source': 'https://www.youtube.com/watch?v=2222222222', + }, + { + 'mbid': '3968a9d6-8d92-4051-8f76-674e157b6eed', + 'source': 'https://www.youtube.com/watch?v=3333333333', + }, + ] + } + ] + } + url = reverse('api:submit-artist') + self.client.login(username=user.username, password='test') + with self.settings(CELERY_ALWAYS_EAGER=False): + response = self.client.post(url, json.dumps(payload), content_type="application/json") + + batch = models.ImportBatch.objects.latest('id') + self.assertEqual(batch.jobs.count(), 3) + self.assertEqual(batch.submitted_by, user) + self.assertEqual(batch.status, 'pending') + + album = models.Album.objects.latest('id') + self.assertEqual(str(album.mbid), '47ae093f-1607-49a3-be11-a15d335ccc94') + medium_data = api_data.albums['get_with_includes']['hypnotize']['release']['medium-list'][0] + self.assertEqual(int(medium_data['track-count']), album.tracks.all().count()) + + for track in medium_data['track-list']: + instance = models.Track.objects.get(mbid=track['recording']['id']) + self.assertEqual(instance.title, track['recording']['title']) + self.assertEqual(instance.position, int(track['position'])) + self.assertEqual(instance.title, track['recording']['title']) + + for row in payload['albums'][0]['tracks']: + job = models.ImportJob.objects.get(mbid=row['mbid']) + self.assertEqual(str(job.mbid), row['mbid']) + self.assertEqual(job.status, 'pending') + self.assertEqual(job.source, row['source']) def test_user_can_query_api_for_his_own_batches(self): - user1 = User.objects.create_user(username='test1', email='test1@test.com', password='test') - user2 = User.objects.create_user(username='test2', email='test2@test.com', password='test') + user1 = User.objects.create_superuser(username='test1', email='test1@test.com', password='test') + user2 = User.objects.create_superuser(username='test2', email='test2@test.com', password='test') mbid = '9968a9d6-8d92-4051-8f76-674e157b6eed' source = 'https://www.youtube.com/watch?v=tPEE9ZwTmy0' @@ -154,7 +206,7 @@ class TestAPI(TMPDirTestCaseMixin, TestCase): response = getattr(self.client, method)(url) self.assertEqual(response.status_code, 401) - user = User.objects.create_user(username='test', email='test@test.com', password='test') + user = User.objects.create_superuser(username='test', email='test@test.com', password='test') self.client.login(username=user.username, password='test') for route_name, method in urls: diff --git a/funkwhale_api/music/views.py b/funkwhale_api/music/views.py index a2cceed..6510e57 100644 --- a/funkwhale_api/music/views.py +++ b/funkwhale_api/music/views.py @@ -6,6 +6,7 @@ from django.db.models.functions import Length from rest_framework import viewsets, views from rest_framework.decorators import detail_route, list_route from rest_framework.response import Response +from rest_framework import permissions from musicbrainzngs import ResponseError from django.contrib.auth.decorators import login_required from django.utils.decorators import method_decorator @@ -189,7 +190,11 @@ class Search(views.APIView): return qs.filter(query_obj)[:self.max_results] + class SubmitViewSet(viewsets.ViewSet): + queryset = models.ImportBatch.objects.none() + permission_classes = (permissions.DjangoModelPermissions, ) + @list_route(methods=['post']) @transaction.non_atomic_requests def single(self, request, *args, **kwargs): @@ -208,7 +213,10 @@ class SubmitViewSet(viewsets.ViewSet): @transaction.non_atomic_requests def album(self, request, *args, **kwargs): data = json.loads(request.body.decode('utf-8')) + import_data, batch = self._import_album(data, request, batch=None) + return Response(import_data) + def _import_album(self, data, request, batch=None): # we import the whole album here to prevent race conditions that occurs # when using get_or_create_from_api in tasks album_data = api.releases.get(id=data['releaseId'], includes=models.Album.api_includes)['release'] @@ -218,12 +226,29 @@ class SubmitViewSet(viewsets.ViewSet): album.get_image() except ResponseError: pass - batch = models.ImportBatch.objects.create(submitted_by=request.user) + if not batch: + batch = models.ImportBatch.objects.create(submitted_by=request.user) for row in data['tracks']: try: models.TrackFile.objects.get(track__mbid=row['mbid']) except models.TrackFile.DoesNotExist: job = models.ImportJob.objects.create(mbid=row['mbid'], batch=batch, source=row['source']) - job.run.delay() + # job.run.delay() serializer = serializers.ImportBatchSerializer(batch) - return Response(serializer.data) + return serializer.data, batch + + @list_route(methods=['post']) + @transaction.non_atomic_requests + def artist(self, request, *args, **kwargs): + data = json.loads(request.body.decode('utf-8')) + artist_data = api.artists.get(id=data['artistId'])['artist'] + cleaned_data = models.Artist.clean_musicbrainz_data(artist_data) + artist = importers.load(models.Artist, cleaned_data, artist_data, import_hooks=[]) + + import_data = [] + batch = None + for row in data['albums']: + row_data, batch = self._import_album(row, request, batch=batch) + import_data.append(row_data) + + return Response(import_data[0]) diff --git a/funkwhale_api/musicbrainz/client.py b/funkwhale_api/musicbrainz/client.py index 34eb9a9..e281555 100644 --- a/funkwhale_api/musicbrainz/client.py +++ b/funkwhale_api/musicbrainz/client.py @@ -6,11 +6,12 @@ _api = musicbrainzngs _api.set_useragent('funkwhale', str(__version__), 'contact@eliotberriot.com') -def clean_artist_search(**kwargs): +def clean_artist_search(query, **kwargs): cleaned_kwargs = {} if kwargs.get('name'): cleaned_kwargs['artist'] = kwargs.get('name') - return _api.search_artists(**cleaned_kwargs) + return _api.search_artists(query, **cleaned_kwargs) + class API(object): _api = _api @@ -33,6 +34,13 @@ class API(object): class releases(object): search = _api.search_releases get = _api.get_release_by_id + browse = _api.browse_releases + # get_image_front = _api.get_image_front + + class release_groups(object): + search = _api.search_release_groups + get = _api.get_release_group_by_id + browse = _api.browse_release_groups # get_image_front = _api.get_image_front api = API() diff --git a/funkwhale_api/musicbrainz/tests/data.py b/funkwhale_api/musicbrainz/tests/data.py new file mode 100644 index 0000000..1d7b9a3 --- /dev/null +++ b/funkwhale_api/musicbrainz/tests/data.py @@ -0,0 +1,478 @@ +artists = {'search': {}, 'get': {}} +artists['search']['lost fingers'] = { + 'artist-count': 696, + 'artist-list': [ + { + 'country': 'CA', + 'sort-name': 'Lost Fingers, The', + 'id': 'ac16bbc0-aded-4477-a3c3-1d81693d58c9', + 'type': 'Group', + 'life-span': { + 'ended': 'false', + 'begin': '2008' + }, + 'area': { + 'sort-name': 'Canada', + 'id': '71bbafaa-e825-3e15-8ca9-017dcad1748b', + 'name': 'Canada' + }, + 'ext:score': '100', + 'name': 'The Lost Fingers' + }, + ] +} +artists['get']['lost fingers'] = { + "artist": { + "life-span": { + "begin": "2008" + }, + "type": "Group", + "id": "ac16bbc0-aded-4477-a3c3-1d81693d58c9", + "release-group-count": 8, + "name": "The Lost Fingers", + "release-group-list": [ + { + "title": "Gypsy Kameleon", + "first-release-date": "2010", + "type": "Album", + "id": "03d3f1d4-e2b0-40d3-8314-05f1896e93a0", + "primary-type": "Album" + }, + { + "title": "Gitan Kameleon", + "first-release-date": "2011-11-11", + "type": "Album", + "id": "243c0cd2-2492-4f5d-bf37-c7c76bed05b7", + "primary-type": "Album" + }, + { + "title": "Pump Up the Jam \u2013 Do Not Cover, Pt. 3", + "first-release-date": "2014-03-17", + "type": "Single", + "id": "4429befd-ff45-48eb-a8f4-cdf7bf007f3f", + "primary-type": "Single" + }, + { + "title": "La Marquise", + "first-release-date": "2012-03-27", + "type": "Album", + "id": "4dab4b96-0a6b-4507-a31e-2189e3e7bad1", + "primary-type": "Album" + }, + { + "title": "Christmas Caravan", + "first-release-date": "2016-11-11", + "type": "Album", + "id": "ca0a506d-6ba9-47c3-a712-de5ce9ae6b1f", + "primary-type": "Album" + }, + { + "title": "Rendez-vous rose", + "first-release-date": "2009-06-16", + "type": "Album", + "id": "d002f1a8-5890-4188-be58-1caadbbd767f", + "primary-type": "Album" + }, + { + "title": "Wonders of the World", + "first-release-date": "2014-05-06", + "type": "Album", + "id": "eeb644c2-5000-42fb-b959-e5e9cc2901c5", + "primary-type": "Album" + }, + { + "title": "Lost in the 80s", + "first-release-date": "2008-05-06", + "type": "Album", + "id": "f04ed607-11b7-3843-957e-503ecdd485d1", + "primary-type": "Album" + } + ], + "area": { + "iso-3166-1-code-list": [ + "CA" + ], + "name": "Canada", + "id": "71bbafaa-e825-3e15-8ca9-017dcad1748b", + "sort-name": "Canada" + }, + "sort-name": "Lost Fingers, The", + "country": "CA" + } +} + + +release_groups = {'browse': {}} +release_groups['browse']["lost fingers"] = { + "release-group-list": [ + { + "first-release-date": "2010", + "type": "Album", + "primary-type": "Album", + "title": "Gypsy Kameleon", + "id": "03d3f1d4-e2b0-40d3-8314-05f1896e93a0" + }, + { + "first-release-date": "2011-11-11", + "type": "Album", + "primary-type": "Album", + "title": "Gitan Kameleon", + "id": "243c0cd2-2492-4f5d-bf37-c7c76bed05b7" + }, + { + "first-release-date": "2014-03-17", + "type": "Single", + "primary-type": "Single", + "title": "Pump Up the Jam \u2013 Do Not Cover, Pt. 3", + "id": "4429befd-ff45-48eb-a8f4-cdf7bf007f3f" + }, + { + "first-release-date": "2012-03-27", + "type": "Album", + "primary-type": "Album", + "title": "La Marquise", + "id": "4dab4b96-0a6b-4507-a31e-2189e3e7bad1" + }, + { + "first-release-date": "2016-11-11", + "type": "Album", + "primary-type": "Album", + "title": "Christmas Caravan", + "id": "ca0a506d-6ba9-47c3-a712-de5ce9ae6b1f" + }, + { + "first-release-date": "2009-06-16", + "type": "Album", + "primary-type": "Album", + "title": "Rendez-vous rose", + "id": "d002f1a8-5890-4188-be58-1caadbbd767f" + }, + { + "first-release-date": "2014-05-06", + "type": "Album", + "primary-type": "Album", + "title": "Wonders of the World", + "id": "eeb644c2-5000-42fb-b959-e5e9cc2901c5" + }, + { + "first-release-date": "2008-05-06", + "type": "Album", + "primary-type": "Album", + "title": "Lost in the 80s", + "id": "f04ed607-11b7-3843-957e-503ecdd485d1" + } + ], + "release-group-count": 8 +} + +recordings = {'search': {}, 'get': {}} +recordings['search']['brontide matador'] = { + "recording-count": 1044, + "recording-list": [ + { + "ext:score": "100", + "length": "366280", + "release-list": [ + { + "date": "2011-05-30", + "medium-track-count": 8, + "release-event-list": [ + { + "area": { + "name": "United Kingdom", + "sort-name": "United Kingdom", + "id": "8a754a16-0027-3a29-b6d7-2b40ea0481ed", + "iso-3166-1-code-list": ["GB"] + }, + "date": "2011-05-30" + } + ], + "country": "GB", + "title": "Sans Souci", + "status": "Official", + "id": "fde538c8-ffef-47c6-9b5a-bd28f4070e5c", + "release-group": { + "type": "Album", + "id": "113ab958-cfb8-4782-99af-639d4d9eae8d", + "primary-type": "Album" + }, + "medium-list": [ + { + "format": "CD", + "track-list": [ + { + "track_or_recording_length": "366280", + "id": "fe506782-a5cb-3d89-9b3e-86287be05768", + "length": "366280", + "title": "Matador", "number": "1" + } + ], + "position": "1", + "track-count": 8 + } + ] + }, + ] + } + ] +} + +releases = {'search': {}, 'get': {}, 'browse': {}} +releases['search']['brontide matador'] = { + "release-count": 116, "release-list": [ + { + "ext:score": "100", + "date": "2009-04-02", + "release-event-list": [ + { + "area": { + "name": "[Worldwide]", + "sort-name": "[Worldwide]", + "id": "525d4e18-3d00-31b9-a58b-a146a916de8f", + "iso-3166-1-code-list": ["XW"] + }, + "date": "2009-04-02" + } + ], + "label-info-list": [ + { + "label": { + "name": "Holy Roar", + "id": "6e940f35-961d-4ac3-bc2a-569fc211c2e3" + } + } + ], + "medium-track-count": 3, + "packaging": "None", + "artist-credit": [ + { + "artist": { + "name": "Brontide", + "sort-name": "Brontide", + "id": "2179fbd2-3c88-4b94-a778-eb3daf1e81a1" + } + } + ], + "artist-credit-phrase": "Brontide", + "country": "XW", + "title": "Brontide EP", + "status": "Official", + "barcode": "", + "id": "59fbd4d1-6121-40e3-9b76-079694fe9702", + "release-group": { + "type": "EP", + "secondary-type-list": ["Demo"], + "id": "b9207129-2d03-4a68-8a53-3c46fe7d2810", + "primary-type": "EP" + }, + "medium-list": [ + { + "disc-list": [], + "format": "Digital Media", + "disc-count": 0, + "track-count": 3, + "track-list": [] + } + ], + "medium-count": 1, + "text-representation": { + "script": "Latn", + "language": "eng" + } + }, + ] +} + +releases['browse']['Lost in the 80s'] = { + "release-count": 3, + "release-list": [ + { + "quality": "normal", + "status": "Official", + "text-representation": { + "script": "Latn", + "language": "eng" + }, + "title": "Lost in the 80s", + "date": "2008-05-06", + "release-event-count": 1, + "id": "34e27fa0-aad4-4cc5-83a3-0f97089154dc", + "barcode": "622406580223", + "medium-count": 1, + "release-event-list": [ + { + "area": { + "iso-3166-1-code-list": [ + "CA" + ], + "id": "71bbafaa-e825-3e15-8ca9-017dcad1748b", + "name": "Canada", + "sort-name": "Canada" + }, + "date": "2008-05-06" + } + ], + "country": "CA", + "cover-art-archive": { + "back": "false", + "artwork": "false", + "front": "false", + "count": "0" + }, + "medium-list": [ + { + "position": "1", + "track-count": 12, + "format": "CD", + "track-list": [ + { + "id": "1662bdf8-31d6-3f6e-846b-fe88c087b109", + "length": "228000", + "recording": { + "id": "2e0dbf37-65af-4408-8def-7b0b3cb8426b", + "length": "228000", + "title": "Pump Up the Jam" + }, + "track_or_recording_length": "228000", + "position": "1", + "number": "1" + }, + { + "id": "01a8cf99-2170-3d3f-96ef-5e4ef7a015a4", + "length": "231000", + "recording": { + "id": "57017e2e-625d-4e7b-a445-47cdb0224dd2", + "length": "231000", + "title": "You Give Love a Bad Name" + }, + "track_or_recording_length": "231000", + "position": "2", + "number": "2" + }, + { + "id": "375a7ce7-5a41-3fbf-9809-96d491401034", + "length": "189000", + "recording": { + "id": "a948672b-b42d-44a5-89b0-7e9ab6a7e11d", + "length": "189000", + "title": "You Shook Me All Night Long" + }, + "track_or_recording_length": "189000", + "position": "3", + "number": "3" + }, + { + "id": "ed7d823e-76da-31be-82a8-770288e27d32", + "length": "253000", + "recording": { + "id": "6e097e31-f37b-4fae-8ad0-ada57f3091a7", + "length": "253000", + "title": "Incognito" + }, + "track_or_recording_length": "253000", + "position": "4", + "number": "4" + }, + { + "id": "76ac8c77-6a99-34d9-ae4d-be8f056d50e0", + "length": "221000", + "recording": { + "id": "faa922e6-e834-44ee-8125-79e640a690e3", + "length": "221000", + "title": "Touch Me" + }, + "track_or_recording_length": "221000", + "position": "5", + "number": "5" + }, + { + "id": "d0a87409-2be6-3ab7-8526-4313e7134be1", + "length": "228000", + "recording": { + "id": "02da8148-60d8-4c79-ab31-8d90d233d711", + "length": "228000", + "title": "Part-Time Lover" + }, + "track_or_recording_length": "228000", + "position": "6", + "number": "6" + }, + { + "id": "02c5384b-5ca9-38e9-8b7c-c08dce608deb", + "length": "248000", + "recording": { + "id": "40085704-d6ab-44f6-a4d8-b27c9ca25b31", + "length": "248000", + "title": "Fresh" + }, + "track_or_recording_length": "248000", + "position": "7", + "number": "7" + }, + { + "id": "ab389542-53d5-346a-b168-1d915ecf0ef6", + "length": "257000", + "recording": { + "id": "77edd338-eeaf-4157-9e2a-5cc3bcee8abd", + "length": "257000", + "title": "Billie Jean" + }, + "track_or_recording_length": "257000", + "position": "8", + "number": "8" + }, + { + "id": "6d9e722b-7408-350e-bb7c-2de1e329ae84", + "length": "293000", + "recording": { + "id": "040aaffa-7206-40ff-9930-469413fe2420", + "length": "293000", + "title": "Careless Whisper" + }, + "track_or_recording_length": "293000", + "position": "9", + "number": "9" + }, + { + "id": "63b4e67c-7536-3cd0-8c47-0310c1e40866", + "length": "211000", + "recording": { + "id": "054942f0-4c0f-4e92-a606-d590976b1cff", + "length": "211000", + "title": "Tainted Love" + }, + "track_or_recording_length": "211000", + "position": "10", + "number": "10" + }, + { + "id": "a07f4ca3-dbf0-3337-a247-afcd0509334a", + "length": "245000", + "recording": { + "id": "8023b5ad-649a-4c67-b7a2-e12358606f6e", + "length": "245000", + "title": "Straight Up" + }, + "track_or_recording_length": "245000", + "position": "11", + "number": "11" + }, + { + "id": "73d47f16-b18d-36ff-b0bb-1fa1fd32ebf7", + "length": "322000", + "recording": { + "id": "95a8c8a1-fcb6-4cbb-a853-be86d816b357", + "length": "322000", + "title": "Black Velvet" + }, + "track_or_recording_length": "322000", + "position": "12", + "number": "12" + } + ] + } + ], + "asin": "B0017M8YTO" + }, + ] +} diff --git a/funkwhale_api/musicbrainz/tests/test_api.py b/funkwhale_api/musicbrainz/tests/test_api.py index 849f338..1734c94 100644 --- a/funkwhale_api/musicbrainz/tests/test_api.py +++ b/funkwhale_api/musicbrainz/tests/test_api.py @@ -1,22 +1,87 @@ import json +import unittest from test_plus.test import TestCase from django.core.urlresolvers import reverse from funkwhale_api.musicbrainz import api +from . import data as api_data + class TestAPI(TestCase): - def test_can_search_recording_in_musicbrainz_api(self): + @unittest.mock.patch( + 'funkwhale_api.musicbrainz.api.recordings.search', + return_value=api_data.recordings['search']['brontide matador']) + def test_can_search_recording_in_musicbrainz_api(self, *mocks): query = 'brontide matador' url = reverse('api:providers:musicbrainz:search-recordings') - expected = json.dumps(api.recordings.search(artist=query, recording=query)) - response = self.client.get(url + '?query={0}'.format(query)) + expected = api_data.recordings['search']['brontide matador'] + response = self.client.get(url, data={'query': query}) - self.assertJSONEqual(expected, json.loads(response.content.decode('utf-8'))) + self.assertEqual(expected, json.loads(response.content.decode('utf-8'))) - def test_can_search_release_in_musicbrainz_api(self): + @unittest.mock.patch( + 'funkwhale_api.musicbrainz.api.releases.search', + return_value=api_data.releases['search']['brontide matador']) + def test_can_search_release_in_musicbrainz_api(self, *mocks): query = 'brontide matador' url = reverse('api:providers:musicbrainz:search-releases') - expected = json.dumps(api.releases.search(query, artist=query)) - response = self.client.get(url + '?query={0}'.format(query)) + expected = api_data.releases['search']['brontide matador'] + response = self.client.get(url, data={'query': query}) + + self.assertEqual(expected, json.loads(response.content.decode('utf-8'))) + + @unittest.mock.patch( + 'funkwhale_api.musicbrainz.api.artists.search', + return_value=api_data.artists['search']['lost fingers']) + def test_can_search_artists_in_musicbrainz_api(self, *mocks): + query = 'lost fingers' + url = reverse('api:providers:musicbrainz:search-artists') + expected = api_data.artists['search']['lost fingers'] + response = self.client.get(url, data={'query': query}) + + self.assertEqual(expected, json.loads(response.content.decode('utf-8'))) + + @unittest.mock.patch( + 'funkwhale_api.musicbrainz.api.artists.get', + return_value=api_data.artists['get']['lost fingers']) + def test_can_get_artist_in_musicbrainz_api(self, *mocks): + uuid = 'ac16bbc0-aded-4477-a3c3-1d81693d58c9' + url = reverse('api:providers:musicbrainz:artist-detail', kwargs={ + 'uuid': uuid, + }) + response = self.client.get(url) + expected = api_data.artists['get']['lost fingers'] + + self.assertEqual(expected, json.loads(response.content.decode('utf-8'))) + + @unittest.mock.patch( + 'funkwhale_api.musicbrainz.api.release_groups.browse', + return_value=api_data.release_groups['browse']['lost fingers']) + def test_can_broswe_release_group_using_musicbrainz_api(self, *mocks): + uuid = 'ac16bbc0-aded-4477-a3c3-1d81693d58c9' + url = reverse( + 'api:providers:musicbrainz:release-group-browse', + kwargs={ + 'artist_uuid': uuid, + } + ) + response = self.client.get(url) + expected = api_data.release_groups['browse']['lost fingers'] + + self.assertEqual(expected, json.loads(response.content.decode('utf-8'))) + + @unittest.mock.patch( + 'funkwhale_api.musicbrainz.api.releases.browse', + return_value=api_data.releases['browse']['Lost in the 80s']) + def test_can_broswe_releases_using_musicbrainz_api(self, *mocks): + uuid = 'f04ed607-11b7-3843-957e-503ecdd485d1' + url = reverse( + 'api:providers:musicbrainz:release-browse', + kwargs={ + 'release_group_uuid': uuid, + } + ) + response = self.client.get(url) + expected = api_data.releases['browse']['Lost in the 80s'] - self.assertJSONEqual(expected, json.loads(response.content.decode('utf-8'))) + self.assertEqual(expected, json.loads(response.content.decode('utf-8'))) diff --git a/funkwhale_api/musicbrainz/urls.py b/funkwhale_api/musicbrainz/urls.py index 82f31a9..7befe49 100644 --- a/funkwhale_api/musicbrainz/urls.py +++ b/funkwhale_api/musicbrainz/urls.py @@ -6,5 +6,19 @@ from . import views router = routers.SimpleRouter() router.register(r'search', views.SearchViewSet, 'search') urlpatterns = [ - url('releases/(?P<uuid>[0-9a-z-]+)/$', views.ReleaseDetail.as_view(), name='release-detail') + url('releases/(?P<uuid>[0-9a-z-]+)/$', + views.ReleaseDetail.as_view(), + name='release-detail'), + url('artists/(?P<uuid>[0-9a-z-]+)/$', + views.ArtistDetail.as_view(), + name='artist-detail'), + url('release-groups/browse/(?P<artist_uuid>[0-9a-z-]+)/$', + views.ReleaseGroupBrowse.as_view(), + name='release-group-browse'), + url('releases/browse/(?P<release_group_uuid>[0-9a-z-]+)/$', + views.ReleaseBrowse.as_view(), + name='release-browse'), + # url('release-groups/(?P<uuid>[0-9a-z-]+)/$', + # views.ReleaseGroupDetail.as_view(), + # name='release-group-detail'), ] + router.urls diff --git a/funkwhale_api/musicbrainz/views.py b/funkwhale_api/musicbrainz/views.py index 21b2d88..fac8089 100644 --- a/funkwhale_api/musicbrainz/views.py +++ b/funkwhale_api/musicbrainz/views.py @@ -11,10 +11,43 @@ from .client import api class ReleaseDetail(APIView): permission_classes = [ConditionalAuthentication] + + def get(self, request, *args, **kwargs): + result = api.releases.get( + id=kwargs['uuid'], includes=['artists', 'recordings']) + return Response(result) + + +class ArtistDetail(APIView): + permission_classes = [ConditionalAuthentication] + + def get(self, request, *args, **kwargs): + result = api.artists.get( + id=kwargs['uuid'], + includes=['release-groups']) + # import json; print(json.dumps(result, indent=4)) + return Response(result) + + +class ReleaseGroupBrowse(APIView): + permission_classes = [ConditionalAuthentication] + + def get(self, request, *args, **kwargs): + result = api.release_groups.browse( + artist=kwargs['artist_uuid']) + return Response(result) + + +class ReleaseBrowse(APIView): + permission_classes = [ConditionalAuthentication] + def get(self, request, *args, **kwargs): - result = api.releases.get(id=kwargs['uuid'], includes=['artists', 'recordings']) + result = api.releases.browse( + release_group=kwargs['release_group_uuid'], + includes=['recordings']) return Response(result) + class SearchViewSet(viewsets.ViewSet): permission_classes = [ConditionalAuthentication] @@ -29,3 +62,9 @@ class SearchViewSet(viewsets.ViewSet): query = request.GET['query'] results = api.releases.search(query, artist=query) return Response(results) + + @list_route(methods=['get']) + def artists(self, request, *args, **kwargs): + query = request.GET['query'] + results = api.artists.search(query) + return Response(results) diff --git a/funkwhale_api/providers/youtube/client.py b/funkwhale_api/providers/youtube/client.py index afbd527..7c0ea32 100644 --- a/funkwhale_api/providers/youtube/client.py +++ b/funkwhale_api/providers/youtube/client.py @@ -1,3 +1,5 @@ +import threading + from apiclient.discovery import build from apiclient.errors import HttpError from oauth2client.tools import argparser @@ -13,40 +15,44 @@ YOUTUBE_API_SERVICE_NAME = "youtube" YOUTUBE_API_VERSION = "v3" VIDEO_BASE_URL = 'https://www.youtube.com/watch?v={0}' -class Client(object): - def search(self, query): - youtube = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION, - developerKey=DEVELOPER_KEY) +def _do_search(query): + youtube = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION, + developerKey=DEVELOPER_KEY) - # Call the search.list method to retrieve results matching the specified - # query term. - search_response = youtube.search().list( - q=query, - part="id,snippet", - maxResults=25 - ).execute() + return youtube.search().list( + q=query, + part="id,snippet", + maxResults=25 + ).execute() - videos = [] - # channels = [] - # playlists = [] - # Add each result to the appropriate list, and then display the lists of - # matching videos, channels, and playlists. +class Client(object): + + def search(self, query): + search_response = _do_search(query) + videos = [] for search_result in search_response.get("items", []): if search_result["id"]["kind"] == "youtube#video": search_result['full_url'] = VIDEO_BASE_URL.format(search_result["id"]['videoId']) videos.append(search_result) - # elif search_result["id"]["kind"] == "youtube#channel": - # channels.append("%s (%s)" % (search_result["snippet"]["title"], - # search_result["id"]["channelId"])) - # elif search_result["id"]["kind"] == "youtube#playlist": - # playlists.append("%s (%s)" % (search_result["snippet"]["title"], - # search_result["id"]["playlistId"])) - - # print "Videos:\n", "\n".join(videos), "\n" - # print "Channels:\n", "\n".join(channels), "\n" - # print "Playlists:\n", "\n".join(playlists), "\n" return videos + def search_multiple(self, queries): + results = {} + + def search(key, query): + results[key] = self.search(query) + + threads = [ + threading.Thread(target=search, args=(key, query,)) + for key, query in queries.items() + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + return results + client = Client() diff --git a/funkwhale_api/providers/youtube/tests/data.py b/funkwhale_api/providers/youtube/tests/data.py new file mode 100644 index 0000000..a8372d4 --- /dev/null +++ b/funkwhale_api/providers/youtube/tests/data.py @@ -0,0 +1,162 @@ + + +search = {} + + +search['8 bit adventure'] = { + "pageInfo": { + "totalResults": 1000000, + "resultsPerPage": 25 + }, + "nextPageToken": "CBkQAA", + "etag": "\"gMxXHe-zinKdE9lTnzKu8vjcmDI/1L34zetsKWv-raAFiz0MuT0SsfQ\"", + "items": [ + { + "id": { + "videoId": "0HxZn6CzOIo", + "kind": "youtube#video" + }, + "etag": "\"gMxXHe-zinKdE9lTnzKu8vjcmDI/GxK-wHBWUYfrJsd1dijBPTufrVE\"", + "snippet": { + "liveBroadcastContent": "none", + "description": "Make sure to apply adhesive evenly before use. GET IT HERE: http://adhesivewombat.bandcamp.com/album/marsupial-madness Facebook: ...", + "channelId": "UCps63j3krzAG4OyXeEyuhFw", + "title": "AdhesiveWombat - 8 Bit Adventure", + "channelTitle": "AdhesiveWombat", + "publishedAt": "2012-08-22T18:41:03.000Z", + "thumbnails": { + "medium": { + "url": "https://i.ytimg.com/vi/0HxZn6CzOIo/mqdefault.jpg", + "height": 180, + "width": 320 + }, + "high": { + "url": "https://i.ytimg.com/vi/0HxZn6CzOIo/hqdefault.jpg", + "height": 360, + "width": 480 + }, + "default": { + "url": "https://i.ytimg.com/vi/0HxZn6CzOIo/default.jpg", + "height": 90, + "width": 120 + } + } + }, + "kind": "youtube#searchResult" + }, + { + "id": { + "videoId": "n4A_F5SXmgo", + "kind": "youtube#video" + }, + "etag": "\"gMxXHe-zinKdE9lTnzKu8vjcmDI/aRVESw24jlgiErDgJKxNrazKRDc\"", + "snippet": { + "liveBroadcastContent": "none", + "description": "Free Download: http://bit.ly/1fZ1pMJ I don't post 8 bit'ish music much but damn I must admit this is goood! Enjoy \u2665 \u25bbSpikedGrin: ...", + "channelId": "UCMOgdURr7d8pOVlc-alkfRg", + "title": "\u3010Electro\u3011AdhesiveWombat - 8 Bit Adventure (SpikedGrin Remix) [Free Download]", + "channelTitle": "xKito Music", + "publishedAt": "2013-10-27T13:16:48.000Z", + "thumbnails": { + "medium": { + "url": "https://i.ytimg.com/vi/n4A_F5SXmgo/mqdefault.jpg", + "height": 180, + "width": 320 + }, + "high": { + "url": "https://i.ytimg.com/vi/n4A_F5SXmgo/hqdefault.jpg", + "height": 360, + "width": 480 + }, + "default": { + "url": "https://i.ytimg.com/vi/n4A_F5SXmgo/default.jpg", + "height": 90, + "width": 120 + } + } + }, + "kind": "youtube#searchResult" + }, + ], + "regionCode": "FR", + "kind": "youtube#searchListResponse" +} + +search['system of a down toxicity'] = { + "items": [ + { + "id": { + "kind": "youtube#video", + "videoId": "BorYwGi2SJc" + }, + "kind": "youtube#searchResult", + "snippet": { + "title": "System of a Down: Toxicity", + "channelTitle": "Vedres Csaba", + "channelId": "UCBXeuQORNwPv4m68fgGMtPQ", + "thumbnails": { + "default": { + "height": 90, + "width": 120, + "url": "https://i.ytimg.com/vi/BorYwGi2SJc/default.jpg" + }, + "high": { + "height": 360, + "width": 480, + "url": "https://i.ytimg.com/vi/BorYwGi2SJc/hqdefault.jpg" + }, + "medium": { + "height": 180, + "width": 320, + "url": "https://i.ytimg.com/vi/BorYwGi2SJc/mqdefault.jpg" + } + }, + "publishedAt": "2007-12-17T12:39:54.000Z", + "description": "http://www.vedrescsaba.uw.hu The System of a Down song Toxicity arranged for a classical piano quintet, played by Vedres Csaba and the Kairosz quartet.", + "liveBroadcastContent": "none" + }, + "etag": "\"gMxXHe-zinKdE9lTnzKu8vjcmDI/UwR8H6P6kbijNZmBNkYd2jAzDnI\"" + }, + { + "id": { + "kind": "youtube#video", + "videoId": "ENBv2i88g6Y" + }, + "kind": "youtube#searchResult", + "snippet": { + "title": "System Of A Down - Question!", + "channelTitle": "systemofadownVEVO", + "channelId": "UCvtZDkeFxMkRTNqfqXtxxkw", + "thumbnails": { + "default": { + "height": 90, + "width": 120, + "url": "https://i.ytimg.com/vi/ENBv2i88g6Y/default.jpg" + }, + "high": { + "height": 360, + "width": 480, + "url": "https://i.ytimg.com/vi/ENBv2i88g6Y/hqdefault.jpg" + }, + "medium": { + "height": 180, + "width": 320, + "url": "https://i.ytimg.com/vi/ENBv2i88g6Y/mqdefault.jpg" + } + }, + "publishedAt": "2009-10-03T04:49:03.000Z", + "description": "System of a Down's official music video for 'Question!'. Click to listen to System of a Down on Spotify: http://smarturl.it/SystemSpotify?IQid=SystemQu As featured ...", + "liveBroadcastContent": "none" + }, + "etag": "\"gMxXHe-zinKdE9lTnzKu8vjcmDI/dB-M0N9mB4xE-k4yAF_4d8aU0I4\"" + }, + ], + "etag": "\"gMxXHe-zinKdE9lTnzKu8vjcmDI/yhLQgSpeObNnybd5JqSzlGiJ8Ew\"", + "nextPageToken": "CBkQAA", + "pageInfo": { + "resultsPerPage": 25, + "totalResults": 26825 + }, + "kind": "youtube#searchListResponse", + "regionCode": "FR" +} diff --git a/funkwhale_api/providers/youtube/tests/test_youtube.py b/funkwhale_api/providers/youtube/tests/test_youtube.py index 48087f9..56b87a3 100644 --- a/funkwhale_api/providers/youtube/tests/test_youtube.py +++ b/funkwhale_api/providers/youtube/tests/test_youtube.py @@ -1,12 +1,18 @@ import json +from collections import OrderedDict +import unittest from test_plus.test import TestCase from django.core.urlresolvers import reverse from funkwhale_api.providers.youtube.client import client +from . import data as api_data class TestAPI(TestCase): - def test_can_get_search_results_from_youtube(self): + @unittest.mock.patch( + 'funkwhale_api.providers.youtube.client._do_search', + return_value=api_data.search['8 bit adventure']) + def test_can_get_search_results_from_youtube(self, *mocks): query = '8 bit adventure' results = client.search(query) @@ -14,10 +20,55 @@ class TestAPI(TestCase): self.assertEqual(results[0]['snippet']['title'], 'AdhesiveWombat - 8 Bit Adventure') self.assertEqual(results[0]['full_url'], 'https://www.youtube.com/watch?v=0HxZn6CzOIo') - def test_can_get_search_results_from_funkwhale(self): + @unittest.mock.patch( + 'funkwhale_api.providers.youtube.client._do_search', + return_value=api_data.search['8 bit adventure']) + def test_can_get_search_results_from_funkwhale(self, *mocks): query = '8 bit adventure' expected = json.dumps(client.search(query)) url = self.reverse('api:providers:youtube:search') response = self.client.get(url + '?query={0}'.format(query)) self.assertJSONEqual(expected, json.loads(response.content.decode('utf-8'))) + + @unittest.mock.patch( + 'funkwhale_api.providers.youtube.client._do_search', + side_effect=[ + api_data.search['8 bit adventure'], + api_data.search['system of a down toxicity'], + ] + ) + def test_can_send_multiple_queries_at_once(self, *mocks): + queries = OrderedDict() + queries['1'] = { + 'q': '8 bit adventure', + } + queries['2'] = { + 'q': 'system of a down toxicity', + } + + results = client.search_multiple(queries) + + self.assertEqual(results['1'][0]['id']['videoId'], '0HxZn6CzOIo') + self.assertEqual(results['1'][0]['snippet']['title'], 'AdhesiveWombat - 8 Bit Adventure') + self.assertEqual(results['1'][0]['full_url'], 'https://www.youtube.com/watch?v=0HxZn6CzOIo') + self.assertEqual(results['2'][0]['id']['videoId'], 'BorYwGi2SJc') + self.assertEqual(results['2'][0]['snippet']['title'], 'System of a Down: Toxicity') + self.assertEqual(results['2'][0]['full_url'], 'https://www.youtube.com/watch?v=BorYwGi2SJc') + + @unittest.mock.patch( + 'funkwhale_api.providers.youtube.client._do_search', + return_value=api_data.search['8 bit adventure'], + ) + def test_can_send_multiple_queries_at_once_from_funwkhale(self, *mocks): + queries = OrderedDict() + queries['1'] = { + 'q': '8 bit adventure', + } + + expected = json.dumps(client.search_multiple(queries)) + url = self.reverse('api:providers:youtube:searchs') + response = self.client.post( + url, json.dumps(queries), content_type='application/json') + + self.assertJSONEqual(expected, json.loads(response.content.decode('utf-8'))) diff --git a/funkwhale_api/providers/youtube/urls.py b/funkwhale_api/providers/youtube/urls.py index fc37888..243d2b8 100644 --- a/funkwhale_api/providers/youtube/urls.py +++ b/funkwhale_api/providers/youtube/urls.py @@ -1,7 +1,8 @@ from django.conf.urls import include, url -from .views import APISearch +from .views import APISearch, APISearchs urlpatterns = [ - url(r'^search/$', APISearch.as_view(), name='search') + url(r'^search/$', APISearch.as_view(), name='search'), + url(r'^searchs/$', APISearchs.as_view(), name='searchs'), ] diff --git a/funkwhale_api/providers/youtube/views.py b/funkwhale_api/providers/youtube/views.py index 9e909d7..7ad2c2c 100644 --- a/funkwhale_api/providers/youtube/views.py +++ b/funkwhale_api/providers/youtube/views.py @@ -4,9 +4,18 @@ from funkwhale_api.common.permissions import ConditionalAuthentication from .client import client + class APISearch(APIView): permission_classes = [ConditionalAuthentication] def get(self, request, *args, **kwargs): results = client.search(request.GET['query']) return Response(results) + + +class APISearchs(APIView): + permission_classes = [ConditionalAuthentication] + + def post(self, request, *args, **kwargs): + results = client.search_multiple(request.data) + return Response(results) -- GitLab