From eded32c2e8271cfffb9575a123f0caabdcc851f9 Mon Sep 17 00:00:00 2001
From: Eliot Berriot <contact@eliotberriot.com>
Date: Thu, 24 May 2018 18:53:12 +0200
Subject: [PATCH] See #228: more performante federation import launch via API
---
api/funkwhale_api/federation/serializers.py | 12 ++++++------
api/funkwhale_api/music/tasks.py | 7 +++++++
api/tests/federation/test_views.py | 4 +++-
api/tests/music/test_tasks.py | 9 +++++++++
4 files changed, 25 insertions(+), 7 deletions(-)
diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py
index 59e93a9b1..6ffffaa9a 100644
--- a/api/funkwhale_api/federation/serializers.py
+++ b/api/funkwhale_api/federation/serializers.py
@@ -834,17 +834,17 @@ class LibraryTrackActionSerializer(common_serializers.ActionSerializer):
source='federation',
submitted_by=self.context['submitted_by']
)
+ jobs = []
for lt in objects:
- job = music_models.ImportJob.objects.create(
+ job = music_models.ImportJob(
batch=batch,
library_track=lt,
mbid=lt.mbid,
source=lt.url,
)
- funkwhale_utils.on_commit(
- music_tasks.import_job_run.delay,
- import_job_id=job.pk,
- use_acoustid=False,
- )
+ jobs.append(job)
+
+ music_models.ImportJob.objects.bulk_create(jobs)
+ music_tasks.import_batch_run.delay(import_batch_id=batch.pk)
return {'batch': {'id': batch.pk}}
diff --git a/api/funkwhale_api/music/tasks.py b/api/funkwhale_api/music/tasks.py
index e5426904a..993456c27 100644
--- a/api/funkwhale_api/music/tasks.py
+++ b/api/funkwhale_api/music/tasks.py
@@ -173,6 +173,13 @@ def import_job_run(self, import_job, replace=False, use_acoustid=False):
raise
+@celery.app.task(name='ImportBatch.run')
+@celery.require_instance(models.ImportBatch, 'import_batch')
+def import_batch_run(import_batch):
+ for job_id in import_batch.jobs.order_by('id').values_list('id', flat=True):
+ import_job_run.delay(import_job_id=job_id)
+
+
@celery.app.task(name='Lyrics.fetch_content')
@celery.require_instance(models.Lyrics, 'lyrics')
def fetch_content(lyrics):
diff --git a/api/tests/federation/test_views.py b/api/tests/federation/test_views.py
index 1b0de5754..04a419aed 100644
--- a/api/tests/federation/test_views.py
+++ b/api/tests/federation/test_views.py
@@ -426,7 +426,8 @@ def test_library_track_action_import(
lt2 = factories['federation.LibraryTrack'](library=lt1.library)
lt3 = factories['federation.LibraryTrack']()
lt4 = factories['federation.LibraryTrack'](library=lt3.library)
- mocker.patch('funkwhale_api.music.tasks.import_job_run')
+ mocked_run = mocker.patch(
+ 'funkwhale_api.music.tasks.import_batch_run.delay')
payload = {
'objects': 'all',
@@ -452,3 +453,4 @@ def test_library_track_action_import(
assert batch.jobs.count() == 2
for i, job in enumerate(batch.jobs.all()):
assert job.library_track == imported_lts[i]
+ mocked_run.assert_called_once_with(import_batch_id=batch.pk)
diff --git a/api/tests/music/test_tasks.py b/api/tests/music/test_tasks.py
index 26cb9453e..dfe649be0 100644
--- a/api/tests/music/test_tasks.py
+++ b/api/tests/music/test_tasks.py
@@ -47,6 +47,15 @@ def test_set_acoustid_on_track_file_required_high_score(factories, mocker):
assert track_file.acoustid_track_id is None
+def test_import_batch_run(factories, mocker):
+ job = factories['music.ImportJob']()
+ mocked_job_run = mocker.patch(
+ 'funkwhale_api.music.tasks.import_job_run.delay')
+ tasks.import_batch_run(import_batch_id=job.batch.pk)
+
+ mocked_job_run.assert_called_once_with(import_job_id=job.pk)
+
+
def test_import_job_can_run_with_file_and_acoustid(
artists, albums, tracks, preferences, factories, mocker):
preferences['providers_acoustid__api_key'] = 'test'
--
GitLab