tasks.py 12 KB
Newer Older
1
import logging
2
3
import os

4
5
from django.core.files.base import ContentFile

6
7
from musicbrainzngs import ResponseError

8
from funkwhale_api.common import preferences
9
10
11
12
from funkwhale_api.federation import activity
from funkwhale_api.federation import actors
from funkwhale_api.federation import models as federation_models
from funkwhale_api.federation import serializers as federation_serializers
13
14
from funkwhale_api.taskapp import celery
from funkwhale_api.providers.acoustid import get_acoustid_client
15
from funkwhale_api.providers.audiofile import tasks as audiofile_tasks
16
17
18
19

from django.conf import settings
from . import models
from . import lyrics as lyrics_utils
20
from . import utils as music_utils
21

22
23
logger = logging.getLogger(__name__)

24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

@celery.app.task(name='acoustid.set_on_track_file')
@celery.require_instance(models.TrackFile, 'track_file')
def set_acoustid_on_track_file(track_file):
    client = get_acoustid_client()
    result = client.get_best_match(track_file.audio_file.path)

    def update(id):
        track_file.acoustid_track_id = id
        track_file.save(update_fields=['acoustid_track_id'])
        return id
    if result:
        return update(result['id'])


39
40
41
42
43
44
45
46
def import_track_from_remote(library_track):
    metadata = library_track.metadata
    try:
        track_mbid = metadata['recording']['musicbrainz_id']
        assert track_mbid  # for null/empty values
    except (KeyError, AssertionError):
        pass
    else:
47
        return models.Track.get_or_create_from_api(mbid=track_mbid)[0]
48

49
50
51
52
53
54
    try:
        album_mbid = metadata['release']['musicbrainz_id']
        assert album_mbid  # for null/empty values
    except (KeyError, AssertionError):
        pass
    else:
55
        album, _ = models.Album.get_or_create_from_api(mbid=album_mbid)
56
        return models.Track.get_or_create_from_title(
57
            library_track.title, artist=album.artist, album=album)[0]
58

59
60
61
62
63
64
    try:
        artist_mbid = metadata['artist']['musicbrainz_id']
        assert artist_mbid  # for null/empty values
    except (KeyError, AssertionError):
        pass
    else:
65
66
        artist, _ = models.Artist.get_or_create_from_api(mbid=artist_mbid)
        album, _ = models.Album.get_or_create_from_title(
67
            library_track.album_title, artist=artist)
68
        return models.Track.get_or_create_from_title(
69
            library_track.title, artist=artist, album=album)[0]
70
71
72

    # worst case scenario, we have absolutely no way to link to a
    # musicbrainz resource, we rely on the name/titles
73
    artist, _ = models.Artist.get_or_create_from_name(
74
        library_track.artist_name)
75
    album, _ = models.Album.get_or_create_from_title(
76
        library_track.album_title, artist=artist)
77
    return models.Track.get_or_create_from_title(
78
        library_track.title, artist=artist, album=album)[0]
79
80


81
def _do_import(import_job, replace=False, use_acoustid=False):
82
    logger.info('[Import Job %s] starting job', import_job.pk)
83
84
85
86
87
    from_file = bool(import_job.audio_file)
    mbid = import_job.mbid
    acoustid_track_id = None
    duration = None
    track = None
88
89
90
    # use_acoustid = use_acoustid and preferences.get('providers_acoustid__api_key')
    # Acoustid is not reliable, we disable it for now.
    use_acoustid = False
91
    if not mbid and use_acoustid and from_file:
92
93
94
        # we try to deduce mbid from acoustid
        client = get_acoustid_client()
        match = client.get_best_match(import_job.audio_file.path)
95
96
97
98
        if match:
            duration = match['recordings'][0]['duration']
            mbid = match['recordings'][0]['id']
            acoustid_track_id = match['id']
99
    if mbid:
100
101
102
103
        logger.info(
            '[Import Job %s] importing track from musicbrainz recording %s',
            import_job.pk,
            str(mbid))
104
        track, _ = models.Track.get_or_create_from_api(mbid=mbid)
105
    elif import_job.audio_file:
106
107
108
109
        logger.info(
            '[Import Job %s] importing track from uploaded track data at %s',
            import_job.pk,
            import_job.audio_file.path)
110
111
        track = audiofile_tasks.import_track_data_from_path(
            import_job.audio_file.path)
112
    elif import_job.library_track:
113
114
115
116
        logger.info(
            '[Import Job %s] importing track from federated library track %s',
            import_job.pk,
            import_job.library_track.pk)
117
        track = import_track_from_remote(import_job.library_track)
118
    elif import_job.source.startswith('file://'):
119
120
121
122
123
        tf_path = import_job.source.replace('file://', '', 1)
        logger.info(
            '[Import Job %s] importing track from local track data at %s',
            import_job.pk,
            tf_path)
124
        track = audiofile_tasks.import_track_data_from_path(
125
            tf_path)
126
    else:
127
128
129
        raise ValueError(
            'Not enough data to process import, '
            'add a mbid, an audio file or a library track')
130
131
132

    track_file = None
    if replace:
133
134
        logger.info(
            '[Import Job %s] replacing existing audio file', import_job.pk)
135
136
        track_file = track.files.first()
    elif track.files.count() > 0:
137
138
139
        logger.info(
            '[Import Job %s] skipping, we already have a file for this track',
            import_job.pk)
140
141
142
143
144
145
146
147
148
149
150
151
152
        if import_job.audio_file:
            import_job.audio_file.delete()
        import_job.status = 'skipped'
        import_job.save()
        return

    track_file = track_file or models.TrackFile(
        track=track, source=import_job.source)
    track_file.acoustid_track_id = acoustid_track_id
    if from_file:
        track_file.audio_file = ContentFile(import_job.audio_file.read())
        track_file.audio_file.name = import_job.audio_file.name
        track_file.duration = duration
153
    elif import_job.library_track:
154
        track_file.library_track = import_job.library_track
155
156
        track_file.mimetype = import_job.library_track.audio_mimetype
        if import_job.library_track.library.download_files:
157
158
159
160
            raise NotImplementedError()
        else:
            # no downloading, we hotlink
            pass
Eliot Berriot's avatar
Eliot Berriot committed
161
162
    elif not import_job.audio_file and not import_job.source.startswith('file://'):
        # not an implace import, and we have a source, so let's download it
163
164
165
        logger.info(
            '[Import Job %s] downloading audio file from remote',
            import_job.pk)
166
        track_file.download_file()
167
168
169
170
    elif not import_job.audio_file and import_job.source.startswith('file://'):
        # in place import, we set mimetype from extension
        path, ext = os.path.splitext(import_job.source)
        track_file.mimetype = music_utils.get_type_from_ext(ext)
171
    track_file.set_audio_data()
172
    track_file.save()
173
174
175
176
177
178
    # if no cover is set on track album, we try to update it as well:
    if not track.album.cover:
        logger.info(
            '[Import Job %s] retrieving album cover',
            import_job.pk)
        update_album_cover(track.album, track_file)
179
180
181
182
183
184
    import_job.status = 'finished'
    import_job.track_file = track_file
    if import_job.audio_file:
        # it's imported on the track, we don't need it anymore
        import_job.audio_file.delete()
    import_job.save()
185
186
187
    logger.info(
        '[Import Job %s] job finished',
        import_job.pk)
188
    return track_file
189
190


191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
def update_album_cover(album, track_file, replace=False):
    if album.cover and not replace:
        return

    if track_file:
        # maybe the file has a cover embedded?
        try:
            metadata = track_file.get_metadata()
        except FileNotFoundError:
            metadata = None
        if metadata:
            cover = metadata.get_picture('cover_front')
            if cover:
                # best case scenario, cover is embedded in the track
                logger.info(
                    '[Album %s] Using cover embedded in file',
                    album.pk)
                return album.get_image(data=cover)
        if track_file.source and track_file.source.startswith('file://'):
            # let's look for a cover in the same directory
            path = os.path.dirname(track_file.source.replace('file://', '', 1))
            logger.info(
                '[Album %s] scanning covers from %s',
                album.pk,
                path)
            cover = get_cover_from_fs(path)
            if cover:
                return album.get_image(data=cover)
    if not album.mbid:
        return
    try:
        logger.info(
            '[Album %s] Fetching cover from musicbrainz release %s',
            album.pk,
            str(album.mbid))
        return album.get_image()
    except ResponseError as exc:
        logger.warning(
            '[Album %s] cannot fetch cover from musicbrainz: %s',
            album.pk,
            str(exc))


IMAGE_TYPES = [
    ('jpg', 'image/jpeg'),
    ('png', 'image/png'),
]

def get_cover_from_fs(dir_path):
    if os.path.exists(dir_path):
        for e, m in IMAGE_TYPES:
            cover_path = os.path.join(dir_path, 'cover.{}'.format(e))
            if not os.path.exists(cover_path):
                logger.debug('Cover %s does not exists', cover_path)
                continue
            with open(cover_path, 'rb') as c:
                logger.info('Found cover at %s', cover_path)
                return {
                    'mimetype': m,
                    'content': c.read(),
                }



255
@celery.app.task(name='ImportJob.run', bind=True)
256
257
258
259
@celery.require_instance(
    models.ImportJob.objects.filter(
        status__in=['pending', 'errored']),
    'import_job')
260
def import_job_run(self, import_job, replace=False, use_acoustid=False):
261
    def mark_errored(exc):
262
263
264
        logger.error(
            '[Import Job %s] Error during import: %s',
            import_job.pk, str(exc))
265
        import_job.status = 'errored'
266
        import_job.save(update_fields=['status'])
267

268
    try:
269
270
        tf = _do_import(import_job, replace, use_acoustid=use_acoustid)
        return tf.pk if tf else None
271
272
    except Exception as exc:
        if not settings.DEBUG:
273
274
275
            try:
                self.retry(exc=exc, countdown=30, max_retries=3)
            except:
276
                mark_errored(exc)
277
                raise
278
        mark_errored(exc)
279
280
281
        raise


282
283
284
285
286
287
288
@celery.app.task(name='ImportBatch.run')
@celery.require_instance(models.ImportBatch, 'import_batch')
def import_batch_run(import_batch):
    for job_id in import_batch.jobs.order_by('id').values_list('id', flat=True):
        import_job_run.delay(import_job_id=job_id)


289
290
291
292
293
294
295
296
@celery.app.task(name='Lyrics.fetch_content')
@celery.require_instance(models.Lyrics, 'lyrics')
def fetch_content(lyrics):
    html = lyrics_utils._get_html(lyrics.url)
    content = lyrics_utils.extract_content(html)
    cleaned_content = lyrics_utils.clean_content(content)
    lyrics.content = cleaned_content
    lyrics.save(update_fields=['content'])
297
298
299
300
301
302


@celery.app.task(name='music.import_batch_notify_followers')
@celery.require_instance(
    models.ImportBatch.objects.filter(status='finished'), 'import_batch')
def import_batch_notify_followers(import_batch):
303
    if not preferences.get('federation__enabled'):
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
        return

    if import_batch.source == 'federation':
        return

    library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
    followers = library_actor.get_approved_followers()
    jobs = import_batch.jobs.filter(
        status='finished',
        library_track__isnull=True,
        track_file__isnull=False,
    ).select_related(
        'track_file__track__artist',
        'track_file__track__album__artist',
    )
    track_files = [job.track_file for job in jobs]
    collection = federation_serializers.CollectionSerializer({
        'actor': library_actor,
        'id': import_batch.get_federation_url(),
        'items': track_files,
        'item_serializer': federation_serializers.AudioSerializer
    }).data
    for f in followers:
        create = federation_serializers.ActivitySerializer(
            {
                'type': 'Create',
                'id': collection['id'],
                'object': collection,
                'actor': library_actor.url,
                'to': [f.url],
            }
        ).data

        activity.deliver(create, on_behalf_of=library_actor, to=[f.url])