models.py 46.9 KB
Newer Older
1
import datetime
Eliot Berriot's avatar
Eliot Berriot committed
2
import logging
Eliot Berriot's avatar
Eliot Berriot committed
3
4
import os
import tempfile
5
import urllib.parse
6
import uuid
7

8
import arrow
9
import pydub
10
from django.conf import settings
Eliot Berriot's avatar
Eliot Berriot committed
11
from django.contrib.contenttypes.fields import GenericRelation
12
from django.contrib.postgres.fields import JSONField
13
14
from django.contrib.postgres.search import SearchVectorField
from django.contrib.postgres.indexes import GinIndex
15
from django.core.exceptions import ObjectDoesNotExist
Eliot Berriot's avatar
Eliot Berriot committed
16
from django.core.files.base import ContentFile
17
from django.core.serializers.json import DjangoJSONEncoder
18
from django.db import models, transaction
19
from django.db.models.signals import post_save, pre_save
20
from django.dispatch import receiver
21
from django.urls import reverse
22
23
24
from django.utils import timezone
from versatileimagefield.fields import VersatileImageField

25
26
from funkwhale_api import musicbrainz
from funkwhale_api.common import fields
27
from funkwhale_api.common import models as common_models
Eliot Berriot's avatar
Eliot Berriot committed
28
from funkwhale_api.common import session
29
30
from funkwhale_api.common import utils as common_utils
from funkwhale_api.federation import models as federation_models
31
from funkwhale_api.federation import utils as federation_utils
Eliot Berriot's avatar
Eliot Berriot committed
32
from funkwhale_api.tags import models as tags_models
Eliot Berriot's avatar
Eliot Berriot committed
33
from . import importers, metadata, utils
34

Eliot Berriot's avatar
Eliot Berriot committed
35
logger = logging.getLogger(__name__)
Eliot Berriot's avatar
Eliot Berriot committed
36

37
38
39
40
41
42
43
MAX_LENGTHS = {
    "ARTIST_NAME": 255,
    "ALBUM_TITLE": 255,
    "TRACK_TITLE": 255,
    "COPYRIGHT": 500,
}

44

45
46
47
48
49
50
51
ARTIST_CONTENT_CATEGORY_CHOICES = [
    ("music", "music"),
    ("podcast", "podcast"),
    ("other", "other"),
]


52
53
54
55
def empty_dict():
    return {}


56
class APIModelMixin(models.Model):
Eliot Berriot's avatar
Eliot Berriot committed
57
    fid = models.URLField(unique=True, max_length=500, db_index=True, null=True)
58
    mbid = models.UUIDField(unique=True, db_index=True, null=True, blank=True)
Eliot Berriot's avatar
Eliot Berriot committed
59
    uuid = models.UUIDField(unique=True, db_index=True, default=uuid.uuid4)
Eliot Berriot's avatar
Eliot Berriot committed
60
    from_activity = models.ForeignKey(
61
        "federation.Activity", null=True, blank=True, on_delete=models.SET_NULL
Eliot Berriot's avatar
Eliot Berriot committed
62
    )
63
    api_includes = []
64
    creation_date = models.DateTimeField(default=timezone.now, db_index=True)
65
    import_hooks = []
66
    body_text = SearchVectorField(blank=True)
67

68
69
    class Meta:
        abstract = True
Eliot Berriot's avatar
Eliot Berriot committed
70
        ordering = ["-creation_date"]
71
72
73
        indexes = [
            GinIndex(fields=["body_text"]),
        ]
74
75
76
77
78
79
80
81
82

    @classmethod
    def get_or_create_from_api(cls, mbid):
        try:
            return cls.objects.get(mbid=mbid), False
        except cls.DoesNotExist:
            return cls.create_from_api(id=mbid), True

    def get_api_data(self):
Eliot Berriot's avatar
Eliot Berriot committed
83
84
85
        return self.__class__.api.get(id=self.mbid, includes=self.api_includes)[
            self.musicbrainz_model
        ]
86
87
88

    @classmethod
    def create_from_api(cls, **kwargs):
Eliot Berriot's avatar
Eliot Berriot committed
89
90
91
92
        if kwargs.get("id"):
            raw_data = cls.api.get(id=kwargs["id"], includes=cls.api_includes)[
                cls.musicbrainz_model
            ]
93
        else:
Eliot Berriot's avatar
Eliot Berriot committed
94
95
96
            raw_data = cls.api.search(**kwargs)[
                "{0}-list".format(cls.musicbrainz_model)
            ][0]
97
98
99
100
101
102
103
104
105
106
107
        cleaned_data = cls.clean_musicbrainz_data(raw_data)
        return importers.load(cls, cleaned_data, raw_data, cls.import_hooks)

    @classmethod
    def clean_musicbrainz_data(cls, data):
        cleaned_data = {}
        mapping = importers.Mapping(cls.musicbrainz_mapping)
        for key, value in data.items():
            try:
                cleaned_key, cleaned_value = mapping.from_musicbrainz(key, value)
                cleaned_data[cleaned_key] = cleaned_value
108
            except KeyError:
109
110
111
                pass
        return cleaned_data

112
113
114
    @property
    def musicbrainz_url(self):
        if self.mbid:
Eliot Berriot's avatar
Eliot Berriot committed
115
116
117
            return "https://musicbrainz.org/{}/{}".format(
                self.musicbrainz_model, self.mbid
            )
118

Eliot Berriot's avatar
Eliot Berriot committed
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
    def get_federation_id(self):
        if self.fid:
            return self.fid

        return federation_utils.full_url(
            reverse(
                "federation:music:{}-detail".format(self.federation_namespace),
                kwargs={"uuid": self.uuid},
            )
        )

    def save(self, **kwargs):
        if not self.pk and not self.fid:
            self.fid = self.get_federation_id()

        return super().save(**kwargs)

Eliot Berriot's avatar
Eliot Berriot committed
136
137
    @property
    def is_local(self):
138
        return federation_utils.is_local(self.fid)
Eliot Berriot's avatar
Eliot Berriot committed
139

140
141
142
143
144
145
146
147
    @property
    def domain_name(self):
        if not self.fid:
            return

        parsed = urllib.parse.urlparse(self.fid)
        return parsed.hostname

148
149
150
    def get_tags(self):
        return list(sorted(self.tagged_items.values_list("tag__name", flat=True)))

151

Eliot Berriot's avatar
Eliot Berriot committed
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
class License(models.Model):
    code = models.CharField(primary_key=True, max_length=100)
    url = models.URLField(max_length=500)

    # if true, license is a copyleft license, meaning that derivative
    # work must be shared under the same license
    copyleft = models.BooleanField()
    # if true, commercial use of the work is allowed
    commercial = models.BooleanField()
    # if true, attribution to the original author is required when reusing
    # the work
    attribution = models.BooleanField()
    # if true, derivative work are allowed
    derivative = models.BooleanField()
    # if true, redistribution of the wor is allowed
    redistribute = models.BooleanField()

    @property
    def conf(self):
        from . import licenses

        for row in licenses.LICENSES:
            if self.code == row["code"]:
                return row
        logger.warning("%s do not match any registered license", self.code)


179
class ArtistQuerySet(common_models.LocalFromFidQuerySet, models.QuerySet):
180
    def with_albums_count(self):
Eliot Berriot's avatar
Eliot Berriot committed
181
        return self.annotate(_albums_count=models.Count("albums"))
182

183
184
    def with_albums(self):
        return self.prefetch_related(
185
186
187
188
189
190
            models.Prefetch(
                "albums",
                queryset=Album.objects.with_tracks_count().select_related(
                    "attachment_cover", "attributed_to"
                ),
            )
191
192
        )

193
194
    def annotate_playable_by_actor(self, actor):
        tracks = (
195
196
            Upload.objects.playable_by(actor)
            .filter(track__artist=models.OuterRef("id"))
197
198
199
200
201
202
203
            .order_by("id")
            .values("id")[:1]
        )
        subquery = models.Subquery(tracks)
        return self.annotate(is_playable_by_actor=subquery)

    def playable_by(self, actor, include=True):
204
205
        tracks = Track.objects.playable_by(actor)
        matches = self.filter(pk__in=tracks.values("artist_id")).values_list("pk")
206
        if include:
207
            return self.filter(pk__in=matches)
208
        else:
209
            return self.exclude(pk__in=matches)
210

211

212
class Artist(APIModelMixin):
213
    name = models.CharField(max_length=MAX_LENGTHS["ARTIST_NAME"])
Eliot Berriot's avatar
Eliot Berriot committed
214
    federation_namespace = "artists"
Eliot Berriot's avatar
Eliot Berriot committed
215
    musicbrainz_model = "artist"
216
    musicbrainz_mapping = {
Eliot Berriot's avatar
Eliot Berriot committed
217
218
        "mbid": {"musicbrainz_field_name": "id"},
        "name": {"musicbrainz_field_name": "name"},
219
    }
Eliot Berriot's avatar
Eliot Berriot committed
220
221
222
223
224
225
226
227
228
229
    # Music entities are attributed to actors, to validate that updates occur
    # from an authorized account. On top of that, we consider the instance actor
    # can update anything under it's own domain
    attributed_to = models.ForeignKey(
        "federation.Actor",
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        related_name="attributed_artists",
    )
230
    tagged_items = GenericRelation(tags_models.TaggedItem)
231
232
233
234
235
    fetches = GenericRelation(
        "federation.Fetch",
        content_type_field="object_content_type",
        object_id_field="object_id",
    )
236
237
238
    description = models.ForeignKey(
        "common.Content", null=True, blank=True, on_delete=models.SET_NULL
    )
239
240
241
242
243
244
245
    attachment_cover = models.ForeignKey(
        "common.Attachment",
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        related_name="covered_artist",
    )
246
247
248
249
250
251
252
    content_category = models.CharField(
        max_length=30,
        db_index=True,
        default="music",
        choices=ARTIST_CONTENT_CATEGORY_CHOICES,
        null=True,
    )
253
    modification_date = models.DateTimeField(default=timezone.now, db_index=True)
254
    api = musicbrainz.api.artists
255
    objects = ArtistQuerySet.as_manager()
256
257
258
259

    def __str__(self):
        return self.name

260
261
262
263
264
265
    def get_absolute_url(self):
        return "/library/artists/{}".format(self.pk)

    def get_moderation_url(self):
        return "/manage/library/artists/{}".format(self.pk)

266
267
    @classmethod
    def get_or_create_from_name(cls, name, **kwargs):
Eliot Berriot's avatar
Eliot Berriot committed
268
269
        kwargs.update({"name": name})
        return cls.objects.get_or_create(name__iexact=name, defaults=kwargs)
270

271
272
273
274
    @property
    def cover(self):
        return self.attachment_cover

275
276
277
278
279
280
    def get_channel(self):
        try:
            return self.channel
        except ObjectDoesNotExist:
            return None

281

282
def import_artist(v):
Eliot Berriot's avatar
Eliot Berriot committed
283
    a = Artist.get_or_create_from_api(mbid=v[0]["artist"]["id"])[0]
284
285
    return a

286

287
def parse_date(v):
288
    d = arrow.get(v).date()
289
290
291
292
    return d


def import_tracks(instance, cleaned_data, raw_data):
Eliot Berriot's avatar
Eliot Berriot committed
293
294
295
296
    for track_data in raw_data["medium-list"][0]["track-list"]:
        track_cleaned_data = Track.clean_musicbrainz_data(track_data["recording"])
        track_cleaned_data["album"] = instance
        track_cleaned_data["position"] = int(track_data["position"])
297
        importers.load(Track, track_cleaned_data, track_data, Track.import_hooks)
298

299

300
class AlbumQuerySet(common_models.LocalFromFidQuerySet, models.QuerySet):
301
    def with_tracks_count(self):
Eliot Berriot's avatar
Eliot Berriot committed
302
        return self.annotate(_tracks_count=models.Count("tracks"))
303

304
305
    def annotate_playable_by_actor(self, actor):
        tracks = (
306
307
            Upload.objects.playable_by(actor)
            .filter(track__album=models.OuterRef("id"))
308
309
310
311
312
313
314
            .order_by("id")
            .values("id")[:1]
        )
        subquery = models.Subquery(tracks)
        return self.annotate(is_playable_by_actor=subquery)

    def playable_by(self, actor, include=True):
315
316
        tracks = Track.objects.playable_by(actor)
        matches = self.filter(pk__in=tracks.values("album_id")).values_list("pk")
317
        if include:
318
            return self.filter(pk__in=matches)
319
        else:
320
            return self.exclude(pk__in=matches)
321

322
323
    def with_prefetched_tracks_and_playable_uploads(self, actor):
        tracks = Track.objects.with_playable_uploads(actor)
324
        return self.prefetch_related(models.Prefetch("tracks", queryset=tracks))
325

326

327
class Album(APIModelMixin):
328
    title = models.CharField(max_length=MAX_LENGTHS["ALBUM_TITLE"])
Eliot Berriot's avatar
Eliot Berriot committed
329
    artist = models.ForeignKey(Artist, related_name="albums", on_delete=models.CASCADE)
Eliot Berriot's avatar
Eliot Berriot committed
330
    release_date = models.DateField(null=True, blank=True, db_index=True)
331
    release_group_id = models.UUIDField(null=True, blank=True)
Eliot Berriot's avatar
Eliot Berriot committed
332
    # XXX: 1.0 clean this uneeded field in favor of attachment_cover
Eliot Berriot's avatar
Eliot Berriot committed
333
334
    cover = VersatileImageField(
        upload_to="albums/covers/%Y/%m/%d", null=True, blank=True
335
    )
Eliot Berriot's avatar
Eliot Berriot committed
336
337
338
339
340
341
342
    attachment_cover = models.ForeignKey(
        "common.Attachment",
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        related_name="covered_album",
    )
Eliot Berriot's avatar
Eliot Berriot committed
343
344
    TYPE_CHOICES = (("album", "Album"),)
    type = models.CharField(choices=TYPE_CHOICES, max_length=30, default="album")
345

Eliot Berriot's avatar
Eliot Berriot committed
346
347
348
349
350
351
352
353
354
355
    # Music entities are attributed to actors, to validate that updates occur
    # from an authorized account. On top of that, we consider the instance actor
    # can update anything under it's own domain
    attributed_to = models.ForeignKey(
        "federation.Actor",
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        related_name="attributed_albums",
    )
356
    tagged_items = GenericRelation(tags_models.TaggedItem)
357
358
359
360
361
    fetches = GenericRelation(
        "federation.Fetch",
        content_type_field="object_content_type",
        object_id_field="object_id",
    )
362

363
364
365
366
    description = models.ForeignKey(
        "common.Content", null=True, blank=True, on_delete=models.SET_NULL
    )

Eliot Berriot's avatar
Eliot Berriot committed
367
    api_includes = ["artist-credits", "recordings", "media", "release-groups"]
368
    api = musicbrainz.api.releases
Eliot Berriot's avatar
Eliot Berriot committed
369
    federation_namespace = "albums"
Eliot Berriot's avatar
Eliot Berriot committed
370
    musicbrainz_model = "release"
371
    musicbrainz_mapping = {
Eliot Berriot's avatar
Eliot Berriot committed
372
373
374
375
        "mbid": {"musicbrainz_field_name": "id"},
        "position": {
            "musicbrainz_field_name": "release-list",
            "converter": lambda v: int(v[0]["medium-list"][0]["position"]),
376
        },
Eliot Berriot's avatar
Eliot Berriot committed
377
378
379
        "release_group_id": {
            "musicbrainz_field_name": "release-group",
            "converter": lambda v: v["id"],
380
        },
Eliot Berriot's avatar
Eliot Berriot committed
381
382
383
384
385
386
        "title": {"musicbrainz_field_name": "title"},
        "release_date": {"musicbrainz_field_name": "date", "converter": parse_date},
        "type": {"musicbrainz_field_name": "type", "converter": lambda v: v.lower()},
        "artist": {
            "musicbrainz_field_name": "artist-credit",
            "converter": import_artist,
387
388
        },
    }
389
    objects = AlbumQuerySet.as_manager()
390

Eliot Berriot's avatar
Eliot Berriot committed
391
392
393
    @property
    def cover(self):
        return self.attachment_cover
394
395
396
397

    def __str__(self):
        return self.title

398
399
400
401
402
403
    def get_absolute_url(self):
        return "/library/albums/{}".format(self.pk)

    def get_moderation_url(self):
        return "/manage/library/albums/{}".format(self.pk)

404
405
    @classmethod
    def get_or_create_from_title(cls, title, **kwargs):
Eliot Berriot's avatar
Eliot Berriot committed
406
407
        kwargs.update({"title": title})
        return cls.objects.get_or_create(title__iexact=title, defaults=kwargs)
408
409


410
411
412
def import_tags(instance, cleaned_data, raw_data):
    MINIMUM_COUNT = 2
    tags_to_add = []
Eliot Berriot's avatar
Eliot Berriot committed
413
    for tag_data in raw_data.get("tag-list", []):
414
        try:
Eliot Berriot's avatar
Eliot Berriot committed
415
            if int(tag_data["count"]) < MINIMUM_COUNT:
416
417
418
                continue
        except ValueError:
            continue
Eliot Berriot's avatar
Eliot Berriot committed
419
        tags_to_add.append(tag_data["name"])
Eliot Berriot's avatar
Eliot Berriot committed
420
421

    tags_models.add_tags(instance, *tags_to_add)
422

423

424
def import_album(v):
Eliot Berriot's avatar
Eliot Berriot committed
425
    a = Album.get_or_create_from_api(mbid=v[0]["id"])[0]
426
427
428
    return a


429
class TrackQuerySet(common_models.LocalFromFidQuerySet, models.QuerySet):
430
    def for_nested_serialization(self):
Eliot Berriot's avatar
Eliot Berriot committed
431
432
433
        return self.prefetch_related(
            "artist", "album__artist", "album__attachment_cover"
        )
434
435

    def annotate_playable_by_actor(self, actor):
436

437
        files = (
Eliot Berriot's avatar
Eliot Berriot committed
438
            Upload.objects.playable_by(actor)
439
440
441
            .filter(track=models.OuterRef("id"))
            .order_by("id")
            .values("id")[:1]
Eliot Berriot's avatar
Eliot Berriot committed
442
        )
443
444
445
446
        subquery = models.Subquery(files)
        return self.annotate(is_playable_by_actor=subquery)

    def playable_by(self, actor, include=True):
447
448
449
450
451
452
453
454
455

        if settings.MUSIC_USE_DENORMALIZATION:
            if actor is not None:
                query = models.Q(actor=None) | models.Q(actor=actor)
            else:
                query = models.Q(actor=None, internal=False)
            if not include:
                query = ~query
            return self.filter(pk__in=TrackActor.objects.filter(query).values("track"))
Eliot Berriot's avatar
Eliot Berriot committed
456
        files = Upload.objects.playable_by(actor, include)
457
        matches = self.filter(uploads__in=files).values_list("pk")
458
        if include:
459
            return self.filter(pk__in=matches)
460
        else:
461
            return self.exclude(pk__in=matches)
Eliot Berriot's avatar
Eliot Berriot committed
462

463
    def with_playable_uploads(self, actor):
Eliot Berriot's avatar
Eliot Berriot committed
464
        uploads = Upload.objects.playable_by(actor)
465
        return self.prefetch_related(
466
            models.Prefetch("uploads", queryset=uploads, to_attr="playable_uploads")
Eliot Berriot's avatar
Eliot Berriot committed
467
        )
468

469
470
471
472
473
474
    def order_for_album(self):
        """
        Order by disc number then position
        """
        return self.order_by("disc_number", "position", "title")

475

476
477
def get_artist(release_list):
    return Artist.get_or_create_from_api(
Eliot Berriot's avatar
Eliot Berriot committed
478
479
        mbid=release_list[0]["artist-credits"][0]["artists"]["id"]
    )[0]
480
481


482
class Track(APIModelMixin):
483
    mbid = models.UUIDField(db_index=True, null=True, blank=True)
484
    title = models.CharField(max_length=MAX_LENGTHS["TRACK_TITLE"])
Eliot Berriot's avatar
Eliot Berriot committed
485
    artist = models.ForeignKey(Artist, related_name="tracks", on_delete=models.CASCADE)
486
    disc_number = models.PositiveIntegerField(null=True, blank=True)
487
    position = models.PositiveIntegerField(null=True, blank=True)
Eliot Berriot's avatar
Eliot Berriot committed
488
    album = models.ForeignKey(
Eliot Berriot's avatar
Eliot Berriot committed
489
490
        Album, related_name="tracks", null=True, blank=True, on_delete=models.CASCADE
    )
Eliot Berriot's avatar
Eliot Berriot committed
491
492
493
494
495
496
497
    license = models.ForeignKey(
        License,
        null=True,
        blank=True,
        on_delete=models.DO_NOTHING,
        related_name="tracks",
    )
Eliot Berriot's avatar
Eliot Berriot committed
498
499
500
501
502
503
504
505
506
507
    # Music entities are attributed to actors, to validate that updates occur
    # from an authorized account. On top of that, we consider the instance actor
    # can update anything under it's own domain
    attributed_to = models.ForeignKey(
        "federation.Actor",
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        related_name="attributed_tracks",
    )
508
509
510
    copyright = models.CharField(
        max_length=MAX_LENGTHS["COPYRIGHT"], null=True, blank=True
    )
511
512
513
    description = models.ForeignKey(
        "common.Content", null=True, blank=True, on_delete=models.SET_NULL
    )
514
515
516
517
518
519
520
    attachment_cover = models.ForeignKey(
        "common.Attachment",
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        related_name="covered_track",
    )
521
    downloads_count = models.PositiveIntegerField(default=0)
Eliot Berriot's avatar
Eliot Berriot committed
522
    federation_namespace = "tracks"
Eliot Berriot's avatar
Eliot Berriot committed
523
    musicbrainz_model = "recording"
524
    api = musicbrainz.api.recordings
525
    api_includes = ["artist-credits", "releases", "media", "tags"]
526
    musicbrainz_mapping = {
Eliot Berriot's avatar
Eliot Berriot committed
527
528
529
        "mbid": {"musicbrainz_field_name": "id"},
        "title": {"musicbrainz_field_name": "title"},
        "artist": {
530
531
532
533
            "musicbrainz_field_name": "artist-credit",
            "converter": lambda v: Artist.get_or_create_from_api(
                mbid=v[0]["artist"]["id"]
            )[0],
534
        },
Eliot Berriot's avatar
Eliot Berriot committed
535
        "album": {"musicbrainz_field_name": "release-list", "converter": import_album},
536
    }
Eliot Berriot's avatar
Eliot Berriot committed
537
    import_hooks = [import_tags]
538
    objects = TrackQuerySet.as_manager()
Eliot Berriot's avatar
Eliot Berriot committed
539
    tagged_items = GenericRelation(tags_models.TaggedItem)
540
541
542
543
544
    fetches = GenericRelation(
        "federation.Fetch",
        content_type_field="object_content_type",
        object_id_field="object_id",
    )
545

546
    class Meta:
547
        ordering = ["album", "disc_number", "position"]
548
549
550
        indexes = [
            GinIndex(fields=["body_text"]),
        ]
551

552
553
554
    def __str__(self):
        return self.title

555
556
557
558
559
560
    def get_absolute_url(self):
        return "/library/tracks/{}".format(self.pk)

    def get_moderation_url(self):
        return "/manage/library/tracks/{}".format(self.pk)

561
562
563
    def save(self, **kwargs):
        try:
            self.artist
564
        except Artist.DoesNotExist:
565
566
567
568
569
570
            self.artist = self.album.artist
        super().save(**kwargs)

    @property
    def full_name(self):
        try:
Eliot Berriot's avatar
Eliot Berriot committed
571
            return "{} - {} - {}".format(self.artist.name, self.album.title, self.title)
572
        except AttributeError:
Eliot Berriot's avatar
Eliot Berriot committed
573
            return "{} - {}".format(self.artist.name, self.title)
574

575
576
577
578
    @property
    def cover(self):
        return self.attachment_cover

579
580
    def get_activity_url(self):
        if self.mbid:
Eliot Berriot's avatar
Eliot Berriot committed
581
582
            return "https://musicbrainz.org/recording/{}".format(self.mbid)
        return settings.FUNKWHALE_URL + "/tracks/{}".format(self.pk)
583

584
585
    @classmethod
    def get_or_create_from_title(cls, title, **kwargs):
Eliot Berriot's avatar
Eliot Berriot committed
586
587
        kwargs.update({"title": title})
        return cls.objects.get_or_create(title__iexact=title, defaults=kwargs)
588

589
590
591
592
593
594
595
596
597
598
599
    @classmethod
    def get_or_create_from_release(cls, release_mbid, mbid):
        release_mbid = str(release_mbid)
        mbid = str(mbid)
        try:
            return cls.objects.get(mbid=mbid), False
        except cls.DoesNotExist:
            pass

        album = Album.get_or_create_from_api(release_mbid)[0]
        data = musicbrainz.client.api.releases.get(
Eliot Berriot's avatar
Eliot Berriot committed
600
601
602
            str(album.mbid), includes=Album.api_includes
        )
        tracks = [t for m in data["release"]["medium-list"] for t in m["track-list"]]
603
604
        track_data = None
        for track in tracks:
605
            if track["recording"]["id"] == str(mbid):
606
607
608
                track_data = track
                break
        if not track_data:
Eliot Berriot's avatar
Eliot Berriot committed
609
            raise ValueError("No track found matching this ID")
610

611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
        track_artist_mbid = None
        for ac in track_data["recording"]["artist-credit"]:
            try:
                ac_mbid = ac["artist"]["id"]
            except TypeError:
                # it's probably a string, like "feat."
                continue

            if ac_mbid == str(album.artist.mbid):
                continue

            track_artist_mbid = ac_mbid
            break
        track_artist_mbid = track_artist_mbid or album.artist.mbid
        if track_artist_mbid == str(album.artist.mbid):
            track_artist = album.artist
        else:
            track_artist = Artist.get_or_create_from_api(track_artist_mbid)[0]
629
630
631
        return cls.objects.update_or_create(
            mbid=mbid,
            defaults={
Eliot Berriot's avatar
Eliot Berriot committed
632
633
634
                "position": int(track["position"]),
                "title": track["recording"]["title"],
                "album": album,
635
                "artist": track_artist,
Eliot Berriot's avatar
Eliot Berriot committed
636
            },
637
        )
Eliot Berriot's avatar
Eliot Berriot committed
638

639
640
    @property
    def listen_url(self):
Eliot Berriot's avatar
Eliot Berriot committed
641
642
        # Not using reverse because this is slow
        return "/api/v1/listen/{}/".format(self.uuid)
643

Eliot Berriot's avatar
Eliot Berriot committed
644
645
646
647
648
649
650
651
652
653
654
    @property
    def local_license(self):
        """
        Since license primary keys are strings, and we can get the data
        from our hardcoded licenses.LICENSES list, there is no need
        for extra SQL joins / queries.
        """
        from . import licenses

        return licenses.LICENSES_BY_ID.get(self.license_id)

655

656
class UploadQuerySet(common_models.NullsLastQuerySet):
657
658
659
660
661
662
663
664
    def in_place(self, include=True):
        query = models.Q(source__startswith="file://") & (
            models.Q(audio_file="") | models.Q(audio_file=None)
        )
        if not include:
            query = ~query
        return self.filter(query)

665
    def playable_by(self, actor, include=True):
Eliot Berriot's avatar
Eliot Berriot committed
666
        libraries = Library.objects.viewable_by(actor)
667
668

        if include:
669
670
            return self.filter(library__in=libraries, import_status="finished")
        return self.exclude(library__in=libraries, import_status="finished")
671
672

    def local(self, include=True):
673
674
675
676
        query = models.Q(library__actor__domain_id=settings.FEDERATION_HOSTNAME)
        if not include:
            query = ~query
        return self.filter(query)
677

Eliot Berriot's avatar
Eliot Berriot committed
678
679
680
    def for_federation(self):
        return self.filter(import_status="finished", mimetype__startswith="audio/")

681
    def with_file(self):
Eliot Berriot's avatar
Linting    
Eliot Berriot committed
682
683
        return self.exclude(audio_file=None).exclude(audio_file="")

684
685

TRACK_FILE_IMPORT_STATUS_CHOICES = (
686
    ("draft", "Draft"),
687
688
689
690
691
692
693
694
    ("pending", "Pending"),
    ("finished", "Finished"),
    ("errored", "Errored"),
    ("skipped", "Skipped"),
)


def get_file_path(instance, filename):
695
696
697
    if isinstance(instance, UploadVersion):
        return common_utils.ChunkedPath("transcoded")(instance, filename)

698
    if instance.library.actor.get_user():
699
700
701
702
703
704
705
706
707
        return common_utils.ChunkedPath("tracks")(instance, filename)
    else:
        # we cache remote tracks in a different directory
        return common_utils.ChunkedPath("federation_cache/tracks")(instance, filename)


def get_import_reference():
    return str(uuid.uuid4())

Eliot Berriot's avatar
Eliot Berriot committed
708

Eliot Berriot's avatar
Eliot Berriot committed
709
class Upload(models.Model):
710
    fid = models.URLField(unique=True, max_length=500, null=True, blank=True)
Eliot Berriot's avatar
Eliot Berriot committed
711
    uuid = models.UUIDField(unique=True, db_index=True, default=uuid.uuid4)
712
    track = models.ForeignKey(
Eliot Berriot's avatar
Eliot Berriot committed
713
        Track, related_name="uploads", on_delete=models.CASCADE, null=True, blank=True
714
715
716
717
718
719
720
721
    )
    audio_file = models.FileField(upload_to=get_file_path, max_length=255)
    source = models.CharField(
        # URL validators are not flexible enough for our file:// and upload:// schemes
        null=True,
        blank=True,
        max_length=500,
    )
722
    creation_date = models.DateTimeField(default=timezone.now, db_index=True)
Eliot Berriot's avatar
Eliot Berriot committed
723
    modification_date = models.DateTimeField(default=timezone.now, null=True)
724
    accessed_date = models.DateTimeField(null=True, blank=True)
725
    duration = models.IntegerField(null=True, blank=True)
726
727
    size = models.IntegerField(null=True, blank=True)
    bitrate = models.IntegerField(null=True, blank=True)
728
    acoustid_track_id = models.UUIDField(null=True, blank=True)
729
    mimetype = models.CharField(null=True, blank=True, max_length=200)
730
    library = models.ForeignKey(
Eliot Berriot's avatar
Eliot Berriot committed
731
732
733
734
735
        "library",
        null=True,
        blank=True,
        related_name="uploads",
        on_delete=models.CASCADE,
736
    )
737

738
739
    # metadata from federation
    metadata = JSONField(
740
        default=empty_dict, max_length=50000, encoder=DjangoJSONEncoder, blank=True
741
742
743
744
    )
    import_date = models.DateTimeField(null=True, blank=True)
    # optionnal metadata provided during import
    import_metadata = JSONField(
745
        default=empty_dict, max_length=50000, encoder=DjangoJSONEncoder, blank=True
746
747
748
749
750
751
752
753
754
755
756
    )
    # status / error details for the import
    import_status = models.CharField(
        default="pending", choices=TRACK_FILE_IMPORT_STATUS_CHOICES, max_length=25
    )
    # a short reference provided by the client to group multiple files
    # in the same import
    import_reference = models.CharField(max_length=50, default=get_import_reference)

    # optionnal metadata about import results (error messages, etc.)
    import_details = JSONField(
757
        default=empty_dict, max_length=50000, encoder=DjangoJSONEncoder, blank=True
758
    )
Eliot Berriot's avatar
Eliot Berriot committed
759
    from_activity = models.ForeignKey(
760
        "federation.Activity", null=True, on_delete=models.SET_NULL, blank=True
Eliot Berriot's avatar
Eliot Berriot committed
761
    )
762
    downloads_count = models.PositiveIntegerField(default=0)
763

764
765
766
    # stores checksums such as `sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855`
    checksum = models.CharField(max_length=100, db_index=True, null=True, blank=True)

Eliot Berriot's avatar
Eliot Berriot committed
767
    objects = UploadQuerySet.as_manager()
768

769
770
771
772
773
774
775
776
777
778
779
780
    @property
    def is_local(self):
        return federation_utils.is_local(self.fid)

    @property
    def domain_name(self):
        if not self.fid:
            return

        parsed = urllib.parse.urlparse(self.fid)
        return parsed.hostname

781
    def download_audio_from_remote(self, actor):
782
783
        from funkwhale_api.federation import signing

784
785
        if actor:
            auth = signing.get_auth(actor.private_key, actor.private_key_id)
786
787
788
789
790
791
792
793
        else:
            auth = None

        remote_response = session.get_session().get(
            self.source,
            auth=auth,
            stream=True,
            timeout=20,
Eliot Berriot's avatar
Eliot Berriot committed
794
            headers={"Content-Type": "application/octet-stream"},
795
        )
796
797
798
        with remote_response as r:
            remote_response.raise_for_status()
            extension = utils.get_ext_from_type(self.mimetype)
799
800
801
802
803
804
805
            title_parts = []
            title_parts.append(self.track.title)
            if self.track.album:
                title_parts.append(self.track.album.title)
            title_parts.append(self.track.artist.name)

            title = " - ".join(title_parts)
806
807
808
809
810
811
            filename = "{}.{}".format(title, extension)
            tmp_file = tempfile.TemporaryFile()
            for chunk in r.iter_content(chunk_size=512):
                tmp_file.write(chunk)
            self.audio_file.save(filename, tmp_file, save=False)
            self.save(update_fields=["audio_file"])
812

813
814
815
    def get_federation_id(self):
        if self.fid:
            return self.fid
816

Eliot Berriot's avatar
Eliot Berriot committed
817
818
819
        return federation_utils.full_url(
            reverse("federation:music:uploads-detail", kwargs={"uuid": self.uuid})
        )
820

821
822
    @property
    def filename(self):
Eliot Berriot's avatar
Eliot Berriot committed
823
        return "{}.{}".format(self.track.full_name, self.extension)
824
825
826

    @property
    def extension(self):
827
828
829
830
        try:
            return utils.MIMETYPE_TO_EXTENSION[self.mimetype]
        except KeyError:
            pass
831
832
833
834
        if self.audio_file:
            return os.path.splitext(self.audio_file.name)[-1].replace(".", "", 1)
        if self.in_place_path:
            return os.path.splitext(self.in_place_path)[-1].replace(".", "", 1)
835

836
837
838
839
    def get_file_size(self):
        if self.audio_file:
            return self.audio_file.size

Eliot Berriot's avatar
Eliot Berriot committed
840
841
        if self.source.startswith("file://"):
            return os.path.getsize(self.source.replace("file://", "", 1))
842
843
844
845

    def get_audio_file(self):
        if self.audio_file:
            return self.audio_file.open()
846
        if self.source and self.source.startswith("file://"):
Eliot Berriot's avatar
Eliot Berriot committed
847
            return open(self.source.replace("file://", "", 1), "rb")
848

849
    def get_audio_data(self):
850
        audio_file = self.get_audio_file()
851
852
853
854
855
856
857
858
859
860
        if not audio_file:
            return
        audio_data = utils.get_audio_file_data(audio_file)
        if not audio_data:
            return
        return {
            "duration": int(audio_data["length"]),
            "bitrate": audio_data["bitrate"],
            "size": self.get_file_size(),
        }
861

862
863
864
865
866
867
868
869
870
    def get_audio_segment(self):
        input = self.get_audio_file()
        if not input:
            return

        input_format = utils.MIMETYPE_TO_EXTENSION[self.mimetype]
        audio = pydub.AudioSegment.from_file(input, format=input_format)
        return audio

871
    def save(self, **kwargs):
Eliot Berriot's avatar
Eliot Berriot committed
872
873
874
875
        if not self.mimetype:
            if self.audio_file:
                self.mimetype = utils.guess_mimetype(self.audio_file)
            elif self.source and self.source.startswith("file://"):
876
                self.mimetype = utils.guess_mimetype_from_name(self.source)
877
878
        if not self.size and self.audio_file:
            self.size = self.audio_file.size
879
880
881
882
883
884
885
886
887
        if not self.checksum:
            try:
                audio_file = self.get_audio_file()
            except FileNotFoundError:
                pass
            else:
                if audio_file:
                    self.checksum = common_utils.get_file_hash(audio_file)

888
        if not self.pk and not self.fid and self.library.actor.get_user():
Eliot Berriot's avatar
Eliot Berriot committed
889
            self.fid = self.get_federation_id()
890
        return super().save(**kwargs)
891

892
893
894
895
896
897
    def get_metadata(self):
        audio_file = self.get_audio_file()
        if not audio_file:
            return
        return metadata.Metadata(audio_file)

898
899
    @property
    def listen_url(self):
Eliot Berriot's avatar
Eliot Berriot committed
900
        return self.track.listen_url + "?upload={}".format(self.uuid)
901

Agate's avatar
Agate committed
902
    def get_listen_url(self, to=None, download=None):
903
904
905
        url = self.listen_url
        if to:
            url += "&to={}".format(to)
Agate's avatar
Agate committed
906
907
        if download is not None:
            url += "&download={}".format(str(download).lower())
908
909
        return url

910
911
912
913
914
    @property
    def listen_url_no_download(self):
        # Not using reverse because this is slow
        return self.listen_url + "&download=false"

915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
    def get_transcoded_version(self, format, max_bitrate=None):
        if format:
            mimetype = utils.EXTENSION_TO_MIMETYPE[format]
        else:
            mimetype = self.mimetype or "audio/mpeg"
            format = utils.MIMETYPE_TO_EXTENSION[mimetype]

        existing_versions = self.versions.filter(mimetype=mimetype)
        if max_bitrate is not None:
            # we don't want to transcode if a 320kbps version is available
            # and we're requestiong 300kbps
            acceptable_max_bitrate = max_bitrate * 1.2
            acceptable_min_bitrate = max_bitrate * 0.8
            existing_versions = existing_versions.filter(
                bitrate__gte=acceptable_min_bitrate, bitrate__lte=acceptable_max_bitrate
            ).order_by("-bitrate")
931
932
933
934
        if existing_versions:
            # we found an existing version, no need to transcode again
            return existing_versions[0]

935
        return self.create_transcoded_version(mimetype, format, bitrate=max_bitrate)
936
937

    @transaction.atomic
938
    def create_transcoded_version(self, mimetype, format, bitrate):
939
940
941
        # we create the version with an empty file, then
        # we'll write to it
        f = ContentFile(b"")
942
943
        bitrate = min(bitrate or 320000, self.bitrate or 320000)
        version = self.versions.create(mimetype=mimetype, bitrate=bitrate, size=0)
944
        # we keep the same name, but we update the extension
945
946
947
        new_name = os.path.splitext(os.path.basename(self.audio_file.name))[
            0
        ] + ".{}".format(format)
948
        version.audio_file.save(new_name, f)
949
950
        utils.transcode_audio(
            audio=self.get_audio_segment(),
951
952
            output=version.audio_file,
            output_format=utils.MIMETYPE_TO_EXTENSION[mimetype],
953
            bitrate=str(bitrate),
954
955
        )
        version.size = version.audio_file.size
956
        version.save(update_fields=["size"])
957
958
959

        return version

960
961
962
963
964
965
    @property
    def in_place_path(self):
        if not self.source or not self.source.startswith("file://"):
            return
        return self.source.lstrip("file://")

966
967
968
969
970
971
972
973
974
975
    @property
    def audio_file_path(self):
        if not self.audio_file:
            return None
        try:
            return self.audio_file.path
        except NotImplementedError:
            # external storage
            return self.audio_file.name

976
977
    def get_all_tagged_items(self):
        track_tags = self.track.tagged_items.all()
978
979
980
981
982
        album_tags = (
            self.track.album.tagged_items.all()
            if self.track.album
            else tags_models.TaggedItem.objects.none()
        )
983
984
985
986
987
        artist_tags = self.track.artist.tagged_items.all()

        items = (track_tags | album_tags | artist_tags).order_by("tag__name")
        return items

988

989
MIMETYPE_CHOICES = [(mt, ext) for ext, mt in utils.AUDIO_EXTENSIONS_AND_MIMETYPE]
990
991
992


class UploadVersion(models.Model):
993
994
995
    upload = models.ForeignKey(
        Upload, related_name="versions", on_delete=models.CASCADE
    )
996
997
998
999
1000
    mimetype = models.CharField(max_length=50, choices=MIMETYPE_CHOICES)