models.py 33.5 KB
Newer Older
1
import datetime
Agate's avatar
Agate committed
2
import logging
Agate's avatar
Agate committed
3
import mimetypes
Agate's avatar
Agate committed
4 5
import os
import tempfile
6
import uuid
7

Agate's avatar
Agate committed
8
import markdown
9
import pendulum
10
from django.conf import settings
11
from django.contrib.postgres.fields import JSONField
Agate's avatar
Agate committed
12
from django.core.files.base import ContentFile
13
from django.core.serializers.json import DjangoJSONEncoder
Agate's avatar
Agate committed
14
from django.db import models
15 16
from django.db.models.signals import post_save
from django.dispatch import receiver
Agate's avatar
Agate committed
17
from django.urls import reverse
18 19
from django.utils import timezone
from taggit.managers import TaggableManager
20

21
from versatileimagefield.fields import VersatileImageField
22
from versatileimagefield.image_warmer import VersatileImageFieldWarmer
23

24 25
from funkwhale_api import musicbrainz
from funkwhale_api.common import fields
Agate's avatar
Agate committed
26
from funkwhale_api.common import session
27 28
from funkwhale_api.common import utils as common_utils
from funkwhale_api.federation import models as federation_models
29
from funkwhale_api.federation import utils as federation_utils
Agate's avatar
Agate committed
30
from . import importers, metadata, utils
31

Agate's avatar
Agate committed
32 33
logger = logging.getLogger(__file__)

34

35 36 37 38
def empty_dict():
    return {}


39
class APIModelMixin(models.Model):
Agate's avatar
Agate committed
40
    fid = models.URLField(unique=True, max_length=500, db_index=True, null=True)
41
    mbid = models.UUIDField(unique=True, db_index=True, null=True, blank=True)
Agate's avatar
Agate committed
42
    uuid = models.UUIDField(unique=True, db_index=True, default=uuid.uuid4)
Agate's avatar
Agate committed
43
    from_activity = models.ForeignKey(
44
        "federation.Activity", null=True, blank=True, on_delete=models.SET_NULL
Agate's avatar
Agate committed
45
    )
46 47 48
    api_includes = []
    creation_date = models.DateTimeField(default=timezone.now)
    import_hooks = []
49

50 51
    class Meta:
        abstract = True
Agate's avatar
Agate committed
52
        ordering = ["-creation_date"]
53 54 55 56 57 58 59 60 61

    @classmethod
    def get_or_create_from_api(cls, mbid):
        try:
            return cls.objects.get(mbid=mbid), False
        except cls.DoesNotExist:
            return cls.create_from_api(id=mbid), True

    def get_api_data(self):
Agate's avatar
Agate committed
62 63 64
        return self.__class__.api.get(id=self.mbid, includes=self.api_includes)[
            self.musicbrainz_model
        ]
65 66 67

    @classmethod
    def create_from_api(cls, **kwargs):
Agate's avatar
Agate committed
68 69 70 71
        if kwargs.get("id"):
            raw_data = cls.api.get(id=kwargs["id"], includes=cls.api_includes)[
                cls.musicbrainz_model
            ]
72
        else:
Agate's avatar
Agate committed
73 74 75
            raw_data = cls.api.search(**kwargs)[
                "{0}-list".format(cls.musicbrainz_model)
            ][0]
76 77 78 79 80 81 82 83 84 85 86
        cleaned_data = cls.clean_musicbrainz_data(raw_data)
        return importers.load(cls, cleaned_data, raw_data, cls.import_hooks)

    @classmethod
    def clean_musicbrainz_data(cls, data):
        cleaned_data = {}
        mapping = importers.Mapping(cls.musicbrainz_mapping)
        for key, value in data.items():
            try:
                cleaned_key, cleaned_value = mapping.from_musicbrainz(key, value)
                cleaned_data[cleaned_key] = cleaned_value
87
            except KeyError:
88 89 90
                pass
        return cleaned_data

91 92 93
    @property
    def musicbrainz_url(self):
        if self.mbid:
Agate's avatar
Agate committed
94 95 96
            return "https://musicbrainz.org/{}/{}".format(
                self.musicbrainz_model, self.mbid
            )
97

Agate's avatar
Agate committed
98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114
    def get_federation_id(self):
        if self.fid:
            return self.fid

        return federation_utils.full_url(
            reverse(
                "federation:music:{}-detail".format(self.federation_namespace),
                kwargs={"uuid": self.uuid},
            )
        )

    def save(self, **kwargs):
        if not self.pk and not self.fid:
            self.fid = self.get_federation_id()

        return super().save(**kwargs)

115

Agate's avatar
Agate committed
116 117
class ArtistQuerySet(models.QuerySet):
    def with_albums_count(self):
Agate's avatar
Agate committed
118
        return self.annotate(_albums_count=models.Count("albums"))
Agate's avatar
Agate committed
119

120 121
    def with_albums(self):
        return self.prefetch_related(
Agate's avatar
Agate committed
122
            models.Prefetch("albums", queryset=Album.objects.with_tracks_count())
123 124
        )

125 126 127 128 129 130 131 132 133 134 135 136 137
    def annotate_playable_by_actor(self, actor):
        tracks = (
            Track.objects.playable_by(actor)
            .filter(artist=models.OuterRef("id"))
            .order_by("id")
            .values("id")[:1]
        )
        subquery = models.Subquery(tracks)
        return self.annotate(is_playable_by_actor=subquery)

    def playable_by(self, actor, include=True):
        tracks = Track.objects.playable_by(actor, include)
        if include:
138
            return self.filter(tracks__in=tracks).distinct()
139
        else:
140
            return self.exclude(tracks__in=tracks).distinct()
141

Agate's avatar
Agate committed
142

143 144
class Artist(APIModelMixin):
    name = models.CharField(max_length=255)
Agate's avatar
Agate committed
145
    federation_namespace = "artists"
Agate's avatar
Agate committed
146
    musicbrainz_model = "artist"
147
    musicbrainz_mapping = {
Agate's avatar
Agate committed
148 149
        "mbid": {"musicbrainz_field_name": "id"},
        "name": {"musicbrainz_field_name": "name"},
150 151
    }
    api = musicbrainz.api.artists
Agate's avatar
Agate committed
152
    objects = ArtistQuerySet.as_manager()
153 154 155 156 157 158 159 160 161 162 163 164

    def __str__(self):
        return self.name

    @property
    def tags(self):
        t = []
        for album in self.albums.all():
            for tag in album.tags:
                t.append(tag)
        return set(t)

165 166
    @classmethod
    def get_or_create_from_name(cls, name, **kwargs):
Agate's avatar
Agate committed
167 168
        kwargs.update({"name": name})
        return cls.objects.get_or_create(name__iexact=name, defaults=kwargs)
169 170


171
def import_artist(v):
Agate's avatar
Agate committed
172
    a = Artist.get_or_create_from_api(mbid=v[0]["artist"]["id"])[0]
173 174
    return a

175

176
def parse_date(v):
177
    d = pendulum.parse(v).date()
178 179 180 181
    return d


def import_tracks(instance, cleaned_data, raw_data):
Agate's avatar
Agate committed
182 183 184 185
    for track_data in raw_data["medium-list"][0]["track-list"]:
        track_cleaned_data = Track.clean_musicbrainz_data(track_data["recording"])
        track_cleaned_data["album"] = instance
        track_cleaned_data["position"] = int(track_data["position"])
186
        importers.load(Track, track_cleaned_data, track_data, Track.import_hooks)
187

188

Agate's avatar
Agate committed
189 190
class AlbumQuerySet(models.QuerySet):
    def with_tracks_count(self):
Agate's avatar
Agate committed
191
        return self.annotate(_tracks_count=models.Count("tracks"))
Agate's avatar
Agate committed
192

193 194 195 196 197 198 199 200 201 202 203 204 205
    def annotate_playable_by_actor(self, actor):
        tracks = (
            Track.objects.playable_by(actor)
            .filter(album=models.OuterRef("id"))
            .order_by("id")
            .values("id")[:1]
        )
        subquery = models.Subquery(tracks)
        return self.annotate(is_playable_by_actor=subquery)

    def playable_by(self, actor, include=True):
        tracks = Track.objects.playable_by(actor, include)
        if include:
206
            return self.filter(tracks__in=tracks).distinct()
207
        else:
208
            return self.exclude(tracks__in=tracks).distinct()
209

Agate's avatar
Agate committed
210

211 212
class Album(APIModelMixin):
    title = models.CharField(max_length=255)
Agate's avatar
Agate committed
213
    artist = models.ForeignKey(Artist, related_name="albums", on_delete=models.CASCADE)
214
    release_date = models.DateField(null=True, blank=True)
215
    release_group_id = models.UUIDField(null=True, blank=True)
Agate's avatar
Agate committed
216 217
    cover = VersatileImageField(
        upload_to="albums/covers/%Y/%m/%d", null=True, blank=True
218
    )
Agate's avatar
Agate committed
219 220
    TYPE_CHOICES = (("album", "Album"),)
    type = models.CharField(choices=TYPE_CHOICES, max_length=30, default="album")
221

Agate's avatar
Agate committed
222
    api_includes = ["artist-credits", "recordings", "media", "release-groups"]
223
    api = musicbrainz.api.releases
Agate's avatar
Agate committed
224
    federation_namespace = "albums"
Agate's avatar
Agate committed
225
    musicbrainz_model = "release"
226
    musicbrainz_mapping = {
Agate's avatar
Agate committed
227 228 229 230
        "mbid": {"musicbrainz_field_name": "id"},
        "position": {
            "musicbrainz_field_name": "release-list",
            "converter": lambda v: int(v[0]["medium-list"][0]["position"]),
231
        },
Agate's avatar
Agate committed
232 233 234
        "release_group_id": {
            "musicbrainz_field_name": "release-group",
            "converter": lambda v: v["id"],
235
        },
Agate's avatar
Agate committed
236 237 238 239 240 241
        "title": {"musicbrainz_field_name": "title"},
        "release_date": {"musicbrainz_field_name": "date", "converter": parse_date},
        "type": {"musicbrainz_field_name": "type", "converter": lambda v: v.lower()},
        "artist": {
            "musicbrainz_field_name": "artist-credit",
            "converter": import_artist,
242 243
        },
    }
Agate's avatar
Agate committed
244
    objects = AlbumQuerySet.as_manager()
245

246 247
    def get_image(self, data=None):
        if data:
Agate's avatar
Agate committed
248 249
            extensions = {"image/jpeg": "jpg", "image/png": "png", "image/gif": "gif"}
            extension = extensions.get(data["mimetype"], "jpg")
Agate's avatar
Agate committed
250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272
            if data.get("content"):
                # we have to cover itself
                f = ContentFile(data["content"])
            elif data.get("url"):
                # we can fetch from a url
                try:
                    response = session.get_session().get(
                        data.get("url"),
                        timeout=3,
                        verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
                    )
                    response.raise_for_status()
                except Exception as e:
                    logger.warn(
                        "Cannot download cover at url %s: %s", data.get("url"), e
                    )
                    return
                else:
                    f = ContentFile(response.content)
            self.cover.save("{}.{}".format(self.uuid, extension), f, save=False)
            self.save(update_fields=["cover"])
            return self.cover.file
        if self.mbid:
Agate's avatar
Agate committed
273
            image_data = musicbrainz.api.images.get_front(str(self.mbid))
274
            f = ContentFile(image_data)
Agate's avatar
Agate committed
275 276
            self.cover.save("{0}.jpg".format(self.mbid), f, save=False)
            self.save(update_fields=["cover"])
277 278 279 280 281 282 283 284 285 286 287 288 289
        return self.cover.file

    def __str__(self):
        return self.title

    @property
    def tags(self):
        t = []
        for track in self.tracks.all():
            for tag in track.tags.all():
                t.append(tag)
        return set(t)

290 291
    @classmethod
    def get_or_create_from_title(cls, title, **kwargs):
Agate's avatar
Agate committed
292 293
        kwargs.update({"title": title})
        return cls.objects.get_or_create(title__iexact=title, defaults=kwargs)
294 295


296 297 298
def import_tags(instance, cleaned_data, raw_data):
    MINIMUM_COUNT = 2
    tags_to_add = []
Agate's avatar
Agate committed
299
    for tag_data in raw_data.get("tag-list", []):
300
        try:
Agate's avatar
Agate committed
301
            if int(tag_data["count"]) < MINIMUM_COUNT:
302 303 304
                continue
        except ValueError:
            continue
Agate's avatar
Agate committed
305
        tags_to_add.append(tag_data["name"])
306 307
    instance.tags.add(*tags_to_add)

308

309
def import_album(v):
Agate's avatar
Agate committed
310
    a = Album.get_or_create_from_api(mbid=v[0]["id"])[0]
311 312 313 314
    return a


def link_recordings(instance, cleaned_data, raw_data):
Agate's avatar
Agate committed
315
    tracks = [r["target"] for r in raw_data["recording-relation-list"]]
316 317 318 319 320 321 322
    Track.objects.filter(mbid__in=tracks).update(work=instance)


def import_lyrics(instance, cleaned_data, raw_data):
    try:
        url = [
            url_data
Agate's avatar
Agate committed
323 324 325
            for url_data in raw_data["url-relation-list"]
            if url_data["type"] == "lyrics"
        ][0]["target"]
326 327 328 329 330 331 332 333 334 335 336 337 338
    except (IndexError, KeyError):
        return
    l, _ = Lyrics.objects.get_or_create(work=instance, url=url)

    return l


class Work(APIModelMixin):
    language = models.CharField(max_length=20)
    nature = models.CharField(max_length=50)
    title = models.CharField(max_length=255)

    api = musicbrainz.api.works
Agate's avatar
Agate committed
339 340
    api_includes = ["url-rels", "recording-rels"]
    musicbrainz_model = "work"
Agate's avatar
Agate committed
341 342
    federation_namespace = "works"

343
    musicbrainz_mapping = {
Agate's avatar
Agate committed
344 345 346 347
        "mbid": {"musicbrainz_field_name": "id"},
        "title": {"musicbrainz_field_name": "title"},
        "language": {"musicbrainz_field_name": "language"},
        "nature": {"musicbrainz_field_name": "type", "converter": lambda v: v.lower()},
348
    }
Agate's avatar
Agate committed
349
    import_hooks = [import_lyrics, link_recordings]
350 351

    def fetch_lyrics(self):
352 353 354
        lyric = self.lyrics.first()
        if lyric:
            return lyric
Agate's avatar
Agate committed
355
        data = self.api.get(self.mbid, includes=["url-rels"])["work"]
356
        lyric = import_lyrics(self, {}, data)
357

358
        return lyric
359

Agate's avatar
Agate committed
360 361 362 363 364 365
    def get_federation_id(self):
        if self.fid:
            return self.fid

        return None

366 367

class Lyrics(models.Model):
Agate's avatar
Agate committed
368
    uuid = models.UUIDField(unique=True, db_index=True, default=uuid.uuid4)
Agate's avatar
Agate committed
369
    work = models.ForeignKey(
Agate's avatar
Agate committed
370 371
        Work, related_name="lyrics", null=True, blank=True, on_delete=models.CASCADE
    )
372 373 374 375 376 377 378 379 380
    url = models.URLField(unique=True)
    content = models.TextField(null=True, blank=True)

    @property
    def content_rendered(self):
        return markdown.markdown(
            self.content,
            safe_mode=True,
            enable_attributes=False,
Agate's avatar
Agate committed
381 382
            extensions=["markdown.extensions.nl2br"],
        )
383 384


Agate's avatar
Agate committed
385 386
class TrackQuerySet(models.QuerySet):
    def for_nested_serialization(self):
387 388 389 390
        return self.select_related().select_related("album__artist", "artist")

    def annotate_playable_by_actor(self, actor):
        files = (
Agate's avatar
Agate committed
391
            Upload.objects.playable_by(actor)
392 393 394
            .filter(track=models.OuterRef("id"))
            .order_by("id")
            .values("id")[:1]
Agate's avatar
Agate committed
395
        )
396 397 398 399
        subquery = models.Subquery(files)
        return self.annotate(is_playable_by_actor=subquery)

    def playable_by(self, actor, include=True):
Agate's avatar
Agate committed
400
        files = Upload.objects.playable_by(actor, include)
401
        if include:
402
            return self.filter(uploads__in=files).distinct()
403
        else:
404
            return self.exclude(uploads__in=files).distinct()
Agate's avatar
Agate committed
405 406 407 408 409 410 411 412 413 414 415 416 417 418

    def annotate_duration(self):
        first_upload = Upload.objects.filter(track=models.OuterRef("pk")).order_by("pk")
        return self.annotate(
            duration=models.Subquery(first_upload.values("duration")[:1])
        )

    def annotate_file_data(self):
        first_upload = Upload.objects.filter(track=models.OuterRef("pk")).order_by("pk")
        return self.annotate(
            bitrate=models.Subquery(first_upload.values("bitrate")[:1]),
            size=models.Subquery(first_upload.values("size")[:1]),
            mimetype=models.Subquery(first_upload.values("mimetype")[:1]),
        )
Agate's avatar
Agate committed
419 420


421 422
def get_artist(release_list):
    return Artist.get_or_create_from_api(
Agate's avatar
Agate committed
423 424
        mbid=release_list[0]["artist-credits"][0]["artists"]["id"]
    )[0]
425 426


427 428
class Track(APIModelMixin):
    title = models.CharField(max_length=255)
Agate's avatar
Agate committed
429
    artist = models.ForeignKey(Artist, related_name="tracks", on_delete=models.CASCADE)
430
    position = models.PositiveIntegerField(null=True, blank=True)
Agate's avatar
Agate committed
431
    album = models.ForeignKey(
Agate's avatar
Agate committed
432 433
        Album, related_name="tracks", null=True, blank=True, on_delete=models.CASCADE
    )
Agate's avatar
Agate committed
434
    work = models.ForeignKey(
Agate's avatar
Agate committed
435 436
        Work, related_name="tracks", null=True, blank=True, on_delete=models.CASCADE
    )
Agate's avatar
Agate committed
437
    federation_namespace = "tracks"
Agate's avatar
Agate committed
438
    musicbrainz_model = "recording"
439
    api = musicbrainz.api.recordings
Agate's avatar
Agate committed
440
    api_includes = ["artist-credits", "releases", "media", "tags", "work-rels"]
441
    musicbrainz_mapping = {
Agate's avatar
Agate committed
442 443 444
        "mbid": {"musicbrainz_field_name": "id"},
        "title": {"musicbrainz_field_name": "title"},
        "artist": {
445 446 447 448
            "musicbrainz_field_name": "artist-credit",
            "converter": lambda v: Artist.get_or_create_from_api(
                mbid=v[0]["artist"]["id"]
            )[0],
449
        },
Agate's avatar
Agate committed
450
        "album": {"musicbrainz_field_name": "release-list", "converter": import_album},
451
    }
Agate's avatar
Agate committed
452
    import_hooks = [import_tags]
Agate's avatar
Agate committed
453
    objects = TrackQuerySet.as_manager()
454
    tags = TaggableManager(blank=True)
455

456
    class Meta:
Agate's avatar
Agate committed
457
        ordering = ["album", "position"]
458

459 460 461 462 463 464
    def __str__(self):
        return self.title

    def save(self, **kwargs):
        try:
            self.artist
465
        except Artist.DoesNotExist:
466 467 468 469 470 471
            self.artist = self.album.artist
        super().save(**kwargs)

    def get_work(self):
        if self.work:
            return self.work
Agate's avatar
Agate committed
472
        data = self.api.get(self.mbid, includes=["work-rels"])
473
        try:
Agate's avatar
Agate committed
474
            work_data = data["recording"]["work-relation-list"][0]["work"]
475 476
        except (IndexError, KeyError):
            return
Agate's avatar
Agate committed
477
        work, _ = Work.get_or_create_from_api(mbid=work_data["id"])
478 479 480
        return work

    def get_lyrics_url(self):
Agate's avatar
Agate committed
481
        return reverse("api:v1:tracks-lyrics", kwargs={"pk": self.pk})
482 483 484 485

    @property
    def full_name(self):
        try:
Agate's avatar
Agate committed
486
            return "{} - {} - {}".format(self.artist.name, self.album.title, self.title)
487
        except AttributeError:
Agate's avatar
Agate committed
488
            return "{} - {}".format(self.artist.name, self.title)
489

490 491
    def get_activity_url(self):
        if self.mbid:
Agate's avatar
Agate committed
492 493
            return "https://musicbrainz.org/recording/{}".format(self.mbid)
        return settings.FUNKWHALE_URL + "/tracks/{}".format(self.pk)
494

495 496
    @classmethod
    def get_or_create_from_title(cls, title, **kwargs):
Agate's avatar
Agate committed
497 498
        kwargs.update({"title": title})
        return cls.objects.get_or_create(title__iexact=title, defaults=kwargs)
499

500 501 502 503 504 505 506 507 508 509 510
    @classmethod
    def get_or_create_from_release(cls, release_mbid, mbid):
        release_mbid = str(release_mbid)
        mbid = str(mbid)
        try:
            return cls.objects.get(mbid=mbid), False
        except cls.DoesNotExist:
            pass

        album = Album.get_or_create_from_api(release_mbid)[0]
        data = musicbrainz.client.api.releases.get(
Agate's avatar
Agate committed
511 512 513
            str(album.mbid), includes=Album.api_includes
        )
        tracks = [t for m in data["release"]["medium-list"] for t in m["track-list"]]
514 515
        track_data = None
        for track in tracks:
516
            if track["recording"]["id"] == str(mbid):
517 518 519
                track_data = track
                break
        if not track_data:
Agate's avatar
Agate committed
520
            raise ValueError("No track found matching this ID")
521

522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539
        track_artist_mbid = None
        for ac in track_data["recording"]["artist-credit"]:
            try:
                ac_mbid = ac["artist"]["id"]
            except TypeError:
                # it's probably a string, like "feat."
                continue

            if ac_mbid == str(album.artist.mbid):
                continue

            track_artist_mbid = ac_mbid
            break
        track_artist_mbid = track_artist_mbid or album.artist.mbid
        if track_artist_mbid == str(album.artist.mbid):
            track_artist = album.artist
        else:
            track_artist = Artist.get_or_create_from_api(track_artist_mbid)[0]
540 541 542
        return cls.objects.update_or_create(
            mbid=mbid,
            defaults={
Agate's avatar
Agate committed
543 544 545
                "position": int(track["position"]),
                "title": track["recording"]["title"],
                "album": album,
546
                "artist": track_artist,
Agate's avatar
Agate committed
547
            },
548
        )
Agate's avatar
Agate committed
549

550 551 552 553 554
    @property
    def listen_url(self):
        return reverse("api:v1:listen-detail", kwargs={"uuid": self.uuid})


Agate's avatar
Agate committed
555
class UploadQuerySet(models.QuerySet):
556
    def playable_by(self, actor, include=True):
Agate's avatar
Agate committed
557
        libraries = Library.objects.viewable_by(actor)
558 559

        if include:
Agate's avatar
Agate committed
560 561 562
            return self.filter(
                library__in=libraries, import_status="finished"
            ).distinct()
563
        return self.exclude(library__in=libraries, import_status="finished").distinct()
564 565 566 567

    def local(self, include=True):
        return self.exclude(library__actor__user__isnull=include)

Agate's avatar
Agate committed
568 569 570
    def for_federation(self):
        return self.filter(import_status="finished", mimetype__startswith="audio/")

571 572 573 574 575 576 577 578 579 580

TRACK_FILE_IMPORT_STATUS_CHOICES = (
    ("pending", "Pending"),
    ("finished", "Finished"),
    ("errored", "Errored"),
    ("skipped", "Skipped"),
)


def get_file_path(instance, filename):
581 582 583
    if isinstance(instance, UploadVersion):
        return common_utils.ChunkedPath("transcoded")(instance, filename)

584
    if instance.library.actor.get_user():
585 586 587 588 589 590 591 592 593
        return common_utils.ChunkedPath("tracks")(instance, filename)
    else:
        # we cache remote tracks in a different directory
        return common_utils.ChunkedPath("federation_cache/tracks")(instance, filename)


def get_import_reference():
    return str(uuid.uuid4())

Agate's avatar
Agate committed
594

Agate's avatar
Agate committed
595
class Upload(models.Model):
596
    fid = models.URLField(unique=True, max_length=500, null=True, blank=True)
Agate's avatar
Agate committed
597
    uuid = models.UUIDField(unique=True, db_index=True, default=uuid.uuid4)
598
    track = models.ForeignKey(
Agate's avatar
Agate committed
599
        Track, related_name="uploads", on_delete=models.CASCADE, null=True, blank=True
600 601 602 603 604 605 606 607
    )
    audio_file = models.FileField(upload_to=get_file_path, max_length=255)
    source = models.CharField(
        # URL validators are not flexible enough for our file:// and upload:// schemes
        null=True,
        blank=True,
        max_length=500,
    )
608
    creation_date = models.DateTimeField(default=timezone.now)
Agate's avatar
Agate committed
609
    modification_date = models.DateTimeField(default=timezone.now, null=True)
610
    accessed_date = models.DateTimeField(null=True, blank=True)
611
    duration = models.IntegerField(null=True, blank=True)
612 613
    size = models.IntegerField(null=True, blank=True)
    bitrate = models.IntegerField(null=True, blank=True)
614
    acoustid_track_id = models.UUIDField(null=True, blank=True)
615
    mimetype = models.CharField(null=True, blank=True, max_length=200)
616
    library = models.ForeignKey(
Agate's avatar
Agate committed
617 618 619 620 621
        "library",
        null=True,
        blank=True,
        related_name="uploads",
        on_delete=models.CASCADE,
622
    )
623

624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643
    # metadata from federation
    metadata = JSONField(
        default=empty_dict, max_length=50000, encoder=DjangoJSONEncoder
    )
    import_date = models.DateTimeField(null=True, blank=True)
    # optionnal metadata provided during import
    import_metadata = JSONField(
        default=empty_dict, max_length=50000, encoder=DjangoJSONEncoder
    )
    # status / error details for the import
    import_status = models.CharField(
        default="pending", choices=TRACK_FILE_IMPORT_STATUS_CHOICES, max_length=25
    )
    # a short reference provided by the client to group multiple files
    # in the same import
    import_reference = models.CharField(max_length=50, default=get_import_reference)

    # optionnal metadata about import results (error messages, etc.)
    import_details = JSONField(
        default=empty_dict, max_length=50000, encoder=DjangoJSONEncoder
644
    )
Agate's avatar
Agate committed
645 646 647
    from_activity = models.ForeignKey(
        "federation.Activity", null=True, on_delete=models.SET_NULL
    )
648

Agate's avatar
Agate committed
649
    objects = UploadQuerySet.as_manager()
650 651 652 653 654 655 656 657 658 659 660 661 662 663 664

    def download_audio_from_remote(self, user):
        from funkwhale_api.common import session
        from funkwhale_api.federation import signing

        if user.is_authenticated and user.actor:
            auth = signing.get_auth(user.actor.private_key, user.actor.private_key_id)
        else:
            auth = None

        remote_response = session.get_session().get(
            self.source,
            auth=auth,
            stream=True,
            timeout=20,
Agate's avatar
Agate committed
665
            headers={"Content-Type": "application/octet-stream"},
666
            verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
667
        )
668 669 670 671 672 673 674 675 676 677 678 679
        with remote_response as r:
            remote_response.raise_for_status()
            extension = utils.get_ext_from_type(self.mimetype)
            title = " - ".join(
                [self.track.title, self.track.album.title, self.track.artist.name]
            )
            filename = "{}.{}".format(title, extension)
            tmp_file = tempfile.TemporaryFile()
            for chunk in r.iter_content(chunk_size=512):
                tmp_file.write(chunk)
            self.audio_file.save(filename, tmp_file, save=False)
            self.save(update_fields=["audio_file"])
680

681 682 683
    def get_federation_id(self):
        if self.fid:
            return self.fid
684

Agate's avatar
Agate committed
685 686 687
        return federation_utils.full_url(
            reverse("federation:music:uploads-detail", kwargs={"uuid": self.uuid})
        )
688

689 690
    @property
    def filename(self):
Agate's avatar
Agate committed
691
        return "{}.{}".format(self.track.full_name, self.extension)
692 693 694 695 696

    @property
    def extension(self):
        if not self.audio_file:
            return
Agate's avatar
Agate committed
697
        return os.path.splitext(self.audio_file.name)[-1].replace(".", "", 1)
698

699 700 701 702
    def get_file_size(self):
        if self.audio_file:
            return self.audio_file.size

Agate's avatar
Agate committed
703 704
        if self.source.startswith("file://"):
            return os.path.getsize(self.source.replace("file://", "", 1))
705 706 707 708

    def get_audio_file(self):
        if self.audio_file:
            return self.audio_file.open()
Agate's avatar
Agate committed
709 710
        if self.source.startswith("file://"):
            return open(self.source.replace("file://", "", 1), "rb")
711

712
    def get_audio_data(self):
713
        audio_file = self.get_audio_file()
714 715 716 717 718 719 720 721 722 723
        if not audio_file:
            return
        audio_data = utils.get_audio_file_data(audio_file)
        if not audio_data:
            return
        return {
            "duration": int(audio_data["length"]),
            "bitrate": audio_data["bitrate"],
            "size": self.get_file_size(),
        }
724

725
    def save(self, **kwargs):
Agate's avatar
Agate committed
726 727 728 729 730
        if not self.mimetype:
            if self.audio_file:
                self.mimetype = utils.guess_mimetype(self.audio_file)
            elif self.source and self.source.startswith("file://"):
                self.mimetype = mimetypes.guess_type(self.source)[0]
731 732
        if not self.size and self.audio_file:
            self.size = self.audio_file.size
733
        if not self.pk and not self.fid and self.library.actor.get_user():
Agate's avatar
Agate committed
734
            self.fid = self.get_federation_id()
735
        return super().save(**kwargs)
736

737 738 739 740 741 742
    def get_metadata(self):
        audio_file = self.get_audio_file()
        if not audio_file:
            return
        return metadata.Metadata(audio_file)

743 744
    @property
    def listen_url(self):
Agate's avatar
Agate committed
745
        return self.track.listen_url + "?upload={}".format(self.uuid)
746

747

748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769
MIMETYPE_CHOICES = [
    (mt, ext) for ext, mt in utils.AUDIO_EXTENSIONS_AND_MIMETYPE
]


class UploadVersion(models.Model):
    upload = models.ForeignKey(Upload, related_name='versions', on_delete=models.CASCADE)
    mimetype = models.CharField(max_length=50, choices=MIMETYPE_CHOICES)
    creation_date = models.DateTimeField(default=timezone.now)
    accessed_date = models.DateTimeField(null=True, blank=True)
    audio_file = models.FileField(upload_to=get_file_path, max_length=255)
    bitrate = models.PositiveIntegerField()
    size = models.IntegerField()

    class Meta:
        unique_together = ('upload', 'mimetype', 'bitrate')

    @property
    def filename(self):
        return self.upload.filename


770
IMPORT_STATUS_CHOICES = (
Agate's avatar
Agate committed
771 772 773 774
    ("pending", "Pending"),
    ("finished", "Finished"),
    ("errored", "Errored"),
    ("skipped", "Skipped"),
775 776
)

777

778
class ImportBatch(models.Model):
Agate's avatar
Agate committed
779
    uuid = models.UUIDField(unique=True, db_index=True, default=uuid.uuid4)
780
    IMPORT_BATCH_SOURCES = [
Agate's avatar
Agate committed
781 782 783
        ("api", "api"),
        ("shell", "shell"),
        ("federation", "federation"),
784 785
    ]
    source = models.CharField(
Agate's avatar
Agate committed
786 787
        max_length=30, default="api", choices=IMPORT_BATCH_SOURCES
    )
788
    creation_date = models.DateTimeField(default=timezone.now)
Agate's avatar
Agate committed
789
    submitted_by = models.ForeignKey(
Agate's avatar
Agate committed
790 791
        "users.User",
        related_name="imports",
792 793
        null=True,
        blank=True,
Agate's avatar
Agate committed
794 795
        on_delete=models.CASCADE,
    )
796
    status = models.CharField(
Agate's avatar
Agate committed
797 798
        choices=IMPORT_STATUS_CHOICES, default="pending", max_length=30
    )
799
    import_request = models.ForeignKey(
Agate's avatar
Agate committed
800 801
        "requests.ImportRequest",
        related_name="import_batches",
802 803
        null=True,
        blank=True,
804
        on_delete=models.SET_NULL,
Agate's avatar
Agate committed
805
    )
806 807 808 809 810 811 812
    library = models.ForeignKey(
        "Library",
        related_name="import_batches",
        null=True,
        blank=True,
        on_delete=models.CASCADE,
    )
813

814
    class Meta:
Agate's avatar
Agate committed
815
        ordering = ["-creation_date"]
816 817 818 819

    def __str__(self):
        return str(self.pk)

820
    def update_status(self):
821
        old_status = self.status
822
        self.status = utils.compute_status(self.jobs.all())
823 824
        if self.status == old_status:
            return
Agate's avatar
Agate committed
825 826
        self.save(update_fields=["status"])
        if self.status != old_status and self.status == "finished":
827
            from . import tasks
Agate's avatar
Agate committed
828

829 830
            tasks.import_batch_notify_followers.delay(import_batch_id=self.pk)

831
    def get_federation_id(self):
832
        return federation_utils.full_url(
Agate's avatar
Agate committed
833
            "/federation/music/import/batch/{}".format(self.uuid)
834
        )
835

836 837

class ImportJob(models.Model):
Agate's avatar
Agate committed
838
    uuid = models.UUIDField(unique=True, db_index=True, default=uuid.uuid4)
Renon's avatar
Renon committed
839
    replace_if_duplicate = models.BooleanField(default=False)
Agate's avatar
Agate committed
840
    batch = models.ForeignKey(
Agate's avatar
Agate committed
841 842
        ImportBatch, related_name="jobs", on_delete=models.CASCADE
    )
Agate's avatar
Agate committed
843 844
    upload = models.ForeignKey(
        Upload, related_name="jobs", null=True, blank=True, on_delete=models.CASCADE
Agate's avatar
Agate committed
845
    )
Agate's avatar
Agate committed
846
    source = models.CharField(max_length=500)
847
    mbid = models.UUIDField(editable=False, null=True, blank=True)
848 849

    status = models.CharField(
Agate's avatar
Agate committed
850 851
        choices=IMPORT_STATUS_CHOICES, default="pending", max_length=30
    )
852
    audio_file = models.FileField(
Agate's avatar
Agate committed
853 854
        upload_to="imports/%Y/%m/%d", max_length=255, null=True, blank=True
    )
855 856

    library_track = models.ForeignKey(
Agate's avatar
Agate committed
857 858
        "federation.LibraryTrack",
        related_name="import_jobs",
859 860
        on_delete=models.SET_NULL,
        null=True,
Agate's avatar
Agate committed
861
        blank=True,
862
    )
863
    audio_file_size = models.IntegerField(null=True, blank=True)
864

865
    class Meta:
Agate's avatar
Agate committed
866
        ordering = ("id",)
Agate's avatar
Agate committed
867

868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888
    def save(self, **kwargs):
        if self.audio_file and not self.audio_file_size:
            self.audio_file_size = self.audio_file.size
        return super().save(**kwargs)


LIBRARY_PRIVACY_LEVEL_CHOICES = [
    (k, l) for k, l in fields.PRIVACY_LEVEL_CHOICES if k != "followers"
]


class LibraryQuerySet(models.QuerySet):
    def with_follows(self, actor):
        return self.prefetch_related(
            models.Prefetch(
                "received_follows",
                queryset=federation_models.LibraryFollow.objects.filter(actor=actor),
                to_attr="_follows",
            )
        )

Agate's avatar
Agate committed
889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906
    def viewable_by(self, actor):
        from funkwhale_api.federation.models import LibraryFollow

        if actor is None:
            return Library.objects.filter(privacy_level="everyone")

        me_query = models.Q(privacy_level="me", actor=actor)
        instance_query = models.Q(privacy_level="instance", actor__domain=actor.domain)
        followed_libraries = LibraryFollow.objects.filter(
            actor=actor, approved=True
        ).values_list("target", flat=True)
        return Library.objects.filter(
            me_query
            | instance_query
            | models.Q(privacy_level="everyone")
            | models.Q(pk__in=followed_libraries)
        )

907 908 909 910 911 912 913 914 915 916 917 918 919

class Library(federation_models.FederationMixin):
    uuid = models.UUIDField(unique=True, db_index=True, default=uuid.uuid4)
    actor = models.ForeignKey(
        "federation.Actor", related_name="libraries", on_delete=models.CASCADE
    )
    followers_url = models.URLField(max_length=500)
    creation_date = models.DateTimeField(default=timezone.now)
    name = models.CharField(max_length=100)
    description = models.TextField(max_length=5000, null=True, blank=True)
    privacy_level = models.CharField(
        choices=LIBRARY_PRIVACY_LEVEL_CHOICES, default="me", max_length=25
    )
Agate's avatar
Agate committed
920
    uploads_count = models.PositiveIntegerField(default=0)
921 922 923 924 925 926 927 928
    objects = LibraryQuerySet.as_manager()

    def get_federation_id(self):
        return federation_utils.full_url(
            reverse("federation:music:libraries-detail", kwargs={"uuid": self.uuid})
        )

    def save(self, **kwargs):
929
        if not self.pk and not self.fid and self.actor.get_user():
930 931 932 933 934 935 936 937
            self.fid = self.get_federation_id()
            self.followers_url = self.fid + "/followers"

        return super().save(**kwargs)

    def should_autoapprove_follow(self, actor):
        if self.privacy_level == "everyone":
            return True
938
        if self.privacy_level == "instance" and actor.get_user():
939 940 941
            return True
        return False

Agate's avatar
Agate committed
942 943 944 945
    def schedule_scan(self, actor, force=False):
        latest_scan = (
            self.scans.exclude(status="errored").order_by("-creation_date").first()
        )
946 947
        delay_between_scans = datetime.timedelta(seconds=3600 * 24)
        now = timezone.now()
Agate's avatar
Agate committed
948 949 950 951 952
        if (
            not force
            and latest_scan
            and latest_scan.creation_date + delay_between_scans > now
        ):
953 954
            return

Agate's avatar
Agate committed
955
        scan = self.scans.create(total_files=self.uploads_count, actor=actor)
956 957 958 959 960 961 962 963 964
        from . import tasks

        common_utils.on_commit(tasks.start_library_scan.delay, library_scan_id=scan.pk)
        return scan


SCAN_STATUS = [
    ("pending", "pending"),
    ("scanning", "scanning"),
Agate's avatar
Agate committed
965
    ("errored", "errored"),
966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981
    ("finished", "finished"),
]


class LibraryScan(models.Model):
    actor = models.ForeignKey(
        "federation.Actor", null=True, blank=True, on_delete=models.CASCADE
    )
    library = models.ForeignKey(Library, related_name="scans", on_delete=models.CASCADE)
    total_files = models.PositiveIntegerField(default=0)
    processed_files = models.PositiveIntegerField(default=0)
    errored_files = models.PositiveIntegerField(default=0)
    status = models.CharField(default="pending", max_length=25)
    creation_date = models.DateTimeField(default=timezone.now)
    modification_date = models.DateTimeField(null=True, blank=True)

Agate's avatar
Agate committed
982 983 984 985

@receiver(post_save, sender=ImportJob)
def update_batch_status(sender, instance, **kwargs):
    instance.batch.update_status()
986 987 988 989


@receiver(post_save, sender=ImportBatch)
def update_request_status(sender, instance, created, **kwargs):
Agate's avatar
Agate committed
990
    update_fields = kwargs.get("update_fields", []) or []
991 992 993
    if not instance.import_request:
        return

994
    if not created and "status" not in update_fields:
995 996 997 998 999
        return

    r_status = instance.import_request.status
    status = instance.status

Agate's avatar
Agate committed
1000
    if status == "pending" and r_status == "pending":
1001
        # let's mark the request as accepted since we started an import
Agate's avatar
Agate committed
1002 1003
        instance.import_request.status = "accepted"
        return instance.import_request.save(update_fields=["status"])
1004

Agate's avatar
Agate committed
1005
    if status == "finished" and r_status == "accepted":
1006
        # let's mark the request as imported since the import is over