Newer
Older
Eliot Berriot
committed
import shutil
Eliot Berriot
committed
Eliot Berriot
committed
import pendulum
Eliot Berriot
committed
from django.conf import settings
from django.core.files import File
from django.core.files.base import ContentFile
from django.db import models
from django.db.models.signals import post_save
from django.dispatch import receiver
Eliot Berriot
committed
from django.utils import timezone
from taggit.managers import TaggableManager
Eliot Berriot
committed
from versatileimagefield.fields import VersatileImageField
from versatileimagefield.image_warmer import VersatileImageFieldWarmer
Eliot Berriot
committed
from funkwhale_api import downloader, musicbrainz
from funkwhale_api.federation import utils as federation_utils
Eliot Berriot
committed
class APIModelMixin(models.Model):
mbid = models.UUIDField(unique=True, db_index=True, null=True, blank=True)
uuid = models.UUIDField(unique=True, db_index=True, default=uuid.uuid4)
Eliot Berriot
committed
api_includes = []
creation_date = models.DateTimeField(default=timezone.now)
import_hooks = []
Eliot Berriot
committed
Eliot Berriot
committed
class Meta:
abstract = True
Eliot Berriot
committed
@classmethod
def get_or_create_from_api(cls, mbid):
try:
return cls.objects.get(mbid=mbid), False
except cls.DoesNotExist:
return cls.create_from_api(id=mbid), True
def get_api_data(self):
return self.__class__.api.get(id=self.mbid, includes=self.api_includes)[
self.musicbrainz_model
]
Eliot Berriot
committed
@classmethod
def create_from_api(cls, **kwargs):
if kwargs.get("id"):
raw_data = cls.api.get(id=kwargs["id"], includes=cls.api_includes)[
cls.musicbrainz_model
]
Eliot Berriot
committed
else:
raw_data = cls.api.search(**kwargs)[
"{0}-list".format(cls.musicbrainz_model)
][0]
Eliot Berriot
committed
cleaned_data = cls.clean_musicbrainz_data(raw_data)
return importers.load(cls, cleaned_data, raw_data, cls.import_hooks)
@classmethod
def clean_musicbrainz_data(cls, data):
cleaned_data = {}
mapping = importers.Mapping(cls.musicbrainz_mapping)
for key, value in data.items():
try:
cleaned_key, cleaned_value = mapping.from_musicbrainz(key, value)
cleaned_data[cleaned_key] = cleaned_value
Eliot Berriot
committed
pass
return cleaned_data
@property
def musicbrainz_url(self):
if self.mbid:
return "https://musicbrainz.org/{}/{}".format(
self.musicbrainz_model, self.mbid
)
class ArtistQuerySet(models.QuerySet):
def with_albums_count(self):
return self.annotate(_albums_count=models.Count("albums"))
def with_albums(self):
return self.prefetch_related(
models.Prefetch("albums", queryset=Album.objects.with_tracks_count())
Eliot Berriot
committed
class Artist(APIModelMixin):
name = models.CharField(max_length=255)
Eliot Berriot
committed
musicbrainz_mapping = {
"mbid": {"musicbrainz_field_name": "id"},
"name": {"musicbrainz_field_name": "name"},
Eliot Berriot
committed
}
api = musicbrainz.api.artists
Eliot Berriot
committed
def __str__(self):
return self.name
@property
def tags(self):
t = []
for album in self.albums.all():
for tag in album.tags:
t.append(tag)
return set(t)
@classmethod
def get_or_create_from_name(cls, name, **kwargs):
kwargs.update({"name": name})
return cls.objects.get_or_create(name__iexact=name, defaults=kwargs)
Eliot Berriot
committed
def import_artist(v):
a = Artist.get_or_create_from_api(mbid=v[0]["artist"]["id"])[0]
Eliot Berriot
committed
return a
Eliot Berriot
committed
def parse_date(v):
Eliot Berriot
committed
d = pendulum.parse(v).date()
Eliot Berriot
committed
return d
def import_tracks(instance, cleaned_data, raw_data):
for track_data in raw_data["medium-list"][0]["track-list"]:
track_cleaned_data = Track.clean_musicbrainz_data(track_data["recording"])
track_cleaned_data["album"] = instance
track_cleaned_data["position"] = int(track_data["position"])
importers.load(Track, track_cleaned_data, track_data, Track.import_hooks)
Eliot Berriot
committed
class AlbumQuerySet(models.QuerySet):
def with_tracks_count(self):
return self.annotate(_tracks_count=models.Count("tracks"))
Eliot Berriot
committed
class Album(APIModelMixin):
title = models.CharField(max_length=255)
artist = models.ForeignKey(Artist, related_name="albums", on_delete=models.CASCADE)
release_date = models.DateField(null=True, blank=True)
release_group_id = models.UUIDField(null=True, blank=True)
cover = VersatileImageField(
upload_to="albums/covers/%Y/%m/%d", null=True, blank=True
Eliot Berriot
committed
)
TYPE_CHOICES = (("album", "Album"),)
type = models.CharField(choices=TYPE_CHOICES, max_length=30, default="album")
Eliot Berriot
committed
api_includes = ["artist-credits", "recordings", "media", "release-groups"]
Eliot Berriot
committed
api = musicbrainz.api.releases
Eliot Berriot
committed
musicbrainz_mapping = {
"mbid": {"musicbrainz_field_name": "id"},
"position": {
"musicbrainz_field_name": "release-list",
"converter": lambda v: int(v[0]["medium-list"][0]["position"]),
Eliot Berriot
committed
},
"release_group_id": {
"musicbrainz_field_name": "release-group",
"converter": lambda v: v["id"],
Eliot Berriot
committed
},
"title": {"musicbrainz_field_name": "title"},
"release_date": {"musicbrainz_field_name": "date", "converter": parse_date},
"type": {"musicbrainz_field_name": "type", "converter": lambda v: v.lower()},
"artist": {
"musicbrainz_field_name": "artist-credit",
"converter": import_artist,
Eliot Berriot
committed
},
}
Eliot Berriot
committed
def get_image(self, data=None):
if data:
f = ContentFile(data["content"])
extensions = {"image/jpeg": "jpg", "image/png": "png", "image/gif": "gif"}
extension = extensions.get(data["mimetype"], "jpg")
self.cover.save("{}.{}".format(self.uuid, extension), f)
else:
image_data = musicbrainz.api.images.get_front(str(self.mbid))
f = ContentFile(image_data)
Eliot Berriot
committed
return self.cover.file
def __str__(self):
return self.title
@property
def tags(self):
t = []
for track in self.tracks.all():
for tag in track.tags.all():
t.append(tag)
return set(t)
@classmethod
def get_or_create_from_title(cls, title, **kwargs):
kwargs.update({"title": title})
return cls.objects.get_or_create(title__iexact=title, defaults=kwargs)
Eliot Berriot
committed
def import_tags(instance, cleaned_data, raw_data):
MINIMUM_COUNT = 2
tags_to_add = []
Eliot Berriot
committed
try:
Eliot Berriot
committed
continue
except ValueError:
continue
Eliot Berriot
committed
instance.tags.add(*tags_to_add)
Eliot Berriot
committed
def import_album(v):
Eliot Berriot
committed
return a
def link_recordings(instance, cleaned_data, raw_data):
tracks = [r["target"] for r in raw_data["recording-relation-list"]]
Eliot Berriot
committed
Track.objects.filter(mbid__in=tracks).update(work=instance)
def import_lyrics(instance, cleaned_data, raw_data):
try:
url = [
url_data
for url_data in raw_data["url-relation-list"]
if url_data["type"] == "lyrics"
][0]["target"]
Eliot Berriot
committed
except (IndexError, KeyError):
return
l, _ = Lyrics.objects.get_or_create(work=instance, url=url)
return l
class Work(APIModelMixin):
language = models.CharField(max_length=20)
nature = models.CharField(max_length=50)
title = models.CharField(max_length=255)
api = musicbrainz.api.works
api_includes = ["url-rels", "recording-rels"]
musicbrainz_model = "work"
Eliot Berriot
committed
musicbrainz_mapping = {
"mbid": {"musicbrainz_field_name": "id"},
"title": {"musicbrainz_field_name": "title"},
"language": {"musicbrainz_field_name": "language"},
"nature": {"musicbrainz_field_name": "type", "converter": lambda v: v.lower()},
Eliot Berriot
committed
}
Eliot Berriot
committed
def fetch_lyrics(self):
lyric = self.lyrics.first()
if lyric:
return lyric
data = self.api.get(self.mbid, includes=["url-rels"])["work"]
lyric = import_lyrics(self, {}, data)
Eliot Berriot
committed
Eliot Berriot
committed
class Lyrics(models.Model):
uuid = models.UUIDField(unique=True, db_index=True, default=uuid.uuid4)
Work, related_name="lyrics", null=True, blank=True, on_delete=models.CASCADE
)
Eliot Berriot
committed
url = models.URLField(unique=True)
content = models.TextField(null=True, blank=True)
@property
def content_rendered(self):
return markdown.markdown(
self.content,
safe_mode=True,
enable_attributes=False,
Eliot Berriot
committed
class TrackQuerySet(models.QuerySet):
def for_nested_serialization(self):
return (
self.select_related()
.select_related("album__artist", "artist")
.prefetch_related("files")
)
Eliot Berriot
committed
def get_artist(release_list):
return Artist.get_or_create_from_api(
mbid=release_list[0]["artist-credits"][0]["artists"]["id"]
)[0]
Eliot Berriot
committed
Eliot Berriot
committed
class Track(APIModelMixin):
title = models.CharField(max_length=255)
artist = models.ForeignKey(Artist, related_name="tracks", on_delete=models.CASCADE)
Eliot Berriot
committed
position = models.PositiveIntegerField(null=True, blank=True)
Album, related_name="tracks", null=True, blank=True, on_delete=models.CASCADE
)
Work, related_name="tracks", null=True, blank=True, on_delete=models.CASCADE
)
Eliot Berriot
committed
Eliot Berriot
committed
api = musicbrainz.api.recordings
api_includes = ["artist-credits", "releases", "media", "tags", "work-rels"]
Eliot Berriot
committed
musicbrainz_mapping = {
"mbid": {"musicbrainz_field_name": "id"},
"title": {"musicbrainz_field_name": "title"},
"artist": {
"musicbrainz_field_name": "artist-credit",
"converter": lambda v: Artist.get_or_create_from_api(
mbid=v[0]["artist"]["id"]
)[0],
Eliot Berriot
committed
},
"album": {"musicbrainz_field_name": "release-list", "converter": import_album},
Eliot Berriot
committed
}
tags = TaggableManager(blank=True)
Eliot Berriot
committed
Eliot Berriot
committed
class Meta:
Eliot Berriot
committed
Eliot Berriot
committed
def __str__(self):
return self.title
def save(self, **kwargs):
try:
self.artist
except Artist.DoesNotExist:
Eliot Berriot
committed
self.artist = self.album.artist
super().save(**kwargs)
def get_work(self):
if self.work:
return self.work
Eliot Berriot
committed
try:
work_data = data["recording"]["work-relation-list"][0]["work"]
Eliot Berriot
committed
except (IndexError, KeyError):
return
work, _ = Work.get_or_create_from_api(mbid=work_data["id"])
Eliot Berriot
committed
return work
def get_lyrics_url(self):
return reverse("api:v1:tracks-lyrics", kwargs={"pk": self.pk})
Eliot Berriot
committed
@property
def full_name(self):
try:
return "{} - {} - {}".format(self.artist.name, self.album.title, self.title)
Eliot Berriot
committed
except AttributeError:
Eliot Berriot
committed
def get_activity_url(self):
if self.mbid:
return "https://musicbrainz.org/recording/{}".format(self.mbid)
return settings.FUNKWHALE_URL + "/tracks/{}".format(self.pk)
@classmethod
def get_or_create_from_title(cls, title, **kwargs):
kwargs.update({"title": title})
return cls.objects.get_or_create(title__iexact=title, defaults=kwargs)
@classmethod
def get_or_create_from_release(cls, release_mbid, mbid):
release_mbid = str(release_mbid)
mbid = str(mbid)
try:
return cls.objects.get(mbid=mbid), False
except cls.DoesNotExist:
pass
album = Album.get_or_create_from_api(release_mbid)[0]
data = musicbrainz.client.api.releases.get(
str(album.mbid), includes=Album.api_includes
)
tracks = [t for m in data["release"]["medium-list"] for t in m["track-list"]]
track_data = None
for track in tracks:
if track["recording"]["id"] == str(mbid):
track_data = track
break
if not track_data:
track_artist_mbid = None
for ac in track_data["recording"]["artist-credit"]:
try:
ac_mbid = ac["artist"]["id"]
except TypeError:
# it's probably a string, like "feat."
continue
if ac_mbid == str(album.artist.mbid):
continue
track_artist_mbid = ac_mbid
break
track_artist_mbid = track_artist_mbid or album.artist.mbid
if track_artist_mbid == str(album.artist.mbid):
track_artist = album.artist
else:
track_artist = Artist.get_or_create_from_api(track_artist_mbid)[0]
return cls.objects.update_or_create(
mbid=mbid,
defaults={
"position": int(track["position"]),
"title": track["recording"]["title"],
"album": album,
"artist": track_artist,
)
Eliot Berriot
committed
class TrackFile(models.Model):
uuid = models.UUIDField(unique=True, db_index=True, default=uuid.uuid4)
track = models.ForeignKey(Track, related_name="files", on_delete=models.CASCADE)
audio_file = models.FileField(upload_to="tracks/%Y/%m/%d", max_length=255)
source = models.URLField(null=True, blank=True, max_length=500)
creation_date = models.DateTimeField(default=timezone.now)
modification_date = models.DateTimeField(auto_now=True)
accessed_date = models.DateTimeField(null=True, blank=True)
Eliot Berriot
committed
duration = models.IntegerField(null=True, blank=True)
size = models.IntegerField(null=True, blank=True)
bitrate = models.IntegerField(null=True, blank=True)
acoustid_track_id = models.UUIDField(null=True, blank=True)
mimetype = models.CharField(null=True, blank=True, max_length=200)
Eliot Berriot
committed
library_track = models.OneToOneField(
"federation.LibraryTrack",
related_name="local_track_file",
on_delete=models.CASCADE,
null=True,
blank=True,
)
Eliot Berriot
committed
def download_file(self):
# import the track file, since there is not any
# we create a tmp dir for the download
tmp_dir = tempfile.mkdtemp()
data = downloader.download(self.source, target_directory=tmp_dir)
self.duration = data.get("duration", None)
Eliot Berriot
committed
self.audio_file.save(
os.path.basename(data["audio_file_path"]),
File(open(data["audio_file_path"], "rb")),
Eliot Berriot
committed
)
shutil.rmtree(tmp_dir)
return self.audio_file
def get_federation_url(self):
return federation_utils.full_url("/federation/music/file/{}".format(self.uuid))
Eliot Berriot
committed
@property
def path(self):
return reverse("api:v1:trackfiles-serve", kwargs={"pk": self.pk})
Eliot Berriot
committed
@property
def filename(self):
return "{}.{}".format(self.track.full_name, self.extension)
Eliot Berriot
committed
@property
def extension(self):
if not self.audio_file:
return
return os.path.splitext(self.audio_file.name)[-1].replace(".", "", 1)
def get_file_size(self):
if self.audio_file:
return self.audio_file.size
if self.source.startswith("file://"):
return os.path.getsize(self.source.replace("file://", "", 1))
if self.library_track and self.library_track.audio_file:
return self.library_track.audio_file.size
def get_audio_file(self):
if self.audio_file:
return self.audio_file.open()
if self.source.startswith("file://"):
return open(self.source.replace("file://", "", 1), "rb")
if self.library_track and self.library_track.audio_file:
return self.library_track.audio_file.open()
def set_audio_data(self):
audio_file = self.get_audio_file()
if audio_file:
with audio_file as f:
audio_data = utils.get_audio_file_data(f)
if not audio_data:
return
self.duration = int(audio_data["length"])
self.bitrate = audio_data["bitrate"]
self.size = self.get_file_size()
else:
lt = self.library_track
if lt:
self.duration = lt.get_metadata("length")
self.size = lt.get_metadata("size")
self.bitrate = lt.get_metadata("bitrate")
def save(self, **kwargs):
if not self.mimetype and self.audio_file:
self.mimetype = utils.guess_mimetype(self.audio_file)
return super().save(**kwargs)
Eliot Berriot
committed
def get_metadata(self):
audio_file = self.get_audio_file()
if not audio_file:
return
return metadata.Metadata(audio_file)
IMPORT_STATUS_CHOICES = (
("pending", "Pending"),
("finished", "Finished"),
("errored", "Errored"),
("skipped", "Skipped"),
Eliot Berriot
committed
class ImportBatch(models.Model):
uuid = models.UUIDField(unique=True, db_index=True, default=uuid.uuid4)
IMPORT_BATCH_SOURCES = [
("api", "api"),
("shell", "shell"),
("federation", "federation"),
]
source = models.CharField(
max_length=30, default="api", choices=IMPORT_BATCH_SOURCES
)
Eliot Berriot
committed
creation_date = models.DateTimeField(default=timezone.now)
null=True,
blank=True,
status = models.CharField(
choices=IMPORT_STATUS_CHOICES, default="pending", max_length=30
)
import_request = models.ForeignKey(
"requests.ImportRequest",
related_name="import_batches",
null=True,
blank=True,
on_delete=models.SET_NULL,
Eliot Berriot
committed
class Meta:
Eliot Berriot
committed
def __str__(self):
return str(self.pk)
def update_status(self):
old_status = self.status
self.status = utils.compute_status(self.jobs.all())
if self.status == old_status:
return
self.save(update_fields=["status"])
if self.status != old_status and self.status == "finished":
from . import tasks
tasks.import_batch_notify_followers.delay(import_batch_id=self.pk)
def get_federation_url(self):
return federation_utils.full_url(
Eliot Berriot
committed
class ImportJob(models.Model):
uuid = models.UUIDField(unique=True, db_index=True, default=uuid.uuid4)
replace_if_duplicate = models.BooleanField(default=False)
ImportBatch, related_name="jobs", on_delete=models.CASCADE
)
track_file = models.ForeignKey(
TrackFile, related_name="jobs", null=True, blank=True, on_delete=models.CASCADE
)
mbid = models.UUIDField(editable=False, null=True, blank=True)
status = models.CharField(
choices=IMPORT_STATUS_CHOICES, default="pending", max_length=30
)
audio_file = models.FileField(
upload_to="imports/%Y/%m/%d", max_length=255, null=True, blank=True
)
library_track = models.ForeignKey(
"federation.LibraryTrack",
related_name="import_jobs",
on_delete=models.SET_NULL,
null=True,
)
Eliot Berriot
committed
Eliot Berriot
committed
class Meta:
@receiver(post_save, sender=ImportJob)
def update_batch_status(sender, instance, **kwargs):
instance.batch.update_status()
@receiver(post_save, sender=ImportBatch)
def update_request_status(sender, instance, created, **kwargs):
if not instance.import_request:
return
if not created and "status" not in update_fields:
return
r_status = instance.import_request.status
status = instance.status
# let's mark the request as accepted since we started an import
instance.import_request.status = "accepted"
return instance.import_request.save(update_fields=["status"])
# let's mark the request as imported since the import is over
instance.import_request.status = "imported"
return instance.import_request.save(update_fields=["status"])
@receiver(models.signals.post_save, sender=Album)
def warm_album_covers(sender, instance, **kwargs):
if not instance.cover:
return
album_covers_warmer = VersatileImageFieldWarmer(
instance_or_queryset=instance, rendition_key_set="square", image_attr="cover"
)
num_created, failed_to_create = album_covers_warmer.warm()