Skip to content
Snippets Groups Projects
Verified Commit 08e28aa6 authored by Eliot Berriot's avatar Eliot Berriot
Browse files

Merge branch 'release/0.3.1'

parents c8a2ae42 0b8f61b2
No related branches found
No related tags found
No related merge requests found
Showing
with 367 additions and 68 deletions
Changelog Changelog
========= =========
0.3.2 (Unreleased)
------------------
0.3.1
------------------
- Revamped all import logic, everything is more tested and consistend
- Can now use Acoustid in file imports to automatically grab metadata from musicbrainz
- Brand new file import wizard
0.2.7 (Unreleased) 0.2.7
------------------ ------------------
- Shortcuts: can now use the ``f`` shortcut to toggle the currently playing track - Shortcuts: can now use the ``f`` shortcut to toggle the currently playing track
......
...@@ -6,8 +6,8 @@ ENV PYTHONUNBUFFERED 1 ...@@ -6,8 +6,8 @@ ENV PYTHONUNBUFFERED 1
COPY ./requirements.apt /requirements.apt COPY ./requirements.apt /requirements.apt
RUN apt-get update -qq && grep "^[^#;]" requirements.apt | xargs apt-get install -y RUN apt-get update -qq && grep "^[^#;]" requirements.apt | xargs apt-get install -y
RUN curl -L https://github.com/acoustid/chromaprint/releases/download/v1.4.2/chromaprint-fpcalc-1.4.2-linux-x86_64.tar.gz | tar -xz -C /usr/local/bin --strip 1
RUN fcalc yolofkjdssdhf
COPY ./requirements/base.txt /requirements/base.txt COPY ./requirements/base.txt /requirements/base.txt
RUN pip install -r /requirements/base.txt RUN pip install -r /requirements/base.txt
COPY ./requirements/production.txt /requirements/production.txt COPY ./requirements/production.txt /requirements/production.txt
......
...@@ -15,6 +15,7 @@ router.register(r'trackfiles', views.TrackFileViewSet, 'trackfiles') ...@@ -15,6 +15,7 @@ router.register(r'trackfiles', views.TrackFileViewSet, 'trackfiles')
router.register(r'artists', views.ArtistViewSet, 'artists') router.register(r'artists', views.ArtistViewSet, 'artists')
router.register(r'albums', views.AlbumViewSet, 'albums') router.register(r'albums', views.AlbumViewSet, 'albums')
router.register(r'import-batches', views.ImportBatchViewSet, 'import-batches') router.register(r'import-batches', views.ImportBatchViewSet, 'import-batches')
router.register(r'import-jobs', views.ImportJobViewSet, 'import-jobs')
router.register(r'submit', views.SubmitViewSet, 'submit') router.register(r'submit', views.SubmitViewSet, 'submit')
router.register(r'playlists', playlists_views.PlaylistViewSet, 'playlists') router.register(r'playlists', playlists_views.PlaylistViewSet, 'playlists')
router.register( router.register(
......
...@@ -47,7 +47,6 @@ THIRD_PARTY_APPS = ( ...@@ -47,7 +47,6 @@ THIRD_PARTY_APPS = (
'corsheaders', 'corsheaders',
'rest_framework', 'rest_framework',
'rest_framework.authtoken', 'rest_framework.authtoken',
'djcelery',
'taggit', 'taggit',
'cachalot', 'cachalot',
'rest_auth', 'rest_auth',
...@@ -68,6 +67,7 @@ LOCAL_APPS = ( ...@@ -68,6 +67,7 @@ LOCAL_APPS = (
'funkwhale_api.playlists', 'funkwhale_api.playlists',
'funkwhale_api.providers.audiofile', 'funkwhale_api.providers.audiofile',
'funkwhale_api.providers.youtube', 'funkwhale_api.providers.youtube',
'funkwhale_api.providers.acoustid',
) )
# See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps # See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
...@@ -266,14 +266,14 @@ CACHES["default"]["OPTIONS"] = { ...@@ -266,14 +266,14 @@ CACHES["default"]["OPTIONS"] = {
########## CELERY ########## CELERY
INSTALLED_APPS += ('funkwhale_api.taskapp.celery.CeleryConfig',) INSTALLED_APPS += ('funkwhale_api.taskapp.celery.CeleryConfig',)
BROKER_URL = env( CELERY_BROKER_URL = env(
"CELERY_BROKER_URL", default=env('CACHE_URL', default=CACHE_DEFAULT)) "CELERY_BROKER_URL", default=env('CACHE_URL', default=CACHE_DEFAULT))
########## END CELERY ########## END CELERY
# Location of root django.contrib.admin URL, use {% url 'admin:index' %} # Location of root django.contrib.admin URL, use {% url 'admin:index' %}
ADMIN_URL = r'^admin/' ADMIN_URL = r'^admin/'
# Your common stuff: Below this line define 3rd party library settings # Your common stuff: Below this line define 3rd party library settings
CELERY_DEFAULT_RATE_LIMIT = 1 CELERY_TASK_DEFAULT_RATE_LIMIT = 1
CELERYD_TASK_TIME_LIMIT = 300 CELERY_TASK_TIME_LIMIT = 300
import datetime import datetime
JWT_AUTH = { JWT_AUTH = {
'JWT_ALLOW_REFRESH': True, 'JWT_ALLOW_REFRESH': True,
......
...@@ -54,7 +54,7 @@ TEST_RUNNER = 'django.test.runner.DiscoverRunner' ...@@ -54,7 +54,7 @@ TEST_RUNNER = 'django.test.runner.DiscoverRunner'
########## CELERY ########## CELERY
# In development, all tasks will be executed locally by blocking until the task returns # In development, all tasks will be executed locally by blocking until the task returns
CELERY_ALWAYS_EAGER = False CELERY_TASK_ALWAYS_EAGER = False
########## END CELERY ########## END CELERY
# Your local stuff: Below this line define 3rd party library settings # Your local stuff: Below this line define 3rd party library settings
......
...@@ -23,7 +23,7 @@ CACHES = { ...@@ -23,7 +23,7 @@ CACHES = {
} }
} }
BROKER_URL = 'memory://' CELERY_BROKER_URL = 'memory://'
# TESTING # TESTING
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
...@@ -31,7 +31,7 @@ TEST_RUNNER = 'django.test.runner.DiscoverRunner' ...@@ -31,7 +31,7 @@ TEST_RUNNER = 'django.test.runner.DiscoverRunner'
########## CELERY ########## CELERY
# In development, all tasks will be executed locally by blocking until the task returns # In development, all tasks will be executed locally by blocking until the task returns
CELERY_ALWAYS_EAGER = True CELERY_TASK_ALWAYS_EAGER = True
########## END CELERY ########## END CELERY
# Your local stuff: Below this line define 3rd party library settings # Your local stuff: Below this line define 3rd party library settings
......
...@@ -7,6 +7,7 @@ ENV PYTHONDONTWRITEBYTECODE 1 ...@@ -7,6 +7,7 @@ ENV PYTHONDONTWRITEBYTECODE 1
COPY ./requirements.apt /requirements.apt COPY ./requirements.apt /requirements.apt
COPY ./install_os_dependencies.sh /install_os_dependencies.sh COPY ./install_os_dependencies.sh /install_os_dependencies.sh
RUN bash install_os_dependencies.sh install RUN bash install_os_dependencies.sh install
RUN curl -L https://github.com/acoustid/chromaprint/releases/download/v1.4.2/chromaprint-fpcalc-1.4.2-linux-x86_64.tar.gz | tar -xz -C /usr/local/bin --strip 1
RUN mkdir /requirements RUN mkdir /requirements
COPY ./requirements/base.txt /requirements/base.txt COPY ./requirements/base.txt /requirements/base.txt
......
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
__version__ = '0.3' __version__ = '0.3.1'
__version_info__ = tuple([int(num) if num.isdigit() else num for num in __version__.replace('-', '.', 1).split('.')]) __version_info__ = tuple([int(num) if num.isdigit() else num for num in __version__.replace('-', '.', 1).split('.')])
from django.conf import settings from django.conf import settings
from rest_framework.permissions import BasePermission from rest_framework.permissions import BasePermission, DjangoModelPermissions
class ConditionalAuthentication(BasePermission): class ConditionalAuthentication(BasePermission):
...@@ -9,3 +9,14 @@ class ConditionalAuthentication(BasePermission): ...@@ -9,3 +9,14 @@ class ConditionalAuthentication(BasePermission):
if settings.API_AUTHENTICATION_REQUIRED: if settings.API_AUTHENTICATION_REQUIRED:
return request.user and request.user.is_authenticated return request.user and request.user.is_authenticated
return True return True
class HasModelPermission(DjangoModelPermissions):
"""
Same as DjangoModelPermissions, but we pin the model:
class MyModelPermission(HasModelPermission):
model = User
"""
def get_required_permissions(self, method, model_cls):
return super().get_required_permissions(method, self.model)
...@@ -72,6 +72,14 @@ class ImportJobFactory(factory.django.DjangoModelFactory): ...@@ -72,6 +72,14 @@ class ImportJobFactory(factory.django.DjangoModelFactory):
model = 'music.ImportJob' model = 'music.ImportJob'
@registry.register(name='music.FileImportJob')
class FileImportJobFactory(ImportJobFactory):
source = 'file://'
mbid = None
audio_file = factory.django.FileField(
from_path=os.path.join(SAMPLES_PATH, 'test.ogg'))
@registry.register @registry.register
class WorkFactory(factory.django.DjangoModelFactory): class WorkFactory(factory.django.DjangoModelFactory):
mbid = factory.Faker('uuid4') mbid = factory.Faker('uuid4')
......
# Generated by Django 2.0 on 2017-12-26 16:39
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('music', '0015_bind_track_file_to_import_job'),
]
operations = [
migrations.AddField(
model_name='trackfile',
name='acoustid_track_id',
field=models.UUIDField(blank=True, null=True),
),
]
# Generated by Django 2.0 on 2017-12-27 17:28
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('music', '0016_trackfile_acoustid_track_id'),
]
operations = [
migrations.AddField(
model_name='importbatch',
name='source',
field=models.CharField(choices=[('api', 'api'), ('shell', 'shell')], default='api', max_length=30),
),
migrations.AddField(
model_name='importjob',
name='audio_file',
field=models.FileField(blank=True, max_length=255, null=True, upload_to='imports/%Y/%m/%d'),
),
migrations.AlterField(
model_name='importjob',
name='mbid',
field=models.UUIDField(blank=True, editable=False, null=True),
),
]
...@@ -15,11 +15,9 @@ from django.utils import timezone ...@@ -15,11 +15,9 @@ from django.utils import timezone
from taggit.managers import TaggableManager from taggit.managers import TaggableManager
from versatileimagefield.fields import VersatileImageField from versatileimagefield.fields import VersatileImageField
from funkwhale_api.taskapp import celery
from funkwhale_api import downloader from funkwhale_api import downloader
from funkwhale_api import musicbrainz from funkwhale_api import musicbrainz
from . import importers from . import importers
from . import lyrics as lyrics_utils
class APIModelMixin(models.Model): class APIModelMixin(models.Model):
...@@ -255,14 +253,6 @@ class Lyrics(models.Model): ...@@ -255,14 +253,6 @@ class Lyrics(models.Model):
url = models.URLField(unique=True) url = models.URLField(unique=True)
content = models.TextField(null=True, blank=True) content = models.TextField(null=True, blank=True)
@celery.app.task(name='Lyrics.fetch_content', filter=celery.task_method)
def fetch_content(self):
html = lyrics_utils._get_html(self.url)
content = lyrics_utils.extract_content(html)
cleaned_content = lyrics_utils.clean_content(content)
self.content = cleaned_content
self.save()
@property @property
def content_rendered(self): def content_rendered(self):
return markdown.markdown( return markdown.markdown(
...@@ -362,6 +352,7 @@ class TrackFile(models.Model): ...@@ -362,6 +352,7 @@ class TrackFile(models.Model):
audio_file = models.FileField(upload_to='tracks/%Y/%m/%d', max_length=255) audio_file = models.FileField(upload_to='tracks/%Y/%m/%d', max_length=255)
source = models.URLField(null=True, blank=True) source = models.URLField(null=True, blank=True)
duration = models.IntegerField(null=True, blank=True) duration = models.IntegerField(null=True, blank=True)
acoustid_track_id = models.UUIDField(null=True, blank=True)
def download_file(self): def download_file(self):
# import the track file, since there is not any # import the track file, since there is not any
...@@ -393,9 +384,17 @@ class TrackFile(models.Model): ...@@ -393,9 +384,17 @@ class TrackFile(models.Model):
class ImportBatch(models.Model): class ImportBatch(models.Model):
IMPORT_BATCH_SOURCES = [
('api', 'api'),
('shell', 'shell')
]
source = models.CharField(
max_length=30, default='api', choices=IMPORT_BATCH_SOURCES)
creation_date = models.DateTimeField(default=timezone.now) creation_date = models.DateTimeField(default=timezone.now)
submitted_by = models.ForeignKey( submitted_by = models.ForeignKey(
'users.User', related_name='imports', on_delete=models.CASCADE) 'users.User',
related_name='imports',
on_delete=models.CASCADE)
class Meta: class Meta:
ordering = ['-creation_date'] ordering = ['-creation_date']
...@@ -406,8 +405,11 @@ class ImportBatch(models.Model): ...@@ -406,8 +405,11 @@ class ImportBatch(models.Model):
@property @property
def status(self): def status(self):
pending = any([job.status == 'pending' for job in self.jobs.all()]) pending = any([job.status == 'pending' for job in self.jobs.all()])
errored = any([job.status == 'errored' for job in self.jobs.all()])
if pending: if pending:
return 'pending' return 'pending'
if errored:
return 'errored'
return 'finished' return 'finished'
class ImportJob(models.Model): class ImportJob(models.Model):
...@@ -419,36 +421,17 @@ class ImportJob(models.Model): ...@@ -419,36 +421,17 @@ class ImportJob(models.Model):
null=True, null=True,
blank=True, blank=True,
on_delete=models.CASCADE) on_delete=models.CASCADE)
source = models.URLField() source = models.CharField(max_length=500)
mbid = models.UUIDField(editable=False) mbid = models.UUIDField(editable=False, null=True, blank=True)
STATUS_CHOICES = ( STATUS_CHOICES = (
('pending', 'Pending'), ('pending', 'Pending'),
('finished', 'finished'), ('finished', 'Finished'),
('errored', 'Errored'),
('skipped', 'Skipped'),
) )
status = models.CharField(choices=STATUS_CHOICES, default='pending', max_length=30) status = models.CharField(choices=STATUS_CHOICES, default='pending', max_length=30)
audio_file = models.FileField(
upload_to='imports/%Y/%m/%d', max_length=255, null=True, blank=True)
class Meta: class Meta:
ordering = ('id', ) ordering = ('id', )
@celery.app.task(name='ImportJob.run', filter=celery.task_method)
def run(self, replace=False):
try:
track, created = Track.get_or_create_from_api(mbid=self.mbid)
track_file = None
if replace:
track_file = track.files.first()
elif track.files.count() > 0:
return
track_file = track_file or TrackFile(
track=track, source=self.source)
track_file.download_file()
track_file.save()
self.status = 'finished'
self.track_file = track_file
self.save()
return track.pk
except Exception as exc:
if not settings.DEBUG:
raise ImportJob.run.retry(args=[self], exc=exc, countdown=30, max_retries=3)
raise
...@@ -113,7 +113,8 @@ class ImportJobSerializer(serializers.ModelSerializer): ...@@ -113,7 +113,8 @@ class ImportJobSerializer(serializers.ModelSerializer):
track_file = TrackFileSerializer(read_only=True) track_file = TrackFileSerializer(read_only=True)
class Meta: class Meta:
model = models.ImportJob model = models.ImportJob
fields = ('id', 'mbid', 'source', 'status', 'track_file') fields = ('id', 'mbid', 'batch', 'source', 'status', 'track_file', 'audio_file')
read_only_fields = ('status', 'track_file')
class ImportBatchSerializer(serializers.ModelSerializer): class ImportBatchSerializer(serializers.ModelSerializer):
...@@ -121,3 +122,4 @@ class ImportBatchSerializer(serializers.ModelSerializer): ...@@ -121,3 +122,4 @@ class ImportBatchSerializer(serializers.ModelSerializer):
class Meta: class Meta:
model = models.ImportBatch model = models.ImportBatch
fields = ('id', 'jobs', 'status', 'creation_date') fields = ('id', 'jobs', 'status', 'creation_date')
read_only_fields = ('creation_date',)
from django.core.files.base import ContentFile
from funkwhale_api.taskapp import celery
from funkwhale_api.providers.acoustid import get_acoustid_client
from funkwhale_api.providers.audiofile.tasks import import_track_data_from_path
from django.conf import settings
from . import models
from . import lyrics as lyrics_utils
@celery.app.task(name='acoustid.set_on_track_file')
@celery.require_instance(models.TrackFile, 'track_file')
def set_acoustid_on_track_file(track_file):
client = get_acoustid_client()
result = client.get_best_match(track_file.audio_file.path)
def update(id):
track_file.acoustid_track_id = id
track_file.save(update_fields=['acoustid_track_id'])
return id
if result:
return update(result['id'])
def _do_import(import_job, replace):
from_file = bool(import_job.audio_file)
mbid = import_job.mbid
acoustid_track_id = None
duration = None
track = None
if not mbid and from_file:
# we try to deduce mbid from acoustid
client = get_acoustid_client()
match = client.get_best_match(import_job.audio_file.path)
if not match:
raise ValueError('Cannot get match')
duration = match['recordings'][0]['duration']
mbid = match['recordings'][0]['id']
acoustid_track_id = match['id']
if mbid:
track, _ = models.Track.get_or_create_from_api(mbid=mbid)
else:
track = import_track_data_from_path(import_job.audio_file.path)
track_file = None
if replace:
track_file = track.files.first()
elif track.files.count() > 0:
if import_job.audio_file:
import_job.audio_file.delete()
import_job.status = 'skipped'
import_job.save()
return
track_file = track_file or models.TrackFile(
track=track, source=import_job.source)
track_file.acoustid_track_id = acoustid_track_id
if from_file:
track_file.audio_file = ContentFile(import_job.audio_file.read())
track_file.audio_file.name = import_job.audio_file.name
track_file.duration = duration
else:
track_file.download_file()
track_file.save()
import_job.status = 'finished'
import_job.track_file = track_file
if import_job.audio_file:
# it's imported on the track, we don't need it anymore
import_job.audio_file.delete()
import_job.save()
return track.pk
@celery.app.task(name='ImportJob.run', bind=True)
@celery.require_instance(models.ImportJob, 'import_job')
def import_job_run(self, import_job, replace=False):
def mark_errored():
import_job.status = 'errored'
import_job.save()
try:
return _do_import(import_job, replace)
except Exception as exc:
if not settings.DEBUG:
try:
self.retry(exc=exc, countdown=30, max_retries=3)
except:
mark_errored()
raise
mark_errored()
raise
@celery.app.task(name='Lyrics.fetch_content')
@celery.require_instance(models.Lyrics, 'lyrics')
def fetch_content(lyrics):
html = lyrics_utils._get_html(lyrics.url)
content = lyrics_utils.extract_content(html)
cleaned_content = lyrics_utils.clean_content(content)
lyrics.content = cleaned_content
lyrics.save(update_fields=['content'])
...@@ -6,7 +6,7 @@ from django.urls import reverse ...@@ -6,7 +6,7 @@ from django.urls import reverse
from django.db import models, transaction from django.db import models, transaction
from django.db.models.functions import Length from django.db.models.functions import Length
from django.conf import settings from django.conf import settings
from rest_framework import viewsets, views from rest_framework import viewsets, views, mixins
from rest_framework.decorators import detail_route, list_route from rest_framework.decorators import detail_route, list_route
from rest_framework.response import Response from rest_framework.response import Response
from rest_framework import permissions from rest_framework import permissions
...@@ -15,13 +15,15 @@ from django.contrib.auth.decorators import login_required ...@@ -15,13 +15,15 @@ from django.contrib.auth.decorators import login_required
from django.utils.decorators import method_decorator from django.utils.decorators import method_decorator
from funkwhale_api.musicbrainz import api from funkwhale_api.musicbrainz import api
from funkwhale_api.common.permissions import ConditionalAuthentication from funkwhale_api.common.permissions import (
ConditionalAuthentication, HasModelPermission)
from taggit.models import Tag from taggit.models import Tag
from . import models from . import models
from . import serializers from . import serializers
from . import importers from . import importers
from . import filters from . import filters
from . import tasks
from . import utils from . import utils
...@@ -70,16 +72,45 @@ class AlbumViewSet(SearchMixin, viewsets.ReadOnlyModelViewSet): ...@@ -70,16 +72,45 @@ class AlbumViewSet(SearchMixin, viewsets.ReadOnlyModelViewSet):
ordering_fields = ('creation_date',) ordering_fields = ('creation_date',)
class ImportBatchViewSet(viewsets.ReadOnlyModelViewSet): class ImportBatchViewSet(
mixins.CreateModelMixin,
mixins.ListModelMixin,
mixins.RetrieveModelMixin,
viewsets.GenericViewSet):
queryset = ( queryset = (
models.ImportBatch.objects.all() models.ImportBatch.objects.all()
.prefetch_related('jobs__track_file') .prefetch_related('jobs__track_file')
.order_by('-creation_date')) .order_by('-creation_date'))
serializer_class = serializers.ImportBatchSerializer serializer_class = serializers.ImportBatchSerializer
permission_classes = (permissions.DjangoModelPermissions, )
def get_queryset(self): def get_queryset(self):
return super().get_queryset().filter(submitted_by=self.request.user) return super().get_queryset().filter(submitted_by=self.request.user)
def perform_create(self, serializer):
serializer.save(submitted_by=self.request.user)
class ImportJobPermission(HasModelPermission):
# not a typo, perms on import job is proxied to import batch
model = models.ImportBatch
class ImportJobViewSet(
mixins.CreateModelMixin,
viewsets.GenericViewSet):
queryset = (models.ImportJob.objects.all())
serializer_class = serializers.ImportJobSerializer
permission_classes = (ImportJobPermission, )
def get_queryset(self):
return super().get_queryset().filter(batch__submitted_by=self.request.user)
def perform_create(self, serializer):
source = 'file://' + serializer.validated_data['audio_file'].name
serializer.save(source=source)
tasks.import_job_run.delay(import_job_id=serializer.instance.pk)
class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet): class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet):
""" """
...@@ -129,7 +160,8 @@ class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet): ...@@ -129,7 +160,8 @@ class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet):
lyrics = work.fetch_lyrics() lyrics = work.fetch_lyrics()
try: try:
if not lyrics.content: if not lyrics.content:
lyrics.fetch_content() tasks.fetch_content(lyrics_id=lyrics.pk)
lyrics.refresh_from_db()
except AttributeError: except AttributeError:
return Response({'error': 'unavailable lyrics'}, status=404) return Response({'error': 'unavailable lyrics'}, status=404)
serializer = serializers.LyricsSerializer(lyrics) serializer = serializers.LyricsSerializer(lyrics)
...@@ -244,7 +276,7 @@ class SubmitViewSet(viewsets.ViewSet): ...@@ -244,7 +276,7 @@ class SubmitViewSet(viewsets.ViewSet):
pass pass
batch = models.ImportBatch.objects.create(submitted_by=request.user) batch = models.ImportBatch.objects.create(submitted_by=request.user)
job = models.ImportJob.objects.create(mbid=request.POST['mbid'], batch=batch, source=request.POST['import_url']) job = models.ImportJob.objects.create(mbid=request.POST['mbid'], batch=batch, source=request.POST['import_url'])
job.run.delay() tasks.import_job_run.delay(import_job_id=job.pk)
serializer = serializers.ImportBatchSerializer(batch) serializer = serializers.ImportBatchSerializer(batch)
return Response(serializer.data) return Response(serializer.data)
...@@ -272,7 +304,7 @@ class SubmitViewSet(viewsets.ViewSet): ...@@ -272,7 +304,7 @@ class SubmitViewSet(viewsets.ViewSet):
models.TrackFile.objects.get(track__mbid=row['mbid']) models.TrackFile.objects.get(track__mbid=row['mbid'])
except models.TrackFile.DoesNotExist: except models.TrackFile.DoesNotExist:
job = models.ImportJob.objects.create(mbid=row['mbid'], batch=batch, source=row['source']) job = models.ImportJob.objects.create(mbid=row['mbid'], batch=batch, source=row['source'])
job.run.delay() tasks.import_job_run.delay(import_job_id=job.pk)
serializer = serializers.ImportBatchSerializer(batch) serializer = serializers.ImportBatchSerializer(batch)
return serializer.data, batch return serializer.data, batch
......
import acoustid
from dynamic_preferences.registries import global_preferences_registry
class Client(object):
def __init__(self, api_key):
self.api_key = api_key
def match(self, file_path):
return acoustid.match(self.api_key, file_path, parse=False)
def get_best_match(self, file_path):
results = self.match(file_path=file_path)
MIN_SCORE_FOR_MATCH = 0.8
try:
rows = results['results']
except KeyError:
return
for row in rows:
if row['score'] >= MIN_SCORE_FOR_MATCH:
return row
def get_acoustid_client():
manager = global_preferences_registry.manager()
return Client(api_key=manager['providers_acoustid__api_key'])
from dynamic_preferences.types import StringPreference, Section
from dynamic_preferences.registries import global_preferences_registry
acoustid = Section('providers_acoustid')
@global_preferences_registry.register
class APIKey(StringPreference):
section = acoustid
name = 'api_key'
default = ''
verbose_name = 'Acoustid API key'
help_text = 'The API key used to query AcoustID. Get one at https://acoustid.org/new-application.'
import glob import glob
import os
from django.core.files import File
from django.core.management.base import BaseCommand, CommandError from django.core.management.base import BaseCommand, CommandError
from funkwhale_api.providers.audiofile import tasks from funkwhale_api.music import tasks
from funkwhale_api.users.models import User
class Command(BaseCommand): class Command(BaseCommand):
...@@ -15,6 +19,11 @@ class Command(BaseCommand): ...@@ -15,6 +19,11 @@ class Command(BaseCommand):
default=False, default=False,
help='Will match the pattern recursively (including subdirectories)', help='Will match the pattern recursively (including subdirectories)',
) )
parser.add_argument(
'--username',
dest='username',
help='The username of the user you want to be bound to the import',
)
parser.add_argument( parser.add_argument(
'--async', '--async',
action='store_true', action='store_true',
...@@ -46,6 +55,20 @@ class Command(BaseCommand): ...@@ -46,6 +55,20 @@ class Command(BaseCommand):
if not matching: if not matching:
raise CommandError('No file matching pattern, aborting') raise CommandError('No file matching pattern, aborting')
user = None
if options['username']:
try:
user = User.objects.get(username=options['username'])
except User.DoesNotExist:
raise CommandError('Invalid username')
else:
# we bind the import to the first registered superuser
try:
user = User.objects.filter(is_superuser=True).order_by('pk').first()
assert user is not None
except AssertionError:
raise CommandError(
'No superuser available, please provide a --username')
if options['interactive']: if options['interactive']:
message = ( message = (
'Are you sure you want to do this?\n\n' 'Are you sure you want to do this?\n\n'
...@@ -54,18 +77,35 @@ class Command(BaseCommand): ...@@ -54,18 +77,35 @@ class Command(BaseCommand):
if input(''.join(message)) != 'yes': if input(''.join(message)) != 'yes':
raise CommandError("Import cancelled.") raise CommandError("Import cancelled.")
batch = self.do_import(matching, user=user, options=options)
message = 'Successfully imported {} tracks'
if options['async']:
message = 'Successfully launched import for {} tracks'
self.stdout.write(message.format(len(matching)))
self.stdout.write(
"For details, please refer to import batch #".format(batch.pk))
def do_import(self, matching, user, options):
message = 'Importing {}...' message = 'Importing {}...'
if options['async']: if options['async']:
message = 'Launching import for {}...' message = 'Launching import for {}...'
# we create an import batch binded to the user
batch = user.imports.create(source='shell')
async = options['async']
handler = tasks.import_job_run.delay if async else tasks.import_job_run
for path in matching: for path in matching:
self.stdout.write(message.format(path)) job = batch.jobs.create(
source='file://' + path,
)
name = os.path.basename(path)
with open(path, 'rb') as f:
job.audio_file.save(name, File(f))
job.save()
try: try:
tasks.from_path(path) handler(import_job_id=job.pk)
except Exception as e: except Exception as e:
self.stdout.write('Error: {}'.format(e)) self.stdout.write('Error: {}'.format(e))
return batch
message = 'Successfully imported {} tracks'
if options['async']:
message = 'Successfully launched import for {} tracks'
self.stdout.write(message.format(len(matching)))
import acoustid
import os import os
import datetime import datetime
from django.core.files import File from django.core.files import File
from funkwhale_api.taskapp import celery from funkwhale_api.taskapp import celery
from funkwhale_api.providers.acoustid import get_acoustid_client
from funkwhale_api.music import models, metadata from funkwhale_api.music import models, metadata
@celery.app.task(name='audiofile.from_path') def import_track_data_from_path(path):
def from_path(path):
data = metadata.Metadata(path) data = metadata.Metadata(path)
artist = models.Artist.objects.get_or_create( artist = models.Artist.objects.get_or_create(
name__iexact=data.get('artist'), name__iexact=data.get('artist'),
defaults={ defaults={
'name': data.get('artist'), 'name': data.get('artist'),
'mbid': data.get('musicbrainz_artistid', None), 'mbid': data.get('musicbrainz_artistid', None),
}, },
)[0] )[0]
...@@ -39,11 +39,33 @@ def from_path(path): ...@@ -39,11 +39,33 @@ def from_path(path):
'mbid': data.get('musicbrainz_recordingid', None), 'mbid': data.get('musicbrainz_recordingid', None),
}, },
)[0] )[0]
return track
def import_metadata_with_musicbrainz(path):
pass
@celery.app.task(name='audiofile.from_path')
def from_path(path):
acoustid_track_id = None
try:
client = get_acoustid_client()
result = client.get_best_match(path)
acoustid_track_id = result['id']
except acoustid.WebServiceError:
track = import_track_data_from_path(path)
except (TypeError, KeyError):
track = import_metadata_without_musicbrainz(path)
else:
track, created = models.Track.get_or_create_from_api(
mbid=result['recordings'][0]['id']
)
if track.files.count() > 0: if track.files.count() > 0:
raise ValueError('File already exists for track {}'.format(track.pk)) raise ValueError('File already exists for track {}'.format(track.pk))
track_file = models.TrackFile(track=track) track_file = models.TrackFile(
track=track, acoustid_track_id=acoustid_track_id)
track_file.audio_file.save( track_file.audio_file.save(
os.path.basename(path), os.path.basename(path),
File(open(path, 'rb')) File(open(path, 'rb'))
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment