Merge branch '195-bitrate-size' into 'develop'

Resolve "Store file bitrate and size"

Closes #195

See merge request funkwhale/funkwhale!196

.env.dev

@@ -10,3 +10,4 @@ PYTHONDONTWRITEBYTECODE=true
 WEBPACK_DEVSERVER_PORT=8080
 MUSIC_DIRECTORY_PATH=/music
 BROWSABLE_API_ENABLED=True
+CACHEOPS_ENABLED=False

api/funkwhale_api/federation/factories.py

@@ -233,6 +233,9 @@ class AudioMetadataFactory(factory.Factory):
     release = factory.LazyAttribute(
         lambda o: 'https://musicbrainz.org/release/{}'.format(uuid.uuid4())
     )
+    bitrate = 42
+    length = 43
+    size = 44
 
     class Meta:
         model = dict

api/funkwhale_api/federation/models.py

@@ -216,3 +216,6 @@ class LibraryTrack(models.Model):
             for chunk in r.iter_content(chunk_size=512):
                 tmp_file.write(chunk)
             self.audio_file.save(filename, tmp_file)
+
+    def get_metadata(self, key):
+        return self.metadata.get(key)

api/funkwhale_api/federation/serializers.py

@@ -688,6 +688,12 @@ class AudioMetadataSerializer(serializers.Serializer):
     artist = ArtistMetadataSerializer()
     release = ReleaseMetadataSerializer()
     recording = RecordingMetadataSerializer()
+    bitrate = serializers.IntegerField(
+        required=False, allow_null=True, min_value=0)
+    size = serializers.IntegerField(
+        required=False, allow_null=True, min_value=0)
+    length = serializers.IntegerField(
+        required=False, allow_null=True, min_value=0)
 
 
 class AudioSerializer(serializers.Serializer):
@@ -760,6 +766,9 @@ class AudioSerializer(serializers.Serializer):
                     'musicbrainz_id': str(track.mbid) if track.mbid else None,
                     'title': track.title,
                 },
+                'bitrate': instance.bitrate,
+                'size': instance.size,
+                'length': instance.duration,
             },
             'url': {
                 'href': utils.full_url(instance.path),

api/funkwhale_api/music/admin.py

@@ -74,6 +74,8 @@ class TrackFileAdmin(admin.ModelAdmin):
         'source',
         'duration',
         'mimetype',
+        'size',
+        'bitrate'
     ]
     list_select_related = [
         'track'

api/funkwhale_api/music/factories.py

@@ -54,6 +54,10 @@ class TrackFileFactory(factory.django.DjangoModelFactory):
     audio_file = factory.django.FileField(
         from_path=os.path.join(SAMPLES_PATH, 'test.ogg'))
 
+    bitrate = None
+    size = None
+    duration = None
+
     class Meta:
         model = 'music.TrackFile'
 

api/funkwhale_api/music/management/commands/fix_track_files.py

@@ -2,6 +2,7 @@ import cacheops
 import os
 
 from django.db import transaction
+from django.db.models import Q
 from django.conf import settings
 from django.core.management.base import BaseCommand, CommandError
 
@@ -24,6 +25,8 @@ class Command(BaseCommand):
         if options['dry_run']:
             self.stdout.write('Dry-run on, will not commit anything')
         self.fix_mimetypes(**options)
+        self.fix_file_data(**options)
+        self.fix_file_size(**options)
         cacheops.invalidate_model(models.TrackFile)
 
     @transaction.atomic
@@ -43,3 +46,60 @@ class Command(BaseCommand):
             if not dry_run:
                 self.stdout.write('[mimetypes] commiting...')
                 qs.update(mimetype=mimetype)
+
+    def fix_file_data(self, dry_run, **kwargs):
+        self.stdout.write('Fixing missing bitrate or length...')
+        matching = models.TrackFile.objects.filter(
+            Q(bitrate__isnull=True) | Q(duration__isnull=True))
+        total = matching.count()
+        self.stdout.write(
+            '[bitrate/length] {} entries found with missing values'.format(
+                total))
+        if dry_run:
+            return
+        for i, tf in enumerate(matching.only('audio_file')):
+            self.stdout.write(
+                '[bitrate/length] {}/{} fixing file #{}'.format(
+                    i+1, total, tf.pk
+                ))
+
+            try:
+                audio_file = tf.get_audio_file()
+                if audio_file:
+                    with audio_file as f:
+                        data = utils.get_audio_file_data(audio_file)
+                    tf.bitrate = data['bitrate']
+                    tf.duration = data['length']
+                    tf.save(update_fields=['duration', 'bitrate'])
+                else:
+                    self.stderr.write('[bitrate/length] no file found')
+            except Exception as e:
+                self.stderr.write(
+                    '[bitrate/length] error with file #{}: {}'.format(
+                        tf.pk, str(e)
+                    )
+                )
+
+    def fix_file_size(self, dry_run, **kwargs):
+        self.stdout.write('Fixing missing size...')
+        matching = models.TrackFile.objects.filter(size__isnull=True)
+        total = matching.count()
+        self.stdout.write(
+            '[size] {} entries found with missing values'.format(total))
+        if dry_run:
+            return
+        for i, tf in enumerate(matching.only('size')):
+            self.stdout.write(
+                '[size] {}/{} fixing file #{}'.format(
+                    i+1, total, tf.pk
+                ))
+
+            try:
+                tf.size = tf.get_file_size()
+                tf.save(update_fields=['size'])
+            except Exception as e:
+                self.stderr.write(
+                    '[size] error with file #{}: {}'.format(
+                        tf.pk, str(e)
+                    )
+                )

api/funkwhale_api/music/migrations/0027_auto_20180515_1808.py

+# Generated by Django 2.0.3 on 2018-05-15 18:08
+
+from django.db import migrations, models
+import taggit.managers
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0026_trackfile_accessed_date'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='trackfile',
+            name='bitrate',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='trackfile',
+            name='size',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='track',
+            name='tags',
+            field=taggit.managers.TaggableManager(blank=True, help_text='A comma-separated list of tags.', through='taggit.TaggedItem', to='taggit.Tag', verbose_name='Tags'),
+        ),
+    ]

api/funkwhale_api/music/models.py

@@ -429,6 +429,8 @@ class TrackFile(models.Model):
     modification_date = models.DateTimeField(auto_now=True)
     accessed_date = models.DateTimeField(null=True, blank=True)
     duration = models.IntegerField(null=True, blank=True)
+    size = models.IntegerField(null=True, blank=True)
+    bitrate = models.IntegerField(null=True, blank=True)
     acoustid_track_id = models.UUIDField(null=True, blank=True)
     mimetype = models.CharField(null=True, blank=True, max_length=200)
 
@@ -477,6 +479,41 @@ class TrackFile(models.Model):
             return
         return os.path.splitext(self.audio_file.name)[-1].replace('.', '', 1)
 
+    def get_file_size(self):
+        if self.audio_file:
+            return self.audio_file.size
+
+        if self.source.startswith('file://'):
+            return os.path.getsize(self.source.replace('file://', '', 1))
+
+        if self.library_track and self.library_track.audio_file:
+            return self.library_track.audio_file.size
+
+    def get_audio_file(self):
+        if self.audio_file:
+            return self.audio_file.open()
+        if self.source.startswith('file://'):
+            return open(self.source.replace('file://', '', 1), 'rb')
+        if self.library_track and self.library_track.audio_file:
+            return self.library_track.audio_file.open()
+
+    def set_audio_data(self):
+        audio_file = self.get_audio_file()
+        if audio_file:
+            with audio_file as f:
+                audio_data = utils.get_audio_file_data(f)
+            if not audio_data:
+                return
+            self.duration = int(audio_data['length'])
+            self.bitrate = audio_data['bitrate']
+            self.size = self.get_file_size()
+        else:
+            lt = self.library_track
+            if lt:
+                self.duration = lt.get_metadata('length')
+                self.size = lt.get_metadata('size')
+                self.bitrate = lt.get_metadata('bitrate')
+
     def save(self, **kwargs):
         if not self.mimetype and self.audio_file:
             self.mimetype = utils.guess_mimetype(self.audio_file)

api/funkwhale_api/music/serializers.py

@@ -27,6 +27,7 @@ class SimpleArtistSerializer(serializers.ModelSerializer):
 
 class ArtistSerializer(serializers.ModelSerializer):
     tags = TagSerializer(many=True, read_only=True)
+
     class Meta:
         model = models.Artist
         fields = ('id', 'mbid', 'name', 'tags', 'creation_date')
@@ -40,11 +41,21 @@ class TrackFileSerializer(serializers.ModelSerializer):
         fields = (
             'id',
             'path',
-            'duration',
             'source',
             'filename',
             'mimetype',
-            'track')
+            'track',
+            'duration',
+            'mimetype',
+            'bitrate',
+            'size',
+        )
+        read_only_fields = [
+            'duration',
+            'mimetype',
+            'bitrate',
+            'size',
+        ]
 
     def get_path(self, o):
         url = o.path

api/funkwhale_api/music/tasks.py

@@ -134,6 +134,7 @@ def _do_import(import_job, replace=False, use_acoustid=True):
         # in place import, we set mimetype from extension
         path, ext = os.path.splitext(import_job.source)
         track_file.mimetype = music_utils.get_type_from_ext(ext)
+    track_file.set_audio_data()
     track_file.save()
     import_job.status = 'finished'
     import_job.track_file = track_file

api/funkwhale_api/music/utils.py

 import magic
 import mimetypes
+import mutagen
 import re
 
 from django.db.models import Q
@@ -66,7 +67,7 @@ def compute_status(jobs):
 AUDIO_EXTENSIONS_AND_MIMETYPE = [
     ('ogg', 'audio/ogg'),
     ('mp3', 'audio/mpeg'),
-    ('flac', 'audio/flac'),
+    ('flac', 'audio/x-flac'),
 ]
 
 EXTENSION_TO_MIMETYPE = {ext: mt for ext, mt in AUDIO_EXTENSIONS_AND_MIMETYPE}
@@ -82,3 +83,14 @@ def get_type_from_ext(extension):
         # we remove leading dot
         extension = extension[1:]
     return EXTENSION_TO_MIMETYPE.get(extension)
+
+
+def get_audio_file_data(f):
+    data = mutagen.File(f)
+    if not data:
+        return
+    d = {}
+    d['bitrate'] = data.info.bitrate
+    d['length'] = data.info.length
+
+    return d

api/funkwhale_api/music/views.py

@@ -268,6 +268,10 @@ def handle_serve(track_file):
                 qs = LibraryTrack.objects.select_for_update()
                 library_track = qs.get(pk=library_track.pk)
                 library_track.download_audio()
+            track_file.library_track = library_track
+            track_file.set_audio_data()
+            track_file.save(update_fields=['bitrate', 'duration', 'size'])
+
         audio_file = library_track.audio_file
         file_path = get_file_path(audio_file)
         mt = library_track.audio_mimetype
@@ -296,7 +300,11 @@ def handle_serve(track_file):
 
 
 class TrackFileViewSet(viewsets.ReadOnlyModelViewSet):
-    queryset = (models.TrackFile.objects.all().order_by('-id'))
+    queryset = (
+        models.TrackFile.objects.all()
+            .select_related('track__artist', 'track__album')
+            .order_by('-id')
+    )
     serializer_class = serializers.TrackFileSerializer
     authentication_classes = rest_settings.api_settings.DEFAULT_AUTHENTICATION_CLASSES + [
         SignatureAuthentication

api/funkwhale_api/subsonic/serializers.py

@@ -81,6 +81,10 @@ def get_track_data(album, track, tf):
         'artistId': album.artist.pk,
         'type': 'music',
     }
+    if tf.bitrate:
+        data['bitrate'] = int(tf.bitrate/1000)
+    if tf.size:
+        data['size'] = tf.size
     if album.release_date:
         data['year'] = album.release_date.year
     return data
@@ -211,5 +215,9 @@ def get_music_directory_data(artist):
             'parent': artist.id,
             'type': 'music',
         }
+        if tf.bitrate:
+            td['bitrate'] = int(tf.bitrate/1000)
+        if tf.size:
+            td['size'] = tf.size
         data['child'].append(td)
     return data

api/tests/federation/test_serializers.py

@@ -533,7 +533,12 @@ def test_activity_pub_audio_serializer_to_library_track_no_duplicate(
 
 
 def test_activity_pub_audio_serializer_to_ap(factories):
-    tf = factories['music.TrackFile'](mimetype='audio/mp3')
+    tf = factories['music.TrackFile'](
+        mimetype='audio/mp3',
+        bitrate=42,
+        duration=43,
+        size=44,
+    )
     library = actors.SYSTEM_ACTORS['library'].get_actor_instance()
     expected = {
         '@context': serializers.AP_CONTEXT,
@@ -555,6 +560,9 @@ def test_activity_pub_audio_serializer_to_ap(factories):
                 'musicbrainz_id': tf.track.mbid,
                 'title': tf.track.title,
             },
+            'size': tf.size,
+            'length': tf.duration,
+            'bitrate': tf.bitrate,
         },
         'url': {
             'href': utils.full_url(tf.path),
@@ -599,6 +607,9 @@ def test_activity_pub_audio_serializer_to_ap_no_mbid(factories):
                 'title': tf.track.title,
                 'musicbrainz_id': None,
             },
+            'size': None,
+            'length': None,
+            'bitrate': None,
         },
         'url': {
             'href': utils.full_url(tf.path),

api/tests/music/test_commands.py

+from funkwhale_api.music.management.commands import fix_track_files
+
+
+def test_fix_track_files_bitrate_length(factories, mocker):
+    tf1 = factories['music.TrackFile'](bitrate=1, duration=2)
+    tf2 = factories['music.TrackFile'](bitrate=None, duration=None)
+    c = fix_track_files.Command()
+
+    mocker.patch(
+        'funkwhale_api.music.utils.get_audio_file_data',
+        return_value={'bitrate': 42, 'length': 43})
+
+    c.fix_file_data(dry_run=False)
+
+    tf1.refresh_from_db()
+    tf2.refresh_from_db()
+
+    # not updated
+    assert tf1.bitrate == 1
+    assert tf1.duration == 2
+
+    # updated
+    assert tf2.bitrate == 42
+    assert tf2.duration == 43
+
+
+def test_fix_track_files_size(factories, mocker):
+    tf1 = factories['music.TrackFile'](size=1)
+    tf2 = factories['music.TrackFile'](size=None)
+    c = fix_track_files.Command()
+
+    mocker.patch(
+        'funkwhale_api.music.models.TrackFile.get_file_size',
+        return_value=2)
+
+    c.fix_file_size(dry_run=False)
+
+    tf1.refresh_from_db()
+    tf2.refresh_from_db()
+
+    # not updated
+    assert tf1.size == 1
+
+    # updated
+    assert tf2.size == 2

api/tests/music/test_import.py

 import json
+import os
 import pytest
 
 from django.urls import reverse
@@ -7,6 +8,8 @@ from funkwhale_api.federation import actors
 from funkwhale_api.federation import serializers as federation_serializers
 from funkwhale_api.music import tasks
 
+DATA_DIR = os.path.dirname(os.path.abspath(__file__))
+
 
 def test_create_import_can_bind_to_request(
         artists, albums, mocker, factories, superuser_api_client):
@@ -40,11 +43,20 @@ def test_create_import_can_bind_to_request(
     assert batch.import_request == request
 
 
-def test_import_job_from_federation_no_musicbrainz(factories):
+def test_import_job_from_federation_no_musicbrainz(factories, mocker):
+    mocker.patch(
+        'funkwhale_api.music.utils.get_audio_file_data',
+        return_value={'bitrate': 24, 'length': 666})
+    mocker.patch(
+        'funkwhale_api.music.models.TrackFile.get_file_size',
+        return_value=42)
     lt = factories['federation.LibraryTrack'](
         artist_name='Hello',
         album_title='World',
         title='Ping',
+        metadata__length=42,
+        metadata__bitrate=43,
+        metadata__size=44,
     )
     job = factories['music.ImportJob'](
         federation=True,
@@ -56,6 +68,9 @@ def test_import_job_from_federation_no_musicbrainz(factories):
 
     tf = job.track_file
     assert tf.mimetype == lt.audio_mimetype
+    assert tf.duration == 42
+    assert tf.bitrate == 43
+    assert tf.size == 44
     assert tf.library_track == job.library_track
     assert tf.track.title == 'Ping'
     assert tf.track.artist.name == 'Hello'
@@ -234,13 +249,13 @@ def test_import_batch_notifies_followers(
 
 
 def test__do_import_in_place_mbid(factories, tmpfile):
-    path = '/test.ogg'
+    path = os.path.join(DATA_DIR, 'test.ogg')
     job = factories['music.ImportJob'](
-        in_place=True, source='file:///test.ogg')
+        in_place=True, source='file://{}'.format(path))
 
     track = factories['music.Track'](mbid=job.mbid)
     tf = tasks._do_import(job, use_acoustid=False)
 
     assert bool(tf.audio_file) is False
-    assert tf.source == 'file:///test.ogg'
+    assert tf.source == 'file://{}'.format(path)
     assert tf.mimetype == 'audio/ogg'

api/tests/music/test_models.py

@@ -85,3 +85,28 @@ def test_track_file_file_name(factories):
     tf = factories['music.TrackFile'](audio_file__from_path=path)
 
     assert tf.filename == tf.track.full_name + '.mp3'
+
+
+def test_track_get_file_size(factories):
+    name = 'test.mp3'
+    path = os.path.join(DATA_DIR, name)
+    tf = factories['music.TrackFile'](audio_file__from_path=path)
+
+    assert tf.get_file_size() == 297745
+
+
+def test_track_get_file_size_federation(factories):
+    tf = factories['music.TrackFile'](
+        federation=True,
+        library_track__with_audio_file=True)
+
+    assert tf.get_file_size() == tf.library_track.audio_file.size
+
+
+def test_track_get_file_size_in_place(factories):
+    name = 'test.mp3'
+    path = os.path.join(DATA_DIR, name)
+    tf = factories['music.TrackFile'](
+        in_place=True, source='file://{}'.format(path))
+
+    assert tf.get_file_size() == 297745

api/tests/music/test_tasks.py

@@ -62,6 +62,9 @@ def test_import_job_can_run_with_file_and_acoustid(
             'score': 0.860825}],
         'status': 'ok'
     }
+    mocker.patch(
+        'funkwhale_api.music.utils.get_audio_file_data',
+        return_value={'bitrate': 42, 'length': 43})
     mocker.patch(
         'funkwhale_api.musicbrainz.api.artists.get',
         return_value=artists['get']['adhesive_wombat'])
@@ -82,7 +85,9 @@ def test_import_job_can_run_with_file_and_acoustid(
 
     with open(path, 'rb') as f:
         assert track_file.audio_file.read() == f.read()
-    assert track_file.duration == 268
+    assert track_file.bitrate == 42
+    assert track_file.duration == 43
+    assert track_file.size == os.path.getsize(path)
     # audio file is deleted from import job once persisted to audio file
     assert not job.audio_file
     assert job.status == 'finished'

api/tests/music/test_utils.py

+import os
+import pytest
+
 from funkwhale_api.music import utils
 
+DATA_DIR = os.path.dirname(os.path.abspath(__file__))
+
 
 def test_guess_mimetype_try_using_extension(factories, mocker):
     mocker.patch(
@@ -17,3 +22,16 @@ def test_guess_mimetype_try_using_extension_if_fail(factories, mocker):
         audio_file__filename='test.mp3')
 
     assert utils.guess_mimetype(f.audio_file) == 'audio/mpeg'
+
+
+@pytest.mark.parametrize('name, expected', [
+    ('sample.flac', {'bitrate': 1608000, 'length': 0.001}),
+    ('test.mp3', {'bitrate': 8000, 'length': 267.70285714285717}),
+    ('test.ogg', {'bitrate': 128000, 'length': 229.18304166666667}),
+])
+def test_get_audio_file_data(name, expected):
+    path = os.path.join(DATA_DIR, name)
+    with open(path, 'rb') as f:
+        result = utils.get_audio_file_data(f)
+
+    assert result == expected