Importer updates: watch directories, handle metadata updates

api/config/settings/common.py

@@ -1309,3 +1309,6 @@ IGNORE_FORWARDED_HOST_AND_PROTO = env.bool(
 """
 Use :attr:`FUNKWHALE_HOSTNAME` and :attr:`FUNKWHALE_PROTOCOL ` instead of request header.
 """
+
+HASHING_ALGORITHM = "sha256"
+HASHING_CHUNK_SIZE = 1024 * 100

api/funkwhale_api/common/utils.py

 import datetime
+import hashlib
 
 from django.core.files.base import ContentFile
 from django.http import request
@@ -458,3 +459,19 @@ def monkey_patch_request_build_absolute_uri():
 
     request.HttpRequest.scheme = property(scheme)
     request.HttpRequest.get_host = get_host
+
+
+def get_file_hash(file, algo=None, chunk_size=None, full_read=False):
+    algo = algo or settings.HASHING_ALGORITHM
+    chunk_size = chunk_size or settings.HASHING_CHUNK_SIZE
+    handler = getattr(hashlib, algo)
+    hash = handler()
+    file.seek(0)
+    if full_read:
+        for byte_block in iter(lambda: file.read(chunk_size), b""):
+            hash.update(byte_block)
+    else:
+        # sometimes, it's useful to only hash the beginning of the file, e.g
+        # to avoid a lot of I/O when crawling large libraries
+        hash.update(file.read(chunk_size))
+    return "{}:{}".format(algo, hash.hexdigest())

api/funkwhale_api/music/management/commands/fix_uploads.py

@@ -2,6 +2,7 @@ from django.core.management.base import BaseCommand
 from django.db import transaction
 from django.db.models import Q
 
+from funkwhale_api.common import utils as common_utils
 from funkwhale_api.music import models, utils
 
 
@@ -17,9 +18,9 @@ class Command(BaseCommand):
             help="Do not execute anything",
         )
         parser.add_argument(
-            "--mimetypes",
+            "--mimetype",
             action="store_true",
-            dest="mimetypes",
+            dest="mimetype",
             default=True,
             help="Check and fix mimetypes",
         )
@@ -37,16 +38,33 @@ class Command(BaseCommand):
             default=False,
             help="Check and fix file size, can be really slow because it needs to access files",
         )
+        parser.add_argument(
+            "--checksum",
+            action="store_true",
+            dest="checksum",
+            default=False,
+            help="Check and fix file size, can be really slow because it needs to access files",
+        )
+        parser.add_argument(
+            "--batch-size",
+            "-s",
+            dest="batch_size",
+            default=1000,
+            type=int,
+            help="Size of each updated batch",
+        )
 
     def handle(self, *args, **options):
         if options["dry_run"]:
             self.stdout.write("Dry-run on, will not commit anything")
-        if options["mimetypes"]:
+        if options["mimetype"]:
             self.fix_mimetypes(**options)
         if options["data"]:
             self.fix_file_data(**options)
         if options["size"]:
             self.fix_file_size(**options)
+        if options["checksum"]:
+            self.fix_file_checksum(**options)
 
     @transaction.atomic
     def fix_mimetypes(self, dry_run, **kwargs):
@@ -54,11 +72,12 @@ class Command(BaseCommand):
         matching = models.Upload.objects.filter(
             Q(source__startswith="file://") | Q(source__startswith="upload://")
         ).exclude(mimetype__startswith="audio/")
+        total = matching.count()
         self.stdout.write(
-            "[mimetypes] {} entries found with bad or no mimetype".format(
-                matching.count()
-            )
+            "[mimetypes] {} entries found with bad or no mimetype".format(total)
         )
+        if not total:
+            return
         for extension, mimetype in utils.EXTENSION_TO_MIMETYPE.items():
             qs = matching.filter(source__endswith=".{}".format(extension))
             self.stdout.write(
@@ -81,24 +100,36 @@ class Command(BaseCommand):
         )
         if dry_run:
             return
-        for i, upload in enumerate(matching.only("audio_file")):
-            self.stdout.write(
-                "[bitrate/length] {}/{} fixing file #{}".format(i + 1, total, upload.pk)
-            )
 
-            try:
-                audio_file = upload.get_audio_file()
-                if audio_file:
+        chunks = common_utils.chunk_queryset(
+            matching.only("id", "audio_file", "source"), kwargs["batch_size"]
+        )
+        handled = 0
+        for chunk in chunks:
+            updated = []
+            for upload in chunk:
+                handled += 1
+                self.stdout.write(
+                    "[bitrate/length] {}/{} fixing file #{}".format(
+                        handled, total, upload.pk
+                    )
+                )
+
+                try:
+                    audio_file = upload.get_audio_file()
                     data = utils.get_audio_file_data(audio_file)
                     upload.bitrate = data["bitrate"]
                     upload.duration = data["length"]
-                    upload.save(update_fields=["duration", "bitrate"])
+                except Exception as e:
+                    self.stderr.write(
+                        "[bitrate/length] error with file #{}: {}".format(
+                            upload.pk, str(e)
+                        )
+                    )
                 else:
-                    self.stderr.write("[bitrate/length] no file found")
-            except Exception as e:
-                self.stderr.write(
-                    "[bitrate/length] error with file #{}: {}".format(upload.pk, str(e))
-                )
+                    updated.append(upload)
+
+            models.Upload.objects.bulk_update(updated, ["bitrate", "duration"])
 
     def fix_file_size(self, dry_run, **kwargs):
         self.stdout.write("Fixing missing size...")
@@ -107,15 +138,64 @@ class Command(BaseCommand):
         self.stdout.write("[size] {} entries found with missing values".format(total))
         if dry_run:
             return
-        for i, upload in enumerate(matching.only("size")):
-            self.stdout.write(
-                "[size] {}/{} fixing file #{}".format(i + 1, total, upload.pk)
-            )
 
-            try:
-                upload.size = upload.get_file_size()
-                upload.save(update_fields=["size"])
-            except Exception as e:
-                self.stderr.write(
-                    "[size] error with file #{}: {}".format(upload.pk, str(e))
+        chunks = common_utils.chunk_queryset(
+            matching.only("id", "audio_file", "source"), kwargs["batch_size"]
+        )
+        handled = 0
+        for chunk in chunks:
+            updated = []
+            for upload in chunk:
+                handled += 1
+
+                self.stdout.write(
+                    "[size] {}/{} fixing file #{}".format(handled, total, upload.pk)
+                )
+
+                try:
+                    upload.size = upload.get_file_size()
+                except Exception as e:
+                    self.stderr.write(
+                        "[size] error with file #{}: {}".format(upload.pk, str(e))
+                    )
+                else:
+                    updated.append(upload)
+
+            models.Upload.objects.bulk_update(updated, ["size"])
+
+    def fix_file_checksum(self, dry_run, **kwargs):
+        self.stdout.write("Fixing missing checksums...")
+        matching = models.Upload.objects.filter(
+            Q(checksum=None)
+            & (Q(audio_file__isnull=False) | Q(source__startswith="file://"))
+        )
+        total = matching.count()
+        self.stdout.write(
+            "[checksum] {} entries found with missing values".format(total)
+        )
+        if dry_run:
+            return
+        chunks = common_utils.chunk_queryset(
+            matching.only("id", "audio_file", "source"), kwargs["batch_size"]
+        )
+        handled = 0
+        for chunk in chunks:
+            updated = []
+            for upload in chunk:
+                handled += 1
+                self.stdout.write(
+                    "[checksum] {}/{} fixing file #{}".format(handled, total, upload.pk)
                 )
+
+                try:
+                    upload.checksum = common_utils.get_file_hash(
+                        upload.get_audio_file()
+                    )
+                except Exception as e:
+                    self.stderr.write(
+                        "[checksum] error with file #{}: {}".format(upload.pk, str(e))
+                    )
+                else:
+                    updated.append(upload)
+
+            models.Upload.objects.bulk_update(updated, ["checksum"])

api/funkwhale_api/music/migrations/0052_auto_20200505_0810.py

+# Generated by Django 3.0.4 on 2020-05-05 08:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0051_auto_20200319_1249'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='upload',
+            name='checksum',
+            field=models.CharField(blank=True, db_index=True, max_length=100, null=True),
+        ),
+        migrations.AlterField(
+            model_name='uploadversion',
+            name='mimetype',
+            field=models.CharField(choices=[('audio/mp3', 'mp3'), ('audio/mpeg3', 'mp3'), ('audio/x-mp3', 'mp3'), ('audio/mpeg', 'mp3'), ('video/ogg', 'ogg'), ('audio/ogg', 'ogg'), ('audio/opus', 'opus'), ('audio/x-m4a', 'aac'), ('audio/x-m4a', 'm4a'), ('audio/x-flac', 'flac'), ('audio/flac', 'flac')], max_length=50),
+        ),
+    ]

api/funkwhale_api/music/models.py

@@ -655,6 +655,14 @@ class Track(APIModelMixin):
 
 
 class UploadQuerySet(common_models.NullsLastQuerySet):
+    def in_place(self, include=True):
+        query = models.Q(source__startswith="file://") & (
+            models.Q(audio_file="") | models.Q(audio_file=None)
+        )
+        if not include:
+            query = ~query
+        return self.filter(query)
+
     def playable_by(self, actor, include=True):
         libraries = Library.objects.viewable_by(actor)
 
@@ -754,6 +762,9 @@ class Upload(models.Model):
     )
     downloads_count = models.PositiveIntegerField(default=0)
 
+    # stores checksums such as `sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855`
+    checksum = models.CharField(max_length=100, db_index=True, null=True, blank=True)
+
     objects = UploadQuerySet.as_manager()
 
     @property
@@ -833,7 +844,7 @@ class Upload(models.Model):
     def get_audio_file(self):
         if self.audio_file:
             return self.audio_file.open()
-        if self.source.startswith("file://"):
+        if self.source and self.source.startswith("file://"):
             return open(self.source.replace("file://", "", 1), "rb")
 
     def get_audio_data(self):
@@ -866,6 +877,15 @@ class Upload(models.Model):
                 self.mimetype = mimetypes.guess_type(self.source)[0]
         if not self.size and self.audio_file:
             self.size = self.audio_file.size
+        if not self.checksum:
+            try:
+                audio_file = self.get_audio_file()
+            except FileNotFoundError:
+                pass
+            else:
+                if audio_file:
+                    self.checksum = common_utils.get_file_hash(audio_file)
+
         if not self.pk and not self.fid and self.library.actor.get_user():
             self.fid = self.get_federation_id()
         return super().save(**kwargs)

api/funkwhale_api/music/tasks.py

@@ -851,3 +851,71 @@ def update_library_entity(obj, data):
     obj.save(update_fields=list(data.keys()))
 
     return obj
+
+
+UPDATE_CONFIG = {
+    "track": {
+        "position": {},
+        "title": {},
+        "mbid": {},
+        "disc_number": {},
+        "copyright": {},
+        "license": {
+            "getter": lambda data, field: licenses.match(
+                data.get("license"), data.get("copyright")
+            )
+        },
+    },
+    "album": {"title": {}, "mbid": {}, "release_date": {}},
+    "artist": {"name": {}, "mbid": {}},
+    "album_artist": {"name": {}, "mbid": {}},
+}
+
+
+@transaction.atomic
+def update_track_metadata(audio_metadata, track):
+    # XXX: implement this to support updating metadata when an imported files
+    # is updated by an outside tool (e.g beets).
+    serializer = metadata.TrackMetadataSerializer(data=audio_metadata)
+    serializer.is_valid(raise_exception=True)
+    new_data = serializer.validated_data
+
+    to_update = [
+        ("track", track, lambda data: data),
+        ("album", track.album, lambda data: data["album"]),
+        ("artist", track.artist, lambda data: data["artists"][0]),
+        (
+            "album_artist",
+            track.album.artist if track.album else None,
+            lambda data: data["album"]["artists"][0],
+        ),
+    ]
+    for id, obj, data_getter in to_update:
+        if not obj:
+            continue
+        obj_updated_fields = []
+        try:
+            obj_data = data_getter(new_data)
+        except IndexError:
+            continue
+        for field, config in UPDATE_CONFIG[id].items():
+            getter = config.get(
+                "getter", lambda data, field: data[config.get("field", field)]
+            )
+            try:
+                new_value = getter(obj_data, field)
+            except KeyError:
+                continue
+            old_value = getattr(obj, field)
+            if new_value == old_value:
+                continue
+            obj_updated_fields.append(field)
+            setattr(obj, field, new_value)
+
+        if obj_updated_fields:
+            obj.save(update_fields=obj_updated_fields)
+
+    if track.album and "album" in new_data and new_data["album"].get("cover_data"):
+        common_utils.attach_file(
+            track.album, "attachment_cover", new_data["album"].get("cover_data")
+        )

api/requirements/base.txt

@@ -83,3 +83,4 @@ service_identity==18.1.0
 markdown>=3.2,<4
 bleach>=3,<4
 feedparser==6.0.0b3
+watchdog==0.10.2

api/tests/common/test_utils.py

@@ -258,3 +258,12 @@ def test_monkey_patch_request_build_absolute_uri(
     request = fake_request.get("/", **meta)
 
     assert request.build_absolute_uri(path) == expected
+
+
+def test_get_file_hash(tmpfile, settings):
+    settings.HASHING_ALGORITHM = "sha256"
+    content = b"hello"
+    tmpfile.write(content)
+    # echo -n "hello" | sha256sum
+    expected = "sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824"
+    assert utils.get_file_hash(tmpfile) == expected

api/tests/music/test_commands.py

 import os
 import pytest
 
+from funkwhale_api.common import utils as common_utils
 from funkwhale_api.music.management.commands import check_inplace_files
 from funkwhale_api.music.management.commands import fix_uploads
 from funkwhale_api.music.management.commands import prune_library
@@ -18,7 +19,7 @@ def test_fix_uploads_bitrate_length(factories, mocker):
         return_value={"bitrate": 42, "length": 43},
     )
 
-    c.fix_file_data(dry_run=False)
+    c.fix_file_data(dry_run=False, batch_size=100)
 
     upload1.refresh_from_db()
     upload2.refresh_from_db()
@@ -41,7 +42,7 @@ def test_fix_uploads_size(factories, mocker):
 
     mocker.patch("funkwhale_api.music.models.Upload.get_file_size", return_value=2)
 
-    c.fix_file_size(dry_run=False)
+    c.fix_file_size(dry_run=False, batch_size=100)
 
     upload1.refresh_from_db()
     upload2.refresh_from_db()
@@ -69,7 +70,7 @@ def test_fix_uploads_mimetype(factories, mocker):
         mimetype="audio/something",
     )
     c = fix_uploads.Command()
-    c.fix_mimetypes(dry_run=False)
+    c.fix_mimetypes(dry_run=False, batch_size=100)
 
     upload1.refresh_from_db()
     upload2.refresh_from_db()
@@ -78,6 +79,25 @@ def test_fix_uploads_mimetype(factories, mocker):
     assert upload2.mimetype == "audio/something"
 
 
+def test_fix_uploads_checksum(factories, mocker):
+    upload1 = factories["music.Upload"]()
+    upload2 = factories["music.Upload"]()
+    upload1.__class__.objects.filter(pk=upload1.pk).update(checksum="test")
+    upload2.__class__.objects.filter(pk=upload2.pk).update(checksum=None)
+    c = fix_uploads.Command()
+
+    c.fix_file_checksum(dry_run=False, batch_size=100)
+
+    upload1.refresh_from_db()
+    upload2.refresh_from_db()
+
+    # not updated
+    assert upload1.checksum == "test"
+
+    # updated
+    assert upload2.checksum == common_utils.get_file_hash(upload2.audio_file)
+
+
 def test_prune_library_dry_run(factories):
     prunable = factories["music.Track"]()
     not_prunable = factories["music.Track"]()

api/tests/music/test_models.py

@@ -5,6 +5,7 @@ import pytest
 from django.utils import timezone
 from django.urls import reverse
 
+from funkwhale_api.common import utils as common_utils
 from funkwhale_api.music import importers, models, tasks
 from funkwhale_api.federation import utils as federation_utils
 
@@ -164,6 +165,17 @@ def test_audio_track_mime_type(extention, mimetype, factories):
     assert upload.mimetype == mimetype
 
 
+@pytest.mark.parametrize("name", ["test.ogg", "test.mp3"])
+def test_audio_track_checksum(name, factories):
+
+    path = os.path.join(DATA_DIR, name)
+    upload = factories["music.Upload"](audio_file__from_path=path, mimetype=None)
+
+    with open(path, "rb") as f:
+        expected = common_utils.get_file_hash(f)
+    assert upload.checksum == expected
+
+
 def test_upload_file_name(factories):
     name = "test.mp3"
     path = os.path.join(DATA_DIR, name)

api/tests/music/test_tasks.py

@@ -1329,3 +1329,40 @@ def test_can_import_track_with_same_position_in_same_discs_skipped(factories, mo
     new_upload.refresh_from_db()
 
     assert new_upload.import_status == "skipped"
+
+
+def test_update_track_metadata(factories):
+    track = factories["music.Track"]()
+    data = {
+        "title": "Peer Gynt Suite no. 1, op. 46: I. Morning",
+        "artist": "Edvard Grieg",
+        "album_artist": "Edvard Grieg; Musopen Symphony Orchestra",
+        "album": "Peer Gynt Suite no. 1, op. 46",
+        "date": "2012-08-15",
+        "position": "4",
+        "disc_number": "2",
+        "musicbrainz_albumid": "a766da8b-8336-47aa-a3ee-371cc41ccc75",
+        "mbid": "bd21ac48-46d8-4e78-925f-d9cc2a294656",
+        "musicbrainz_artistid": "013c8e5b-d72a-4cd3-8dee-6c64d6125823",
+        "musicbrainz_albumartistid": "013c8e5b-d72a-4cd3-8dee-6c64d6125823;5b4d7d2d-36df-4b38-95e3-a964234f520f",
+        "license": "Dummy license: http://creativecommons.org/licenses/by-sa/4.0/",
+        "copyright": "Someone",
+        "comment": "hello there",
+    }
+    tasks.update_track_metadata(metadata.FakeMetadata(data), track)
+
+    track.refresh_from_db()
+
+    assert track.title == data["title"]
+    assert track.position == int(data["position"])
+    assert track.disc_number == int(data["disc_number"])
+    assert track.license.code == "cc-by-sa-4.0"
+    assert track.copyright == data["copyright"]
+    assert str(track.mbid) == data["mbid"]
+    assert track.album.title == data["album"]
+    assert track.album.release_date == datetime.date(2012, 8, 15)
+    assert str(track.album.mbid) == data["musicbrainz_albumid"]
+    assert track.artist.name == data["artist"]
+    assert str(track.artist.mbid) == data["musicbrainz_artistid"]
+    assert track.album.artist.name == "Edvard Grieg"
+    assert str(track.album.artist.mbid) == "013c8e5b-d72a-4cd3-8dee-6c64d6125823"

api/tests/test_import_audio_file.py

@@ -4,6 +4,8 @@ import pytest
 from django.core.management import call_command
 from django.core.management.base import CommandError
 
+from funkwhale_api.common import utils as common_utils
+from funkwhale_api.music.management.commands import import_files
 
 DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "files")
 
@@ -159,3 +161,194 @@ def test_import_files_in_place(factories, mocker, settings):
 def test_storage_rename_utf_8_files(factories):
     upload = factories["music.Upload"](audio_file__filename="été.ogg")
     assert upload.audio_file.name.endswith("ete.ogg")
+
+
+@pytest.mark.parametrize("name", ["modified", "moved", "created", "deleted"])
+def test_handle_event(name, mocker):
+    handler = mocker.patch.object(import_files, "handle_{}".format(name))
+
+    event = {"type": name}
+    stdout = mocker.Mock()
+    kwargs = {"hello": "world"}
+    import_files.handle_event(event, stdout, **kwargs)
+
+    handler.assert_called_once_with(event=event, stdout=stdout, **kwargs)
+
+
+def test_handle_created(mocker):
+    handle_modified = mocker.patch.object(import_files, "handle_modified")
+
+    event = mocker.Mock()
+    stdout = mocker.Mock()
+    kwargs = {"hello": "world"}
+    import_files.handle_created(event, stdout, **kwargs)
+
+    handle_modified.assert_called_once_with(event, stdout, **kwargs)
+
+
+def test_handle_deleted(factories, mocker):
+    stdout = mocker.Mock()
+    event = {
+        "path": "/path.mp3",
+    }
+    library = factories["music.Library"]()
+    deleted = factories["music.Upload"](
+        library=library,
+        source="file://{}".format(event["path"]),
+        import_status="finished",
+        audio_file=None,
+    )
+    kept = [
+        factories["music.Upload"](
+            library=library,
+            source="file://{}".format(event["path"]),
+            import_status="finished",
+        ),
+        factories["music.Upload"](
+            source="file://{}".format(event["path"]),
+            import_status="finished",
+            audio_file=None,
+        ),
+    ]
+
+    import_files.handle_deleted(
+        event=event, stdout=stdout, library=library, in_place=True
+    )
+
+    with pytest.raises(deleted.DoesNotExist):
+        deleted.refresh_from_db()
+
+    for upload in kept:
+        upload.refresh_from_db()
+
+
+def test_handle_moved(factories, mocker):
+    stdout = mocker.Mock()
+    event = {
+        "src_path": "/path.mp3",
+        "dest_path": "/new_path.mp3",
+    }
+    library = factories["music.Library"]()
+    updated = factories["music.Upload"](
+        library=library,
+        source="file://{}".format(event["src_path"]),
+        import_status="finished",
+        audio_file=None,
+    )
+    untouched = [
+        factories["music.Upload"](
+            library=library,
+            source="file://{}".format(event["src_path"]),
+            import_status="finished",
+        ),
+        factories["music.Upload"](
+            source="file://{}".format(event["src_path"]),
+            import_status="finished",
+            audio_file=None,
+        ),
+    ]
+
+    import_files.handle_moved(
+        event=event, stdout=stdout, library=library, in_place=True
+    )
+
+    updated.refresh_from_db()
+    assert updated.source == "file://{}".format(event["dest_path"])
+    for upload in untouched:
+        source = upload.source
+        upload.refresh_from_db()
+        assert source == upload.source
+
+
+def test_handle_modified_creates_upload(tmpfile, factories, mocker):
+    stdout = mocker.Mock()
+    event = {
+        "path": tmpfile.name,
+    }
+    process_upload = mocker.patch("funkwhale_api.music.tasks.process_upload")
+    library = factories["music.Library"]()
+    import_files.handle_modified(
+        event=event,
+        stdout=stdout,
+        library=library,
+        in_place=True,
+        reference="hello",
+        replace=False,
+        dispatch_outbox=False,
+        broadcast=False,
+    )
+    upload = library.uploads.latest("id")
+    assert upload.source == "file://{}".format(event["path"])
+
+    process_upload.assert_called_once_with(upload_id=upload.pk)
+
+
+def test_handle_modified_skips_existing_checksum(tmpfile, factories, mocker):
+    stdout = mocker.Mock()
+    event = {
+        "path": tmpfile.name,
+    }
+    tmpfile.write(b"hello")
+
+    library = factories["music.Library"]()
+    factories["music.Upload"](
+        checksum=common_utils.get_file_hash(tmpfile),
+        library=library,
+        import_status="finished",
+    )
+    import_files.handle_modified(
+        event=event, stdout=stdout, library=library, in_place=True,
+    )
+    assert library.uploads.count() == 1
+
+
+def test_handle_modified_update_existing_path_if_found(tmpfile, factories, mocker):
+    stdout = mocker.Mock()
+    event = {
+        "path": tmpfile.name,
+    }
+    update_track_metadata = mocker.patch(
+        "funkwhale_api.music.tasks.update_track_metadata"
+    )
+    get_metadata = mocker.patch("funkwhale_api.music.models.Upload.get_metadata")
+    library = factories["music.Library"]()
+    track = factories["music.Track"](attributed_to=library.actor)
+    upload = factories["music.Upload"](
+        source="file://{}".format(event["path"]),
+        track=track,
+        checksum="old",
+        library=library,
+        import_status="finished",
+        audio_file=None,
+    )
+    import_files.handle_modified(
+        event=event, stdout=stdout, library=library, in_place=True,
+    )
+    update_track_metadata.assert_called_once_with(
+        get_metadata.return_value, upload.track,
+    )
+
+
+def test_handle_modified_update_existing_path_if_found_and_attributed_to(
+    tmpfile, factories, mocker
+):
+    stdout = mocker.Mock()
+    event = {
+        "path": tmpfile.name,
+    }
+    update_track_metadata = mocker.patch(
+        "funkwhale_api.music.tasks.update_track_metadata"
+    )
+    library = factories["music.Library"]()
+    factories["music.Upload"](
+        source="file://{}".format(event["path"]),
+        checksum="old",
+        library=library,
+        track__attributed_to=factories["federation.Actor"](),
+        import_status="finished",
+        audio_file=None,
+    )
+    import_files.handle_modified(
+        event=event, stdout=stdout, library=library, in_place=True,
+    )
+    update_track_metadata.assert_not_called()

changes/changelog.d/1093.bugfix

-Fixed mimetype detection issue that broke transcoding on some tracks (#1093). Run ``python manage.py fix_uploads --mimetypes`` to set proper mimetypes on existing uploads.
+Fixed mimetype detection issue that broke transcoding on some tracks (#1093). Run ``python manage.py fix_uploads --mimetype`` to set proper mimetypes on existing uploads.

changes/changelog.d/721.feature

+Support a --watch mode with ``import_files`` to automatically add, update and remove files when filesystem is updated (#721)

docs/admin/importing-music.rst

-Importing music
-================
+Importing music from the server
+===============================
 
-From music directory on the server
-----------------------------------
-
-You can import music files in Funkwhale assuming they are located on the server
-and readable by the Funkwhale application. Your music files should contain at
+Funkwhale can import music files that are located on the server assuming
+they readable by the Funkwhale application. Your music files should contain at
 least an ``artist``, ``album`` and ``title`` tags, but we recommend you tag
 it extensively using a proper tool, such as Beets or Musicbrainz Picard.
 
-You can import those tracks as follows, assuming they are located in
+Funkwhale supports two different import modes:
+
+- copy (the default): files are copied into Funkwhale's internal storage. This means importing a 1GB library will result in the same amount of space being used by Funkwhale.
+- :ref:`in-place <in-place-import>` (when the ``--in-place`` is provided): files are referenced in Funkwhale's DB but not copied or touched in anyway. This is useful if you have a huge library, or one that is updated by an external tool such as Beets..
+
+.. note::
+
+    In Funkwhale 1.0, **the default behaviour will change to in-place import**
+
+Regardless of the mode you're choosing, import works as described below, assuming your files are located in
 ``/srv/funkwhale/data/music``:
 
 .. code-block:: bash
@@ -17,6 +23,17 @@ You can import those tracks as follows, assuming they are located in
     export LIBRARY_ID="<your_libary_id>"
     python api/manage.py import_files $LIBRARY_ID "/srv/funkwhale/data/music/" --recursive --noinput
 
+.. note::
+    You'll have to create a library in the Web UI before to get your library ID. Simply visit
+    https://yourdomain/content/libraries/ to create one.
+
+    Library IDs are available in library urls or sharing link. In this example:
+    https://funkwhale.instance/content/libraries/769a2ae3-eb3d-4aff-9f94-2c4d80d5c2d1,
+    the library ID is 769a2bc3-eb1d-4aff-9f84-2c4d80d5c2d1
+
+    You can use only the first characters of the ID when calling the command, like that:
+    ``export LIBRARY_ID="769a2bc3"``
+
 When you use docker, the ``/srv/funkwhale/data/music`` is mounted from the host
 to the ``/music`` directory on the container:
 
@@ -32,16 +49,6 @@ When you installed Funkwhale via ansible, you need to call a script instead of P
     export LIBRARY_ID="<your_libary_id>"
     /srv/funkwhale/manage import_files $LIBRARY_ID "/srv/funkwhale/data/music/" --recursive --noinput
 
-.. note::
-    You'll have to create a library in the Web UI before to get your library ID. Simply visit
-    https://yourdomain/content/libraries/ to create one.
-
-    Library IDs are available in library urls or sharing link. In this example:
-    https://funkwhale.instance/content/libraries/769a2ae3-eb3d-4aff-9f94-2c4d80d5c2d1,
-    the library ID is 769a2bc3-eb1d-4aff-9f84-2c4d80d5c2d1
-
-    You can use only the first characters of the ID when calling the command, like that:
-    ``export LIBRARY_ID="769a2bc3"``
 
 The import command supports several options, and you can check the help to
 get details::
@@ -63,6 +70,7 @@ get details::
     At the moment, only Flac, OGG/Vorbis and MP3 files with ID3 tags are supported
 
 
+
 .. _in-place-import:
 
 In-place import
@@ -88,14 +96,6 @@ configuration options to ensure the webserver can serve them properly:
 - :ref:`setting-MUSIC_DIRECTORY_PATH`
 - :ref:`setting-MUSIC_DIRECTORY_SERVE_PATH`
 
-.. warning::
-
-    While in-place import is faster and less disk-space-hungry, it's also
-    more fragile: if, for some reason, you move or rename the source files,
-    Funkwhale will not be able to serve those files anymore.
-
-    Thus, be especially careful when you manipulate the source files.
-
 We recommend you symlink all your music directories into ``/srv/funkwhale/data/music``
 and run the `import_files` command from that directory. This will make it possible
 to use multiple music directories, without any additional configuration
@@ -134,6 +134,49 @@ If you want to go with symlinks, ensure each symlinked directory is mounted as a
       # add your symlinked dirs here
       - /media/nfsshare:/media/nfsshare:ro
 
+Metadata updates
+^^^^^^^^^^^^^^^^
+
+When doing an import with in ``in-place`` mode, the importer will also check and update existing entries
+found in the database. For instance, if a file was imported, the ID3 Title tag was updated, and you rerun a scan,
+Funkwhale will pick up the new title. The following fields can be updated this way:
+
+- Track mbid
+- Track title
+- Track position and disc number
+- Track license and copyright
+- Album cover
+- Album title
+- Album mbid
+- Album release date
+- Artist name
+- Artist mbid
+- Album artist name
+- Album artist mbid
+
+
+React to filesystem events with ``--watch``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you have a really big library or one that is updated quite often, running the ``import_files`` command by hand
+may not be practical. To help with this use case, the ``import_files`` command supports a ``--watch`` flag that will observes filesystem events
+instead of performing a full import.
+
+File creation, move, update and removal are handled when ``--watch`` is provided:
+
+- Files created in the watched directory are imported immediatly
+- If using ``in-place`` mode, files updates trigger a metadata update on the corresponding entries
+- If using ``in-place`` mode, files that are moved and known by Funkwhale will see their path updated in Funkwhale's DB
+- If using ``in-place`` mode, files that are removed and known by Funkwhale will be removed from Funkwhale's DB
+
+Pruning dangling metadata with ``--prune``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Funkwhale is, by design, conservative with music metadata in its database. If you remove a file from Funkwhale's DB,
+the corresponding artist, album and track object won't be deleted by default.
+
+If you want to prune dangling metadata from the database once the ``import_files`` command is over, simply add the ``--prune`` flag.
+This also works in with ``--watch``.
 
 Album covers
 ^^^^^^^^^^^^
@@ -159,9 +202,3 @@ under creative commons (courtesy of Jamendo):
     ./download-tracks.sh music.txt
 
 This will download a bunch of zip archives (one per album) under the ``data/music`` directory and unzip their content.
-
-From other instances
---------------------
-
-Funkwhale also supports importing music from other instances. Please refer
-to :doc:`../federation/index` for more details.