From f1cf250e2959e2fe7a333994fd10fe1817795e3b Mon Sep 17 00:00:00 2001
From: Eliot Berriot <contact@eliotberriot.com>
Date: Sun, 25 Mar 2018 15:40:37 +0200
Subject: [PATCH] Fix #138: Better handling of utf-8 filenames during file
import
---
api/config/settings/common.py | 1 +
api/funkwhale_api/common/storage.py | 12 ++++++++++
.../utf8-\303\251\303\240\342\227\214.ogg" | 0
api/tests/test_import_audio_file.py | 24 +++++++++++++++++++
changes/changelog.d/138.bugfix | 1 +
5 files changed, 38 insertions(+)
create mode 100644 api/funkwhale_api/common/storage.py
create mode 100644 "api/tests/files/utf8-\303\251\303\240\342\227\214.ogg"
create mode 100644 changes/changelog.d/138.bugfix
diff --git a/api/config/settings/common.py b/api/config/settings/common.py
index 077566d1..f8bd7725 100644
--- a/api/config/settings/common.py
+++ b/api/config/settings/common.py
@@ -231,6 +231,7 @@ STATIC_ROOT = env("STATIC_ROOT", default=str(ROOT_DIR('staticfiles')))
# See: https://docs.djangoproject.com/en/dev/ref/settings/#static-url
STATIC_URL = env("STATIC_URL", default='/staticfiles/')
+DEFAULT_FILE_STORAGE = 'funkwhale_api.common.storage.ASCIIFileSystemStorage'
# See: https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#std:setting-STATICFILES_DIRS
STATICFILES_DIRS = (
diff --git a/api/funkwhale_api/common/storage.py b/api/funkwhale_api/common/storage.py
new file mode 100644
index 00000000..658ce795
--- /dev/null
+++ b/api/funkwhale_api/common/storage.py
@@ -0,0 +1,12 @@
+import unicodedata
+
+from django.core.files.storage import FileSystemStorage
+
+
+class ASCIIFileSystemStorage(FileSystemStorage):
+ """
+ Convert unicode characters in name to ASCII characters.
+ """
+ def get_valid_name(self, name):
+ name = unicodedata.normalize('NFKD', name).encode('ascii', 'ignore')
+ return super().get_valid_name(name)
diff --git "a/api/tests/files/utf8-\303\251\303\240\342\227\214.ogg" "b/api/tests/files/utf8-\303\251\303\240\342\227\214.ogg"
new file mode 100644
index 00000000..e69de29b
diff --git a/api/tests/test_import_audio_file.py b/api/tests/test_import_audio_file.py
index 4f3de27d..67263e66 100644
--- a/api/tests/test_import_audio_file.py
+++ b/api/tests/test_import_audio_file.py
@@ -98,3 +98,27 @@ def test_import_files_skip_acoustid(factories, mocker):
music_tasks.import_job_run.delay,
import_job_id=job.pk,
use_acoustid=False)
+
+
+def test_import_files_works_with_utf8_file_name(factories, mocker):
+ m = mocker.patch('funkwhale_api.common.utils.on_commit')
+ user = factories['users.User'](username='me')
+ path = os.path.join(DATA_DIR, 'utf8-éà ◌.ogg')
+ call_command(
+ 'import_files',
+ path,
+ username='me',
+ async=True,
+ no_acoustid=True,
+ interactive=False)
+ batch = user.imports.latest('id')
+ job = batch.jobs.first()
+ m.assert_called_once_with(
+ music_tasks.import_job_run.delay,
+ import_job_id=job.pk,
+ use_acoustid=False)
+
+
+def test_storage_rename_utf_8_files(factories):
+ tf = factories['music.TrackFile'](audio_file__filename='été.ogg')
+ assert tf.audio_file.name.endswith('ete.ogg')
diff --git a/changes/changelog.d/138.bugfix b/changes/changelog.d/138.bugfix
new file mode 100644
index 00000000..2a8f7aeb
--- /dev/null
+++ b/changes/changelog.d/138.bugfix
@@ -0,0 +1 @@
+Better handling of utf-8 filenames during file import (#138)
--
GitLab