Importer chokes on files with bad unicode characters
Using the command line importer, I got the following backtrace:
Traceback (most recent call last):
File "manage.py", line 13, in <module>
execute_from_command_line(sys.argv)
File "/usr/local/lib/python3.5/site-packages/django/core/management/__init__.py", line 371, in execute_from_command_line
utility.execute()
File "/usr/local/lib/python3.5/site-packages/django/core/management/__init__.py", line 365, in execute
self.fetch_command(subcommand).run_from_argv(self.argv)
File "/usr/local/lib/python3.5/site-packages/django/core/management/base.py", line 288, in run_from_argv
self.execute(*args, **cmd_options)
File "/usr/local/lib/python3.5/site-packages/django/core/management/base.py", line 335, in execute
output = self.handle(*args, **options)
File "/app/funkwhale_api/providers/audiofile/management/commands/import_files.py", line 90, in handle
batch = self.do_import(matching, user=user, options=options)
File "/usr/local/lib/python3.5/contextlib.py", line 30, in inner
return func(*args, **kwds)
File "/app/funkwhale_api/providers/audiofile/management/commands/import_files.py", line 110, in do_import
source='file://' + path,
File "/usr/local/lib/python3.5/site-packages/django/db/models/fields/related_descriptors.py", line 623, in create
return super(RelatedManager, self.db_manager(db)).create(**kwargs)
File "/usr/local/lib/python3.5/site-packages/django/db/models/manager.py", line 82, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "/usr/local/lib/python3.5/site-packages/django/db/models/query.py", line 417, in create
obj.save(force_insert=True, using=self.db)
File "/usr/local/lib/python3.5/site-packages/django/db/models/base.py", line 729, in save
force_update=force_update, update_fields=update_fields)
File "/usr/local/lib/python3.5/site-packages/django/db/models/base.py", line 759, in save_base
updated = self._save_table(raw, cls, force_insert, force_update, using, update_fields)
File "/usr/local/lib/python3.5/site-packages/django/db/models/base.py", line 842, in _save_table
result = self._do_insert(cls._base_manager, using, fields, update_pk, raw)
File "/usr/local/lib/python3.5/site-packages/django/db/models/base.py", line 880, in _do_insert
using=using, raw=raw)
File "/usr/local/lib/python3.5/site-packages/django/db/models/manager.py", line 82, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "/usr/local/lib/python3.5/site-packages/django/db/models/query.py", line 1125, in _insert
return query.get_compiler(using=using).execute_sql(return_id)
File "/usr/local/lib/python3.5/site-packages/django/db/models/sql/compiler.py", line 1283, in execute_sql
cursor.execute(sql, params)
File "/usr/local/lib/python3.5/site-packages/cacheops/transaction.py", line 99, in execute
result = self._no_monkey.execute(self, sql, params)
File "/usr/local/lib/python3.5/site-packages/django/db/backends/utils.py", line 68, in execute
return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
File "/usr/local/lib/python3.5/site-packages/django/db/backends/utils.py", line 77, in _execute_with_wrappers
return executor(sql, params, many, context)
File "/usr/local/lib/python3.5/site-packages/django/db/backends/utils.py", line 85, in _execute
return self.cursor.execute(sql, params)
UnicodeEncodeError: 'utf-8' codec can't encode character '\udce8' in position 51: surrogates not allowedThis aborts the whole import.