MultiPart Upload with GCP as S3 Storage
There seems to be an issue with multi-part upload if the S3 back-end is GCP storage:
I came across this issue which sounded similar, and am wondering it it's the root of the issue: https://github.com/boto/boto3/issues/2207
I am able to upload a small file fine (likely under the multipart limit), but any of my larger flac files won't work.
Here is an output of the logs:
/venv/lib/python3.7/site-packages/rest_framework/pagination.py:200: UnorderedObjectListWarning: Pagination may yield inconsistent results with an unordered object_list: <class 'funkwhale_api.playlists
.models.Playlist'> PlaylistQuerySet.
paginator = self.django_paginator_class(queryset, page_size)
2020-08-17 00:19:46,800 django.request ERROR Internal Server Error: /api/v1/uploads/
Traceback (most recent call last):
File "/venv/lib/python3.7/site-packages/django/core/handlers/exception.py", line 34, in inner
response = get_response(request)
File "/venv/lib/python3.7/site-packages/django/core/handlers/base.py", line 115, in _get_response
response = self.process_exception_by_middleware(e, request)
File "/venv/lib/python3.7/site-packages/django/core/handlers/base.py", line 113, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "/usr/lib/python3.7/contextlib.py", line 74, in inner
return func(*args, **kwds)
File "/venv/lib/python3.7/site-packages/django/views/decorators/csrf.py", line 54, in wrapped_view
return view_func(*args, **kwargs)
File "/venv/lib/python3.7/site-packages/rest_framework/viewsets.py", line 114, in view
return self.dispatch(request, *args, **kwargs)
File "/venv/lib/python3.7/site-packages/rest_framework/views.py", line 505, in dispatch
response = self.handle_exception(exc)
File "/venv/lib/python3.7/site-packages/rest_framework/views.py", line 465, in handle_exception
self.raise_uncaught_exception(exc)
File "/venv/lib/python3.7/site-packages/rest_framework/views.py", line 476, in raise_uncaught_exception
raise exc
File "/venv/lib/python3.7/site-packages/rest_framework/views.py", line 502, in dispatch
response = handler(request, *args, **kwargs)
File "/venv/lib/python3.7/site-packages/rest_framework/mixins.py", line 19, in create
self.perform_create(serializer)
File "/app/funkwhale_api/music/views.py", line 688, in perform_create
upload = serializer.save()
File "/venv/lib/python3.7/site-packages/rest_framework/serializers.py", line 212, in save
self.instance = self.create(validated_data)
File "/venv/lib/python3.7/site-packages/rest_framework/serializers.py", line 948, in create
instance = ModelClass._default_manager.create(**validated_data)
File "/venv/lib/python3.7/site-packages/django/db/models/manager.py", line 82, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "/venv/lib/python3.7/site-packages/django/db/models/query.py", line 433, in create
obj.save(force_insert=True, using=self.db)
File "/app/funkwhale_api/music/models.py", line 890, in save
return super().save(**kwargs)
File "/venv/lib/python3.7/site-packages/django/db/models/base.py", line 746, in save
force_update=force_update, update_fields=update_fields)
File "/venv/lib/python3.7/site-packages/django/db/models/base.py", line 784, in save_base
force_update, using, update_fields,
File "/venv/lib/python3.7/site-packages/django/db/models/base.py", line 887, in _save_table
results = self._do_insert(cls._base_manager, using, fields, returning_fields, raw)
File "/venv/lib/python3.7/site-packages/django/db/models/base.py", line 926, in _do_insert
using=using, raw=raw,
File "/venv/lib/python3.7/site-packages/django/db/models/manager.py", line 82, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "/venv/lib/python3.7/site-packages/django/db/models/query.py", line 1204, in _insert
return query.get_compiler(using=using).execute_sql(returning_fields)
File "/venv/lib/python3.7/site-packages/django/db/models/sql/compiler.py", line 1391, in execute_sql
for sql, params in self.as_sql():
File "/venv/lib/python3.7/site-packages/django/db/models/sql/compiler.py", line 1336, in as_sql
for obj in self.query.objs
File "/venv/lib/python3.7/site-packages/django/db/models/sql/compiler.py", line 1336, in <listcomp>
for obj in self.query.objs
File "/venv/lib/python3.7/site-packages/django/db/models/sql/compiler.py", line 1335, in <listcomp>
[self.prepare_value(field, self.pre_save_val(field, obj)) for field in fields]
File "/venv/lib/python3.7/site-packages/django/db/models/sql/compiler.py", line 1286, in pre_save_val
return field.pre_save(obj, add=True)
File "/venv/lib/python3.7/site-packages/django/db/models/fields/files.py", line 288, in pre_save
file.save(file.name, file.file, save=False)
File "/venv/lib/python3.7/site-packages/django/db/models/fields/files.py", line 87, in save
self.name = self.storage.save(name, content, max_length=self.field.max_length)
File "/venv/lib/python3.7/site-packages/django/core/files/storage.py", line 52, in save
return self._save(name, content)
File "/venv/lib/python3.7/site-packages/storages/backends/s3boto3.py", line 547, in _save
obj.upload_fileobj(content, ExtraArgs=params)
File "/venv/lib/python3.7/site-packages/boto3/s3/inject.py", line 621, in object_upload_fileobj
ExtraArgs=ExtraArgs, Callback=Callback, Config=Config)
File "/venv/lib/python3.7/site-packages/boto3/s3/inject.py", line 539, in upload_fileobj
return future.result()
File "/venv/lib/python3.7/site-packages/s3transfer/futures.py", line 106, in result
return self._coordinator.result()
File "/venv/lib/python3.7/site-packages/s3transfer/futures.py", line 265, in result
raise self._exception
File "/venv/lib/python3.7/site-packages/s3transfer/tasks.py", line 126, in __call__
return self._execute_main(kwargs)
File "/venv/lib/python3.7/site-packages/s3transfer/tasks.py", line 150, in _execute_main
return_value = self._main(**kwargs)
File "/venv/lib/python3.7/site-packages/s3transfer/tasks.py", line 333, in _main
Bucket=bucket, Key=key, **extra_args)
File "/venv/lib/python3.7/site-packages/botocore/client.py", line 316, in _api_call
return self._make_api_call(operation_name, kwargs)
File "/venv/lib/python3.7/site-packages/botocore/client.py", line 635, in _make_api_call
raise error_class(parsed_response, operation_name)