Skip to content

funkwhale
funkwhale
Commit f49b3a89 authored by Eliot Berriot's avatar Eliot Berriot 💬
Browse files
Options

See #463: better import error handling

parent 91a47e7a
Hide whitespace changes
Inline Side-by-side
Showing with 138 additions and 29 deletions
api/funkwhale_api/federation/models.py
View file @ f49b3a89
......@@ -42,7 +42,7 @@ class ActorQuerySet(models.QuerySet):
def with_current_usage(self):
qs = self
for s in ["pending", "skipped", "error", "finished"]:
for s in ["pending", "skipped", "errored", "finished"]:
qs = qs.annotate(
**{
"_usage_{}".format(s): models.Sum(
......@@ -150,7 +150,7 @@ class Actor(models.Model):
def get_current_usage(self):
actor = self.__class__.objects.filter(pk=self.pk).with_current_usage().get()
data = {}
for s in ["pending", "skipped", "error", "finished"]:
for s in ["pending", "skipped", "errored", "finished"]:
data[s] = getattr(actor, "_usage_{}".format(s)) or 0
data["total"] = sum(data.values())
......
api/funkwhale_api/music/tasks.py
View file @ f49b3a89
......@@ -440,7 +440,9 @@ def import_track_file(track_file):
track = import_track_from_remote(track_file.metadata)
except TrackFileImportError as e:
return fail_import(track_file, e.code)
except Exception:
fail_import(track_file, "unknown_error")
raise
# under some situations, we want to skip the import (
# for instance if the user already owns the files)
owned_duplicates = get_owned_duplicates(track_file, track)
......
api/funkwhale_api/taskapp/celery.py
View file @ f49b3a89
......@@ -2,20 +2,29 @@
from __future__ import absolute_import
import functools
import traceback as tb
import os
from celery import Celery
import logging
import celery.app.task
from django.apps import AppConfig
from django.conf import settings
logger = logging.getLogger("celery")
if not settings.configured:
# set the default Django settings module for the 'celery' program.
os.environ.setdefault(
"DJANGO_SETTINGS_MODULE", "config.settings.local"
) # pragma: no cover
app = celery.Celery("funkwhale_api")
app = Celery("funkwhale_api")
@celery.signals.task_failure.connect
def process_failure(sender, task_id, exception, args, kwargs, traceback, einfo, **kw):
print("[celery] Error during task {}: {}".format(task_id, einfo.exception))
tb.print_exc()
class CeleryConfig(AppConfig):
......
api/funkwhale_api/users/models.py
View file @ f49b3a89
......@@ -202,7 +202,7 @@ class User(AbstractUser):
"skipped": data["skipped"] / 1000 / 1000,
"pending": data["pending"] / 1000 / 1000,
"finished": data["finished"] / 1000 / 1000,
"error": data["error"] / 1000 / 1000,
"errored": data["errored"] / 1000 / 1000,
}
def get_channels_groups(self):
......
api/tests/federation/test_models.py
View file @ f49b3a89
......@@ -41,7 +41,7 @@ def test_actor_get_quota(factories):
)
factories["music.TrackFile"](
library=library,
import_status="error",
import_status="errored",
audio_file__from_path=None,
audio_file__data=b"aaa",
)
......@@ -51,6 +51,6 @@ def test_actor_get_quota(factories):
audio_file__from_path=None,
audio_file__data=b"aaaa",
)
expected = {"total": 10, "pending": 1, "skipped": 2, "error": 3, "finished": 4}
expected = {"total": 10, "pending": 1, "skipped": 2, "errored": 3, "finished": 4}
assert library.actor.get_current_usage() == expected
front/src/components/library/FileUpload.vue
View file @ f49b3a89
<template>
<template>
<div>
<div class="ui hidden clearing divider"></div>
<!-- <div v-if="files.length > 0" class="ui indicating progress">
......@@ -8,17 +8,27 @@
{{ processedFilesCount }}/{{ processableFiles }} files processed
</div>
</div> -->
<div class="ui form">
<div class="fields">
<div class="ui four wide field">
<label><translate>Import reference</translate></label>
<input type="text" v-model="importReference" />
</div>
</div>
</div>
<p><translate>This reference will be used to group imported files together.</translate></p>
<div class="ui top attached tabular menu">
<a :class="['item', {active: currentTab === 'uploads'}]" @click="currentTab = 'uploads'">
<translate>Uploading</translate>
<div v-if="files.length === 0" class="ui label">
0
</div>
<div v-else-if="files.length > uploadedFilesCount + errorFilesCount" class="ui yellow label">
{{ uploadedFilesCount + errorFilesCount }}/{{ files.length }}
<div v-else-if="files.length > uploadedFilesCount + erroredFilesCount" class="ui yellow label">
{{ uploadedFilesCount + erroredFilesCount }}/{{ files.length }}
</div>
<div v-else :class="['ui', {'green': errorFilesCount === 0}, {'red': errorFilesCount > 0}, 'label']">
{{ uploadedFilesCount + errorFilesCount }}/{{ files.length }}
<div v-else :class="['ui', {'green': erroredFilesCount === 0}, {'red': erroredFilesCount > 0}, 'label']">
{{ uploadedFilesCount + erroredFilesCount }}/{{ files.length }}
</div>
</a>
<a :class="['item', {active: currentTab === 'processing'}]" @click="currentTab = 'processing'">
......@@ -29,7 +39,7 @@
<div v-else-if="processableFiles > processedFilesCount" class="ui yellow label">
{{ processedFilesCount }}/{{ processableFiles }}
</div>
<div v-else :class="['ui', {'green': trackFiles.error === 0}, {'red': trackFiles > 0}, 'label']">
<div v-else :class="['ui', {'green': trackFiles.errored === 0}, {'red': trackFiles.errored > 0}, 'label']">
{{ processedFilesCount }}/{{ processableFiles }}
</div>
</a>
......@@ -119,15 +129,12 @@ export default {
files: [],
currentTab: 'uploads',
uploadUrl: '/api/v1/track-files/',
batch: null,
interval: null,
batchStats: null,
importReference,
trackFiles: {
pending: 0,
finished: 0,
skipped: 0,
error: 0,
errored: 0,
objects: {},
},
bridge: null,
......@@ -156,7 +163,7 @@ export default {
},
fetchStatus () {
let self = this
let statuses = ['pending', 'error', 'skipped', 'finished']
let statuses = ['pending', 'errored', 'skipped', 'finished']
statuses.forEach((status) => {
axios.get('track-files/', {params: {import_reference: self.importReference, import_status: status, page_size: 1}}).then((response) => {
self.trackFiles[status] = response.data.count
......@@ -213,7 +220,7 @@ export default {
},
computed: {
labels () {
let denied = this.$gettext('Upload refused, ensure you have not reached your quota')
let denied = this.$gettext('Upload refused, ensure the file is not too big and you have not reached your quota')
let server = this.$gettext('Impossible to upload this file, ensure it is not too big')
let network = this.$gettext('A network error occured while uploading this file')
let timeout = this.$gettext('Upload timeout, please try again')
......@@ -236,16 +243,16 @@ export default {
return !f.success && !f.error
}).length
},
errorFilesCount () {
erroredFilesCount () {
return this.files.filter((f) => {
return f.error
}).length
},
processableFiles () {
return this.trackFiles.pending + this.trackFiles.skipped + this.trackFiles.error + this.trackFiles.finished + this.uploadedFilesCount
return this.trackFiles.pending + this.trackFiles.skipped + this.trackFiles.errored + this.trackFiles.finished + this.uploadedFilesCount
},
processedFilesCount () {
return this.trackFiles.skipped + this.trackFiles.error + this.trackFiles.finished
return this.trackFiles.skipped + this.trackFiles.errored + this.trackFiles.finished
},
uploadData: function () {
return {
......@@ -256,7 +263,7 @@ export default {
sortedFiles () {
// return errored files on top
return this.files.sort((f) => {
if (f.error) {
if (f.errored) {
return -5
}
if (f.success) {
......@@ -272,7 +279,10 @@ export default {
},
finishedJobs () {
this.updateProgressBar()
}
},
importReference: _.debounce(function () {
this.$router.replace({query: {import: this.importReference}})
}, 500)
}
}
</script>
......
front/src/views/libraries/FilesTable.vue
View file @ f49b3a89
......@@ -12,7 +12,7 @@
<option :value="null"><translate>All</translate></option>
<option :value="'pending'"><translate>Pending</translate></option>
<option :value="'skipped'"><translate>Skipped</translate></option>
<option :value="'error'"><translate>Error</translate></option>
<option :value="'errored'"><translate>Errored</translate></option>
<option :value="'finished'"><translate>Finished</translate></option>
</select>
</div>
......@@ -206,8 +206,8 @@ export default {
label: this.$gettext('Pending'),
help: this.$gettext('Track is uploaded but not processed by the server yet'),
},
error: {
label: this.$gettext('Error'),
errored: {
label: this.$gettext('Errored'),
help: this.$gettext('An error occured while processing this track, ensure the track is correctly tagged'),
},
finished: {
......
front/src/views/libraries/Quota.vue
View file @ f49b3a89
......@@ -12,6 +12,72 @@
<translate :translate-params="{max: humanSize(quotaStatus.max * 1000 * 1000), current: humanSize(quotaStatus.current * 1000 * 1000)}">%{ current } used on %{ max } allowed</translate>
</div>
</div>
<div class="ui hidden divider"></div>
<div v-if="quotaStatus" class="ui stackable three column grid">
<div v-if="quotaStatus.pending > 0" class="column">
<div class="ui tiny yellow statistic">
<div class="value">
{{ humanSize(quotaStatus.pending * 1000 * 1000) }}
</div>
<div class="label">
<translate>Pending files</translate>
</div>
</div>
<div>
<dangerous-button
color="grey"
class="basic tiny"
:action="purgePendingFiles">
<translate>Purge</translate>
<p slot="modal-header"><translate>Purge pending files?</translate></p>
<p slot="modal-content"><translate>This will remove tracks that were uploaded but not processed yet. This will remove those files completely and you will regain the corresponding quota.</translate></p>
<p slot="modal-confirm"><translate>Purge</translate></p>
</dangerous-button>
</div>
</div>
<div v-if="quotaStatus.skipped > 0" class="column">
<div class="ui tiny grey statistic">
<div class="value">
{{ humanSize(quotaStatus.skipped * 1000 * 1000) }}
</div>
<div class="label">
<translate>Skipped files</translate>
</div>
</div>
<div>
<dangerous-button
color="grey"
class="basic tiny"
:action="purgeSkippedFiles">
<translate>Purge</translate>
<p slot="modal-header"><translate>Purge skipped files?</translate></p>
<p slot="modal-content"><translate>This will remove tracks that were uploaded but skipped during import processes for various reasons. This will remove those files completely and you will regain the corresponding quota.</translate></p>
<p slot="modal-confirm"><translate>Purge</translate></p>
</dangerous-button>
</div>
</div>
<div v-if="quotaStatus.errored > 0" class="column">
<div class="ui tiny red statistic">
<div class="value">
{{ humanSize(quotaStatus.errored * 1000 * 1000) }}
</div>
<div class="label">
<translate>Errored files</translate>
</div>
</div>
<div>
<dangerous-button
color="grey"
class="basic tiny"
:action="purgeErroredFiles">
<translate>Purge</translate>
<p slot="modal-header"><translate>Purge errored files?</translate></p>
<p slot="modal-content"><translate>This will remove tracks that were uploaded but failed to be process by the server. This will remove those files completely and you will regain the corresponding quota.</translate></p>
<p slot="modal-confirm"><translate>Purge</translate></p>
</dangerous-button>
</div>
</div>
</div>
</div>
</template>
<script>
......@@ -39,6 +105,28 @@ export default {
self.isLoading = false
})
},
purge (status) {
let self = this
let payload = {
action: 'delete',
objects: 'all',
filters: {
import_status: status
}
}
axios.post('track-files/action/', payload).then((response) => {
self.fetch()
})
},
purgeSkippedFiles () {
this.purge('skipped')
},
purgePendingFiles () {
this.purge('pending')
},
purgeErroredFiles () {
this.purge('errored')
},
updateProgressBar () {
$(this.$el).find('.ui.progress').progress({
percent: this.progress,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment