Newer
Older
Eliot Berriot
committed
import glob
from django.core.files import File
Eliot Berriot
committed
from django.core.management.base import BaseCommand, CommandError
from funkwhale_api.users.models import User
Eliot Berriot
committed
class Command(BaseCommand):
Eliot Berriot
committed
def add_arguments(self, parser):
parser.add_argument("path", nargs="+", type=str)
Eliot Berriot
committed
parser.add_argument(
"--recursive",
action="store_true",
dest="recursive",
Eliot Berriot
committed
default=False,
help="Will match the pattern recursively (including subdirectories)",
Eliot Berriot
committed
)
parser.add_argument(
"--username",
dest="username",
help="The username of the user you want to be bound to the import",
Eliot Berriot
committed
parser.add_argument(
Eliot Berriot
committed
default=False,
help="Will launch celery tasks for each file to import instead of doing it synchronously and block the CLI",
Eliot Berriot
committed
)
"--exit",
"-x",
action="store_true",
dest="exit_on_failure",
"--in-place",
"-i",
action="store_true",
dest="in_place",
"Import files without duplicating them into the media directory."
"For in-place import to work, the music files must be readable"
"by the web-server and funkwhale api and celeryworker processes."
"You may want to use this if you have a big music library to "
"import and not much disk space available."
),
parser.add_argument(
"--update",
action="store_true",
dest="update",
default=False,
help=(
"Use this flag to replace duplicates (tracks with same "
"musicbrainz mbid) on import with their newest version."
),
)
Eliot Berriot
committed
parser.add_argument(
"--noinput",
"--no-input",
action="store_false",
dest="interactive",
Eliot Berriot
committed
help="Do NOT prompt the user for input of any kind.",
)
def handle(self, *args, **options):
glob_kwargs = {}
if options["recursive"]:
glob_kwargs["recursive"] = True
for import_path in options["path"]:
matching += glob.glob(import_path, **glob_kwargs)
matching = sorted(list(set(matching)))
except TypeError:
raise Exception("You need Python 3.5 to use the --recursive flag")
"Checking imported paths against settings.MUSIC_DIRECTORY_PATH"
)
p = settings.MUSIC_DIRECTORY_PATH
if not p:
raise CommandError(
"Importing in-place requires setting the "
"MUSIC_DIRECTORY_PATH variable"
)
for m in matching:
if not m.startswith(p):
raise CommandError(
"Importing in-place only works if importing"
"from {} (MUSIC_DIRECTORY_PATH), as this directory"
"needs to be accessible by the webserver."
"Culprit: {}".format(p, m)
)
Eliot Berriot
committed
if not matching:
raise CommandError("No file matching pattern, aborting")
Eliot Berriot
committed
except User.DoesNotExist:
else:
# we bind the import to the first registered superuser
try:
user = User.objects.filter(is_superuser=True).order_by("pk").first()
assert user is not None
except AssertionError:
raise CommandError(
"No superuser available, please provide a --username"
)
filtered = self.filter_matching(matching)
self.stdout.write("Import summary:")
self.stdout.write(
"- {} files found matching this pattern: {}".format(
len(matching), options["path"]
)
)
self.stdout.write(
"- {} files already found in database".format(len(filtered["skipped"]))
)
self.stdout.write("- {} new files".format(len(filtered["new"])))
self.stdout.write(
"Selected options: {}".format(
", ".join(["in place" if options["in_place"] else "copy music files"])
)
)
if len(filtered["new"]) == 0:
self.stdout.write("Nothing new to import, exiting")
return
Eliot Berriot
committed
message = (
Eliot Berriot
committed
"Type 'yes' to continue, or 'no' to cancel: "
)
Eliot Berriot
committed
raise CommandError("Import cancelled.")
batch, errors = self.do_import(filtered["new"], user=user, options=options)
message = "Successfully imported {} tracks"
if options["async"]:
message = "Successfully launched import for {} tracks"
self.stdout.write(message.format(len(filtered["new"])))
if len(errors) > 0:
self.stderr.write("{} tracks could not be imported:".format(len(errors)))
for path, error in errors:
self.stdout.write(
"For details, please refer to import batch #{}".format(batch.pk)
)
def filter_matching(self, matching):
# we skip reimport for path that are already found
# as a TrackFile.source
existing = models.TrackFile.objects.filter(source__in=sources)
existing = existing.values_list("source", flat=True)
existing = set([p.replace("file://", "", 1) for p in existing])
skipped = set(matching) & existing
result = {
"initial": matching,
"skipped": list(sorted(skipped)),
"new": list(sorted(set(matching) - skipped)),
}
return result
def do_import(self, paths, user, options):
message = "{i}/{total} Importing {path}..."
if options["async"]:
message = "{i}/{total} Launching import for {path}..."
Eliot Berriot
committed
# we create an import batch binded to the user
import_handler = tasks.import_job_run.delay if async else tasks.import_job_run
errors = []
for i, path in list(enumerate(paths)):
Eliot Berriot
committed
try:
self.stdout.write(message.format(path=path, i=i + 1, total=len(paths)))
self.import_file(path, batch, import_handler, options)
Eliot Berriot
committed
except Exception as e:
m = "Error while importing {}: {} {}".format(
path, e.__class__.__name__, e
)
errors.append((path, "{} {}".format(e.__class__.__name__, e)))
return batch, errors
def import_file(self, path, batch, import_handler, options):
job = batch.jobs.create(source="file://" + path)
if not options["in_place"]:
job.audio_file.save(name, File(f))
import_handler(import_job_id=job.pk, use_acoustid=False)