import_files.py 8.3 KB
Newer Older
1
import glob
2
3
import os

4
from django.conf import settings
5
from django.core.files import File
6
from django.core.management.base import BaseCommand, CommandError
7

Eliot Berriot's avatar
Eliot Berriot committed
8
from funkwhale_api.music import models, tasks
9
from funkwhale_api.users.models import User
10
11
12


class Command(BaseCommand):
Eliot Berriot's avatar
Eliot Berriot committed
13
    help = "Import audio files mathinc given glob pattern"
14
15

    def add_arguments(self, parser):
16
        parser.add_argument("path", nargs="+", type=str)
17
        parser.add_argument(
Eliot Berriot's avatar
Eliot Berriot committed
18
19
20
            "--recursive",
            action="store_true",
            dest="recursive",
21
            default=False,
Eliot Berriot's avatar
Eliot Berriot committed
22
            help="Will match the pattern recursively (including subdirectories)",
23
        )
24
        parser.add_argument(
Eliot Berriot's avatar
Eliot Berriot committed
25
26
27
            "--username",
            dest="username",
            help="The username of the user you want to be bound to the import",
28
        )
29
        parser.add_argument(
Eliot Berriot's avatar
Eliot Berriot committed
30
31
32
            "--async",
            action="store_true",
            dest="async",
33
            default=False,
Eliot Berriot's avatar
Eliot Berriot committed
34
            help="Will launch celery tasks for each file to import instead of doing it synchronously and block the CLI",
35
        )
36
        parser.add_argument(
Eliot Berriot's avatar
Eliot Berriot committed
37
38
39
40
            "--exit",
            "-x",
            action="store_true",
            dest="exit_on_failure",
41
            default=False,
Eliot Berriot's avatar
Eliot Berriot committed
42
            help="Use this flag to disable error catching",
43
44
        )
        parser.add_argument(
Eliot Berriot's avatar
Eliot Berriot committed
45
46
47
48
            "--in-place",
            "-i",
            action="store_true",
            dest="in_place",
49
50
            default=False,
            help=(
Eliot Berriot's avatar
Eliot Berriot committed
51
52
53
54
55
56
                "Import files without duplicating them into the media directory."
                "For in-place import to work, the music files must be readable"
                "by the web-server and funkwhale api and celeryworker processes."
                "You may want to use this if you have a big music library to "
                "import and not much disk space available."
            ),
57
        )
RenonDis's avatar
RenonDis committed
58
        parser.add_argument(
59
            "--replace",
RenonDis's avatar
RenonDis committed
60
            action="store_true",
61
            dest="replace",
RenonDis's avatar
RenonDis committed
62
63
64
            default=False,
            help=(
                "Use this flag to replace duplicates (tracks with same "
65
66
                "musicbrainz mbid, or same artist, album and title) on import "
                "with their newest version."
RenonDis's avatar
RenonDis committed
67
68
            ),
        )
69
        parser.add_argument(
Eliot Berriot's avatar
Eliot Berriot committed
70
71
72
73
            "--noinput",
            "--no-input",
            action="store_false",
            dest="interactive",
74
75
76
77
            help="Do NOT prompt the user for input of any kind.",
        )

    def handle(self, *args, **options):
78
        glob_kwargs = {}
79
        matching = []
Eliot Berriot's avatar
Eliot Berriot committed
80
81
        if options["recursive"]:
            glob_kwargs["recursive"] = True
82
        try:
83
84
85
            for import_path in options["path"]:
                matching += glob.glob(import_path, **glob_kwargs)
            matching = sorted(list(set(matching)))
86
        except TypeError:
Eliot Berriot's avatar
Eliot Berriot committed
87
            raise Exception("You need Python 3.5 to use the --recursive flag")
88

Eliot Berriot's avatar
Eliot Berriot committed
89
        if options["in_place"]:
90
            self.stdout.write(
Eliot Berriot's avatar
Eliot Berriot committed
91
92
                "Checking imported paths against settings.MUSIC_DIRECTORY_PATH"
            )
93
94
95
            p = settings.MUSIC_DIRECTORY_PATH
            if not p:
                raise CommandError(
Eliot Berriot's avatar
Eliot Berriot committed
96
97
98
                    "Importing in-place requires setting the "
                    "MUSIC_DIRECTORY_PATH variable"
                )
99
100
101
            for m in matching:
                if not m.startswith(p):
                    raise CommandError(
Eliot Berriot's avatar
Eliot Berriot committed
102
103
104
105
106
                        "Importing in-place only works if importing"
                        "from {} (MUSIC_DIRECTORY_PATH), as this directory"
                        "needs to be accessible by the webserver."
                        "Culprit: {}".format(p, m)
                    )
107
        if not matching:
Eliot Berriot's avatar
Eliot Berriot committed
108
            raise CommandError("No file matching pattern, aborting")
109

110
        user = None
Eliot Berriot's avatar
Eliot Berriot committed
111
        if options["username"]:
112
            try:
Eliot Berriot's avatar
Eliot Berriot committed
113
                user = User.objects.get(username=options["username"])
114
            except User.DoesNotExist:
Eliot Berriot's avatar
Eliot Berriot committed
115
                raise CommandError("Invalid username")
116
117
118
        else:
            # we bind the import to the first registered superuser
            try:
Eliot Berriot's avatar
Eliot Berriot committed
119
                user = User.objects.filter(is_superuser=True).order_by("pk").first()
120
121
122
                assert user is not None
            except AssertionError:
                raise CommandError(
Eliot Berriot's avatar
Eliot Berriot committed
123
124
                    "No superuser available, please provide a --username"
                )
125

126
127
128
129
130
131
132
133
134
        if options["replace"]:
            filtered = {"initial": matching, "skipped": [], "new": matching}
            message = "- {} files to be replaced"
            import_paths = matching
        else:
            filtered = self.filter_matching(matching)
            message = "- {} files already found in database"
            import_paths = filtered["new"]

Eliot Berriot's avatar
Eliot Berriot committed
135
136
137
138
139
140
        self.stdout.write("Import summary:")
        self.stdout.write(
            "- {} files found matching this pattern: {}".format(
                len(matching), options["path"]
            )
        )
141
142
        self.stdout.write(message.format(len(filtered["skipped"])))

Eliot Berriot's avatar
Eliot Berriot committed
143
144
145
146
147
148
149
150
151
        self.stdout.write("- {} new files".format(len(filtered["new"])))

        self.stdout.write(
            "Selected options: {}".format(
                ", ".join(["in place" if options["in_place"] else "copy music files"])
            )
        )
        if len(filtered["new"]) == 0:
            self.stdout.write("Nothing new to import, exiting")
152
153
            return

Eliot Berriot's avatar
Eliot Berriot committed
154
        if options["interactive"]:
155
            message = (
Eliot Berriot's avatar
Eliot Berriot committed
156
                "Are you sure you want to do this?\n\n"
157
158
                "Type 'yes' to continue, or 'no' to cancel: "
            )
Eliot Berriot's avatar
Eliot Berriot committed
159
            if input("".join(message)) != "yes":
160
161
                raise CommandError("Import cancelled.")

162
        batch, errors = self.do_import(import_paths, user=user, options=options)
Eliot Berriot's avatar
Eliot Berriot committed
163
164
165
        message = "Successfully imported {} tracks"
        if options["async"]:
            message = "Successfully launched import for {} tracks"
166

167
        self.stdout.write(message.format(len(import_paths)))
168
        if len(errors) > 0:
Eliot Berriot's avatar
Eliot Berriot committed
169
            self.stderr.write("{} tracks could not be imported:".format(len(errors)))
170
171

            for path, error in errors:
Eliot Berriot's avatar
Eliot Berriot committed
172
                self.stderr.write("- {}: {}".format(path, error))
173
        self.stdout.write(
Eliot Berriot's avatar
Eliot Berriot committed
174
175
            "For details, please refer to import batch #{}".format(batch.pk)
        )
176

177
    def filter_matching(self, matching):
Eliot Berriot's avatar
Eliot Berriot committed
178
        sources = ["file://{}".format(p) for p in matching]
179
180
181
        # we skip reimport for path that are already found
        # as a TrackFile.source
        existing = models.TrackFile.objects.filter(source__in=sources)
Eliot Berriot's avatar
Eliot Berriot committed
182
183
        existing = existing.values_list("source", flat=True)
        existing = set([p.replace("file://", "", 1) for p in existing])
184
185
        skipped = set(matching) & existing
        result = {
Eliot Berriot's avatar
Eliot Berriot committed
186
187
188
            "initial": matching,
            "skipped": list(sorted(skipped)),
            "new": list(sorted(set(matching) - skipped)),
189
190
191
192
        }
        return result

    def do_import(self, paths, user, options):
Eliot Berriot's avatar
Eliot Berriot committed
193
194
195
        message = "{i}/{total} Importing {path}..."
        if options["async"]:
            message = "{i}/{total} Launching import for {path}..."
196

197
        # we create an import batch binded to the user
Eliot Berriot's avatar
Eliot Berriot committed
198
        async = options["async"]
199
        import_handler = tasks.import_job_run.delay if async else tasks.import_job_run
Eliot Berriot's avatar
Eliot Berriot committed
200
        batch = user.imports.create(source="shell")
201
        errors = []
202
        for i, path in list(enumerate(paths)):
203
            try:
Eliot Berriot's avatar
Eliot Berriot committed
204
                self.stdout.write(message.format(path=path, i=i + 1, total=len(paths)))
205
                self.import_file(path, batch, import_handler, options)
206
            except Exception as e:
Eliot Berriot's avatar
Eliot Berriot committed
207
                if options["exit_on_failure"]:
208
                    raise
Eliot Berriot's avatar
Eliot Berriot committed
209
210
211
                m = "Error while importing {}: {} {}".format(
                    path, e.__class__.__name__, e
                )
212
                self.stderr.write(m)
Eliot Berriot's avatar
Eliot Berriot committed
213
                errors.append((path, "{} {}".format(e.__class__.__name__, e)))
214
        return batch, errors
215
216

    def import_file(self, path, batch, import_handler, options):
Eliot Berriot's avatar
Eliot Berriot committed
217
218
        job = batch.jobs.create(source="file://" + path)
        if not options["in_place"]:
219
            name = os.path.basename(path)
Eliot Berriot's avatar
Eliot Berriot committed
220
            with open(path, "rb") as f:
221
                job.audio_file.save(name, File(f))
222

223
            job.save()
Eliot Berriot's avatar
Eliot Berriot committed
224
        import_handler(import_job_id=job.pk, use_acoustid=False)