import_files.py 9.18 KB
Newer Older
1
import glob
2
3
import os

4
from django.conf import settings
5
from django.core.files import File
6
from django.core.management.base import BaseCommand, CommandError
7

Eliot Berriot's avatar
Eliot Berriot committed
8
from funkwhale_api.music import models, tasks
9
from funkwhale_api.users.models import User
10
11
12


class Command(BaseCommand):
Eliot Berriot's avatar
Eliot Berriot committed
13
    help = "Import audio files mathinc given glob pattern"
14
15

    def add_arguments(self, parser):
16
        parser.add_argument("path", nargs="+", type=str)
17
        parser.add_argument(
Eliot Berriot's avatar
Eliot Berriot committed
18
19
20
            "--recursive",
            action="store_true",
            dest="recursive",
21
            default=False,
Eliot Berriot's avatar
Eliot Berriot committed
22
            help="Will match the pattern recursively (including subdirectories)",
23
        )
24
        parser.add_argument(
Eliot Berriot's avatar
Eliot Berriot committed
25
26
27
            "--username",
            dest="username",
            help="The username of the user you want to be bound to the import",
28
        )
29
        parser.add_argument(
Eliot Berriot's avatar
Eliot Berriot committed
30
31
32
            "--async",
            action="store_true",
            dest="async",
33
            default=False,
Eliot Berriot's avatar
Eliot Berriot committed
34
            help="Will launch celery tasks for each file to import instead of doing it synchronously and block the CLI",
35
        )
36
        parser.add_argument(
Eliot Berriot's avatar
Eliot Berriot committed
37
38
39
40
            "--exit",
            "-x",
            action="store_true",
            dest="exit_on_failure",
41
            default=False,
Eliot Berriot's avatar
Eliot Berriot committed
42
            help="Use this flag to disable error catching",
43
44
        )
        parser.add_argument(
Eliot Berriot's avatar
Eliot Berriot committed
45
46
47
48
            "--in-place",
            "-i",
            action="store_true",
            dest="in_place",
49
50
            default=False,
            help=(
Eliot Berriot's avatar
Eliot Berriot committed
51
52
53
54
55
56
                "Import files without duplicating them into the media directory."
                "For in-place import to work, the music files must be readable"
                "by the web-server and funkwhale api and celeryworker processes."
                "You may want to use this if you have a big music library to "
                "import and not much disk space available."
            ),
57
        )
RenonDis's avatar
RenonDis committed
58
        parser.add_argument(
59
            "--replace",
RenonDis's avatar
RenonDis committed
60
            action="store_true",
61
            dest="replace",
RenonDis's avatar
RenonDis committed
62
63
64
            default=False,
            help=(
                "Use this flag to replace duplicates (tracks with same "
65
66
                "musicbrainz mbid, or same artist, album and title) on import "
                "with their newest version."
RenonDis's avatar
RenonDis committed
67
68
            ),
        )
69
        parser.add_argument(
Eliot Berriot's avatar
Eliot Berriot committed
70
71
72
73
            "--noinput",
            "--no-input",
            action="store_false",
            dest="interactive",
74
75
76
77
            help="Do NOT prompt the user for input of any kind.",
        )

    def handle(self, *args, **options):
78
        glob_kwargs = {}
79
        matching = []
Eliot Berriot's avatar
Eliot Berriot committed
80
81
        if options["recursive"]:
            glob_kwargs["recursive"] = True
82
        try:
83
84
            for import_path in options["path"]:
                matching += glob.glob(import_path, **glob_kwargs)
85
            raw_matching = sorted(list(set(matching)))
86
        except TypeError:
Eliot Berriot's avatar
Eliot Berriot committed
87
            raise Exception("You need Python 3.5 to use the --recursive flag")
88

89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
        matching = []
        for m in raw_matching:
            # In some situations, the path is encoded incorrectly on the filesystem
            # so we filter out faulty paths and display a warning to the user.
            # see https://code.eliotberriot.com/funkwhale/funkwhale/issues/138
            try:
                m.encode("utf-8")
                matching.append(m)
            except UnicodeEncodeError:
                try:
                    previous = matching[-1]
                except IndexError:
                    previous = None
                self.stderr.write(
                    self.style.WARNING(
                        "[warning] Ignoring undecodable path. Previous ok file was {}".format(
                            previous
                        )
                    )
                )

Eliot Berriot's avatar
Eliot Berriot committed
110
        if options["in_place"]:
111
            self.stdout.write(
Eliot Berriot's avatar
Eliot Berriot committed
112
113
                "Checking imported paths against settings.MUSIC_DIRECTORY_PATH"
            )
114
115
116
            p = settings.MUSIC_DIRECTORY_PATH
            if not p:
                raise CommandError(
Eliot Berriot's avatar
Eliot Berriot committed
117
118
119
                    "Importing in-place requires setting the "
                    "MUSIC_DIRECTORY_PATH variable"
                )
120
121
122
            for m in matching:
                if not m.startswith(p):
                    raise CommandError(
Eliot Berriot's avatar
Eliot Berriot committed
123
124
125
126
127
                        "Importing in-place only works if importing"
                        "from {} (MUSIC_DIRECTORY_PATH), as this directory"
                        "needs to be accessible by the webserver."
                        "Culprit: {}".format(p, m)
                    )
128
        if not matching:
Eliot Berriot's avatar
Eliot Berriot committed
129
            raise CommandError("No file matching pattern, aborting")
130

131
        user = None
Eliot Berriot's avatar
Eliot Berriot committed
132
        if options["username"]:
133
            try:
Eliot Berriot's avatar
Eliot Berriot committed
134
                user = User.objects.get(username=options["username"])
135
            except User.DoesNotExist:
Eliot Berriot's avatar
Eliot Berriot committed
136
                raise CommandError("Invalid username")
137
138
139
        else:
            # we bind the import to the first registered superuser
            try:
Eliot Berriot's avatar
Eliot Berriot committed
140
                user = User.objects.filter(is_superuser=True).order_by("pk").first()
141
142
143
                assert user is not None
            except AssertionError:
                raise CommandError(
Eliot Berriot's avatar
Eliot Berriot committed
144
145
                    "No superuser available, please provide a --username"
                )
146

147
148
149
150
151
152
153
154
155
        if options["replace"]:
            filtered = {"initial": matching, "skipped": [], "new": matching}
            message = "- {} files to be replaced"
            import_paths = matching
        else:
            filtered = self.filter_matching(matching)
            message = "- {} files already found in database"
            import_paths = filtered["new"]

Eliot Berriot's avatar
Eliot Berriot committed
156
157
158
159
160
161
        self.stdout.write("Import summary:")
        self.stdout.write(
            "- {} files found matching this pattern: {}".format(
                len(matching), options["path"]
            )
        )
162
163
        self.stdout.write(message.format(len(filtered["skipped"])))

Eliot Berriot's avatar
Eliot Berriot committed
164
165
166
167
168
169
170
171
172
        self.stdout.write("- {} new files".format(len(filtered["new"])))

        self.stdout.write(
            "Selected options: {}".format(
                ", ".join(["in place" if options["in_place"] else "copy music files"])
            )
        )
        if len(filtered["new"]) == 0:
            self.stdout.write("Nothing new to import, exiting")
173
174
            return

Eliot Berriot's avatar
Eliot Berriot committed
175
        if options["interactive"]:
176
            message = (
Eliot Berriot's avatar
Eliot Berriot committed
177
                "Are you sure you want to do this?\n\n"
178
179
                "Type 'yes' to continue, or 'no' to cancel: "
            )
Eliot Berriot's avatar
Eliot Berriot committed
180
            if input("".join(message)) != "yes":
181
182
                raise CommandError("Import cancelled.")

183
        batch, errors = self.do_import(import_paths, user=user, options=options)
Eliot Berriot's avatar
Eliot Berriot committed
184
185
186
        message = "Successfully imported {} tracks"
        if options["async"]:
            message = "Successfully launched import for {} tracks"
187

188
        self.stdout.write(message.format(len(import_paths)))
189
        if len(errors) > 0:
Eliot Berriot's avatar
Eliot Berriot committed
190
            self.stderr.write("{} tracks could not be imported:".format(len(errors)))
191
192

            for path, error in errors:
Eliot Berriot's avatar
Eliot Berriot committed
193
                self.stderr.write("- {}: {}".format(path, error))
194
        self.stdout.write(
Eliot Berriot's avatar
Eliot Berriot committed
195
196
            "For details, please refer to import batch #{}".format(batch.pk)
        )
197

198
    def filter_matching(self, matching):
Eliot Berriot's avatar
Eliot Berriot committed
199
        sources = ["file://{}".format(p) for p in matching]
200
201
202
        # we skip reimport for path that are already found
        # as a TrackFile.source
        existing = models.TrackFile.objects.filter(source__in=sources)
Eliot Berriot's avatar
Eliot Berriot committed
203
204
        existing = existing.values_list("source", flat=True)
        existing = set([p.replace("file://", "", 1) for p in existing])
205
206
        skipped = set(matching) & existing
        result = {
Eliot Berriot's avatar
Eliot Berriot committed
207
208
209
            "initial": matching,
            "skipped": list(sorted(skipped)),
            "new": list(sorted(set(matching) - skipped)),
210
211
212
213
        }
        return result

    def do_import(self, paths, user, options):
Eliot Berriot's avatar
Eliot Berriot committed
214
215
216
        message = "{i}/{total} Importing {path}..."
        if options["async"]:
            message = "{i}/{total} Launching import for {path}..."
217

218
        # we create an import batch binded to the user
219
220
        async_ = options["async"]
        import_handler = tasks.import_job_run.delay if async_ else tasks.import_job_run
Eliot Berriot's avatar
Eliot Berriot committed
221
        batch = user.imports.create(source="shell")
222
        errors = []
223
        for i, path in list(enumerate(paths)):
224
            try:
Eliot Berriot's avatar
Eliot Berriot committed
225
                self.stdout.write(message.format(path=path, i=i + 1, total=len(paths)))
226
                self.import_file(path, batch, import_handler, options)
227
            except Exception as e:
Eliot Berriot's avatar
Eliot Berriot committed
228
                if options["exit_on_failure"]:
229
                    raise
Eliot Berriot's avatar
Eliot Berriot committed
230
231
232
                m = "Error while importing {}: {} {}".format(
                    path, e.__class__.__name__, e
                )
233
                self.stderr.write(m)
Eliot Berriot's avatar
Eliot Berriot committed
234
                errors.append((path, "{} {}".format(e.__class__.__name__, e)))
235
        return batch, errors
236
237

    def import_file(self, path, batch, import_handler, options):
RenonDis's avatar
RenonDis committed
238
239
240
        job = batch.jobs.create(
            source="file://" + path, replace_if_duplicate=options["replace"]
        )
Eliot Berriot's avatar
Eliot Berriot committed
241
        if not options["in_place"]:
242
            name = os.path.basename(path)
Eliot Berriot's avatar
Eliot Berriot committed
243
            with open(path, "rb") as f:
244
                job.audio_file.save(name, File(f))
245

246
            job.save()
Eliot Berriot's avatar
Eliot Berriot committed
247
        import_handler(import_job_id=job.pk, use_acoustid=False)