models.py 15.3 KB
Newer Older
1
import tempfile
Eliot Berriot's avatar
Eliot Berriot committed
2
import uuid
Eliot Berriot's avatar
Eliot Berriot committed
3

Eliot Berriot's avatar
Eliot Berriot committed
4
from django.conf import settings
5
from django.contrib.postgres.fields import JSONField
6
7
from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
8
from django.core.exceptions import ObjectDoesNotExist
9
from django.core.serializers.json import DjangoJSONEncoder
Eliot Berriot's avatar
Eliot Berriot committed
10
11
from django.db import models
from django.utils import timezone
12
from django.urls import reverse
Eliot Berriot's avatar
Eliot Berriot committed
13

14
from funkwhale_api.common import session
Eliot Berriot's avatar
Eliot Berriot committed
15
from funkwhale_api.common import utils as common_utils
16
from funkwhale_api.common import validators as common_validators
17
18
from funkwhale_api.music import utils as music_utils

19
20
from . import utils as federation_utils

Eliot Berriot's avatar
Eliot Berriot committed
21
TYPE_CHOICES = [
Eliot Berriot's avatar
Eliot Berriot committed
22
23
24
25
26
    ("Person", "Person"),
    ("Application", "Application"),
    ("Group", "Group"),
    ("Organization", "Organization"),
    ("Service", "Service"),
Eliot Berriot's avatar
Eliot Berriot committed
27
28
29
]


30
31
32
33
def empty_dict():
    return {}


34
35
36
37
def get_shared_inbox_url():
    return federation_utils.full_url(reverse("federation:shared-inbox"))


38
39
40
41
42
43
44
45
46
class FederationMixin(models.Model):
    # federation id/url
    fid = models.URLField(unique=True, max_length=500, db_index=True)
    url = models.URLField(max_length=500, null=True, blank=True)

    class Meta:
        abstract = True


47
48
49
50
class ActorQuerySet(models.QuerySet):
    def local(self, include=True):
        return self.exclude(user__isnull=include)

51
52
53
    def with_current_usage(self):
        qs = self
        for s in ["pending", "skipped", "errored", "finished"]:
54
55
56
57
58
            uploads_query = models.Q(
                libraries__uploads__import_status=s,
                libraries__uploads__audio_file__isnull=False,
                libraries__uploads__audio_file__ne="",
            )
59
60
61
            qs = qs.annotate(
                **{
                    "_usage_{}".format(s): models.Sum(
62
                        "libraries__uploads__size", filter=uploads_query
63
64
65
66
67
68
                    )
                }
            )

        return qs

69
70
71
72
73
    def with_uploads_count(self):
        return self.annotate(
            uploads_count=models.Count("libraries__uploads", distinct=True)
        )

74

75
76
77
78
79
80
81
82
83
class DomainQuerySet(models.QuerySet):
    def external(self):
        return self.exclude(pk=settings.FEDERATION_HOSTNAME)

    def with_actors_count(self):
        return self.annotate(actors_count=models.Count("actors", distinct=True))

    def with_outbox_activities_count(self):
        return self.annotate(
84
85
86
            outbox_activities_count=models.Count(
                "actors__outbox_activities", distinct=True
            )
87
88
89
        )


90
class Domain(models.Model):
91
92
93
94
95
    name = models.CharField(
        primary_key=True,
        max_length=255,
        validators=[common_validators.DomainValidator()],
    )
96
    creation_date = models.DateTimeField(default=timezone.now)
Eliot Berriot's avatar
Eliot Berriot committed
97
    nodeinfo_fetch_date = models.DateTimeField(default=None, null=True, blank=True)
98
    nodeinfo = JSONField(default=empty_dict, max_length=50000, blank=True)
Eliot Berriot's avatar
Eliot Berriot committed
99

100
    objects = DomainQuerySet.as_manager()
101
102
103
104
105
106
107
108
109
110
111
112
113

    def __str__(self):
        return self.name

    def save(self, **kwargs):
        lowercase_fields = ["name"]
        for field in lowercase_fields:
            v = getattr(self, field, None)
            if v:
                setattr(self, field, v.lower())

        super().save(**kwargs)

114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
    def get_stats(self):
        from funkwhale_api.music import models as music_models

        data = Domain.objects.filter(pk=self.pk).aggregate(
            actors=models.Count("actors", distinct=True),
            outbox_activities=models.Count("actors__outbox_activities", distinct=True),
            libraries=models.Count("actors__libraries", distinct=True),
            received_library_follows=models.Count(
                "actors__libraries__received_follows", distinct=True
            ),
            emitted_library_follows=models.Count(
                "actors__library_follows", distinct=True
            ),
        )
        data["artists"] = music_models.Artist.objects.filter(
            from_activity__actor__domain_id=self.pk
        ).count()
        data["albums"] = music_models.Album.objects.filter(
            from_activity__actor__domain_id=self.pk
        ).count()
        data["tracks"] = music_models.Track.objects.filter(
            from_activity__actor__domain_id=self.pk
        ).count()

        uploads = music_models.Upload.objects.filter(library__actor__domain_id=self.pk)
139
        data["uploads"] = uploads.count()
140
141
142
143
144
145
        data["media_total_size"] = uploads.aggregate(v=models.Sum("size"))["v"] or 0
        data["media_downloaded_size"] = (
            uploads.with_file().aggregate(v=models.Sum("size"))["v"] or 0
        )
        return data

146

Eliot Berriot's avatar
Eliot Berriot committed
147
class Actor(models.Model):
Eliot Berriot's avatar
Eliot Berriot committed
148
    ap_type = "Actor"
Eliot Berriot's avatar
Eliot Berriot committed
149

150
151
    fid = models.URLField(unique=True, max_length=500, db_index=True)
    url = models.URLField(max_length=500, null=True, blank=True)
Eliot Berriot's avatar
Eliot Berriot committed
152
153
154
155
156
    outbox_url = models.URLField(max_length=500)
    inbox_url = models.URLField(max_length=500)
    following_url = models.URLField(max_length=500, null=True, blank=True)
    followers_url = models.URLField(max_length=500, null=True, blank=True)
    shared_inbox_url = models.URLField(max_length=500, null=True, blank=True)
Eliot Berriot's avatar
Eliot Berriot committed
157
    type = models.CharField(choices=TYPE_CHOICES, default="Person", max_length=25)
Eliot Berriot's avatar
Eliot Berriot committed
158
    name = models.CharField(max_length=200, null=True, blank=True)
159
    domain = models.ForeignKey(Domain, on_delete=models.CASCADE, related_name="actors")
Eliot Berriot's avatar
Eliot Berriot committed
160
    summary = models.CharField(max_length=500, null=True, blank=True)
Eliot Berriot's avatar
Eliot Berriot committed
161
    preferred_username = models.CharField(max_length=200, null=True, blank=True)
Eliot Berriot's avatar
Eliot Berriot committed
162
163
    public_key = models.TextField(max_length=5000, null=True, blank=True)
    private_key = models.TextField(max_length=5000, null=True, blank=True)
Eliot Berriot's avatar
Eliot Berriot committed
164
    creation_date = models.DateTimeField(default=timezone.now)
Eliot Berriot's avatar
Eliot Berriot committed
165
    last_fetch_date = models.DateTimeField(default=timezone.now)
Eliot Berriot's avatar
Eliot Berriot committed
166
    manually_approves_followers = models.NullBooleanField(default=None)
167
    followers = models.ManyToManyField(
Eliot Berriot's avatar
Eliot Berriot committed
168
        to="self",
169
        symmetrical=False,
Eliot Berriot's avatar
Eliot Berriot committed
170
171
172
        through="Follow",
        through_fields=("target", "actor"),
        related_name="following",
173
    )
Eliot Berriot's avatar
Eliot Berriot committed
174

175
176
    objects = ActorQuerySet.as_manager()

177
    class Meta:
Eliot Berriot's avatar
Eliot Berriot committed
178
        unique_together = ["domain", "preferred_username"]
179

Eliot Berriot's avatar
Eliot Berriot committed
180
181
    @property
    def webfinger_subject(self):
Eliot Berriot's avatar
Eliot Berriot committed
182
        return "{}@{}".format(self.preferred_username, settings.FEDERATION_HOSTNAME)
183
184
185

    @property
    def private_key_id(self):
186
        return "{}#main-key".format(self.fid)
Eliot Berriot's avatar
Eliot Berriot committed
187
188

    @property
189
    def full_username(self):
190
        return "{}@{}".format(self.preferred_username, self.domain_id)
191
192

    def __str__(self):
193
        return "{}@{}".format(self.preferred_username, self.domain_id)
Eliot Berriot's avatar
Eliot Berriot committed
194

195
196
    @property
    def is_local(self):
197
        return self.domain_id == settings.FEDERATION_HOSTNAME
Eliot Berriot's avatar
Eliot Berriot committed
198

199
200
    def get_approved_followers(self):
        follows = self.received_follows.filter(approved=True)
Eliot Berriot's avatar
Eliot Berriot committed
201
        return self.followers.filter(pk__in=follows.values_list("actor", flat=True))
202

203
204
    def should_autoapprove_follow(self, actor):
        return False
Eliot Berriot's avatar
Eliot Berriot committed
205

206
207
208
209
210
211
212
213
214
215
216
217
218
219
    def get_user(self):
        try:
            return self.user
        except ObjectDoesNotExist:
            return None

    def get_current_usage(self):
        actor = self.__class__.objects.filter(pk=self.pk).with_current_usage().get()
        data = {}
        for s in ["pending", "skipped", "errored", "finished"]:
            data[s] = getattr(actor, "_usage_{}".format(s)) or 0

        data["total"] = sum(data.values())
        return data
Eliot Berriot's avatar
Eliot Berriot committed
220

221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
    def get_stats(self):
        from funkwhale_api.music import models as music_models

        data = Actor.objects.filter(pk=self.pk).aggregate(
            outbox_activities=models.Count("outbox_activities", distinct=True),
            libraries=models.Count("libraries", distinct=True),
            received_library_follows=models.Count(
                "libraries__received_follows", distinct=True
            ),
            emitted_library_follows=models.Count("library_follows", distinct=True),
        )
        data["artists"] = music_models.Artist.objects.filter(
            from_activity__actor=self.pk
        ).count()
        data["albums"] = music_models.Album.objects.filter(
            from_activity__actor=self.pk
        ).count()
        data["tracks"] = music_models.Track.objects.filter(
            from_activity__actor=self.pk
        ).count()

        uploads = music_models.Upload.objects.filter(library__actor=self.pk)
        data["uploads"] = uploads.count()
        data["media_total_size"] = uploads.aggregate(v=models.Sum("size"))["v"] or 0
        data["media_downloaded_size"] = (
            uploads.with_file().aggregate(v=models.Sum("size"))["v"] or 0
        )
        return data

250
251
252
253
254
255
256
257
258
    @property
    def keys(self):
        return self.private_key, self.public_key

    @keys.setter
    def keys(self, v):
        self.private_key = v[0].decode("utf-8")
        self.public_key = v[1].decode("utf-8")

259
260

class InboxItem(models.Model):
Eliot Berriot's avatar
Eliot Berriot committed
261
262
263
264
    """
    Store activities binding to local actors, with read/unread status.
    """

265
266
267
268
269
270
271
    actor = models.ForeignKey(
        Actor, related_name="inbox_items", on_delete=models.CASCADE
    )
    activity = models.ForeignKey(
        "Activity", related_name="inbox_items", on_delete=models.CASCADE
    )
    type = models.CharField(max_length=10, choices=[("to", "to"), ("cc", "cc")])
272
    is_read = models.BooleanField(default=False)
273

Eliot Berriot's avatar
Eliot Berriot committed
274
275
276
277
278
279
280
281
282
283
284
285
286
287

class Delivery(models.Model):
    """
    Store deliveries attempt to remote inboxes
    """

    is_delivered = models.BooleanField(default=False)
    last_attempt_date = models.DateTimeField(null=True, blank=True)
    attempts = models.PositiveIntegerField(default=0)
    inbox_url = models.URLField(max_length=500)

    activity = models.ForeignKey(
        "Activity", related_name="deliveries", on_delete=models.CASCADE
    )
288
289
290
291
292
293
294
295
296


class Activity(models.Model):
    actor = models.ForeignKey(
        Actor, related_name="outbox_activities", on_delete=models.CASCADE
    )
    recipients = models.ManyToManyField(
        Actor, related_name="inbox_activities", through=InboxItem
    )
Eliot Berriot's avatar
Eliot Berriot committed
297
    uuid = models.UUIDField(default=uuid.uuid4, unique=True)
298
299
300
    fid = models.URLField(unique=True, max_length=500, null=True, blank=True)
    url = models.URLField(max_length=500, null=True, blank=True)
    payload = JSONField(default=empty_dict, max_length=50000, encoder=DjangoJSONEncoder)
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
    creation_date = models.DateTimeField(default=timezone.now, db_index=True)
    type = models.CharField(db_index=True, null=True, max_length=100)

    # generic relations
    object_id = models.IntegerField(null=True)
    object_content_type = models.ForeignKey(
        ContentType,
        null=True,
        on_delete=models.SET_NULL,
        related_name="objecting_activities",
    )
    object = GenericForeignKey("object_content_type", "object_id")
    target_id = models.IntegerField(null=True)
    target_content_type = models.ForeignKey(
        ContentType,
        null=True,
        on_delete=models.SET_NULL,
        related_name="targeting_activities",
    )
    target = GenericForeignKey("target_content_type", "target_id")
    related_object_id = models.IntegerField(null=True)
    related_object_content_type = models.ForeignKey(
        ContentType,
        null=True,
        on_delete=models.SET_NULL,
        related_name="related_objecting_activities",
    )
    related_object = GenericForeignKey(
        "related_object_content_type", "related_object_id"
    )
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350


class AbstractFollow(models.Model):
    ap_type = "Follow"
    fid = models.URLField(unique=True, max_length=500, null=True, blank=True)
    uuid = models.UUIDField(default=uuid.uuid4, unique=True)
    creation_date = models.DateTimeField(default=timezone.now)
    modification_date = models.DateTimeField(auto_now=True)
    approved = models.NullBooleanField(default=None)

    class Meta:
        abstract = True

    def get_federation_id(self):
        return federation_utils.full_url(
            "{}#follows/{}".format(self.actor.fid, self.uuid)
        )


class Follow(AbstractFollow):
Eliot Berriot's avatar
Eliot Berriot committed
351
    actor = models.ForeignKey(
Eliot Berriot's avatar
Eliot Berriot committed
352
        Actor, related_name="emitted_follows", on_delete=models.CASCADE
Eliot Berriot's avatar
Eliot Berriot committed
353
354
    )
    target = models.ForeignKey(
Eliot Berriot's avatar
Eliot Berriot committed
355
        Actor, related_name="received_follows", on_delete=models.CASCADE
Eliot Berriot's avatar
Eliot Berriot committed
356
357
358
    )

    class Meta:
Eliot Berriot's avatar
Eliot Berriot committed
359
        unique_together = ["actor", "target"]
Eliot Berriot's avatar
Eliot Berriot committed
360

361
362
363
364
365
366
367
368
369
370
371

class LibraryFollow(AbstractFollow):
    actor = models.ForeignKey(
        Actor, related_name="library_follows", on_delete=models.CASCADE
    )
    target = models.ForeignKey(
        "music.Library", related_name="received_follows", on_delete=models.CASCADE
    )

    class Meta:
        unique_together = ["actor", "target"]
Eliot Berriot's avatar
Eliot Berriot committed
372
373


374
375
class Library(models.Model):
    creation_date = models.DateTimeField(default=timezone.now)
Eliot Berriot's avatar
Eliot Berriot committed
376
    modification_date = models.DateTimeField(auto_now=True)
377
378
    fetched_date = models.DateTimeField(null=True, blank=True)
    actor = models.OneToOneField(
Eliot Berriot's avatar
Eliot Berriot committed
379
380
        Actor, on_delete=models.CASCADE, related_name="library"
    )
381
    uuid = models.UUIDField(default=uuid.uuid4)
382
    url = models.URLField(max_length=500)
383

384
385
386
387
    # use this flag to disable federation with a library
    federation_enabled = models.BooleanField()
    # should we mirror files locally or hotlink them?
    download_files = models.BooleanField()
388
389
390
    # should we automatically import new files from this library?
    autoimport = models.BooleanField()
    tracks_count = models.PositiveIntegerField(null=True, blank=True)
391
    follow = models.OneToOneField(
Eliot Berriot's avatar
Eliot Berriot committed
392
        Follow, related_name="library", null=True, blank=True, on_delete=models.SET_NULL
393
    )
394
395


Eliot Berriot's avatar
Eliot Berriot committed
396
get_file_path = common_utils.ChunkedPath("federation_cache")
397
398


399
class LibraryTrack(models.Model):
400
401
    url = models.URLField(unique=True, max_length=500)
    audio_url = models.URLField(max_length=500)
402
    audio_mimetype = models.CharField(max_length=200)
Eliot Berriot's avatar
Eliot Berriot committed
403
    audio_file = models.FileField(upload_to=get_file_path, null=True, blank=True)
404

405
    creation_date = models.DateTimeField(default=timezone.now)
Eliot Berriot's avatar
Eliot Berriot committed
406
    modification_date = models.DateTimeField(auto_now=True)
407
408
409
    fetched_date = models.DateTimeField(null=True, blank=True)
    published_date = models.DateTimeField(null=True, blank=True)
    library = models.ForeignKey(
Eliot Berriot's avatar
Eliot Berriot committed
410
411
        Library, related_name="tracks", on_delete=models.CASCADE
    )
412
413
414
    artist_name = models.CharField(max_length=500)
    album_title = models.CharField(max_length=500)
    title = models.CharField(max_length=500)
415
416
417
    metadata = JSONField(
        default=empty_dict, max_length=10000, encoder=DjangoJSONEncoder
    )
418
419
420
421

    @property
    def mbid(self):
        try:
Eliot Berriot's avatar
Eliot Berriot committed
422
            return self.metadata["recording"]["musicbrainz_id"]
423
424
        except KeyError:
            pass
425
426
427

    def download_audio(self):
        from . import actors
Eliot Berriot's avatar
Eliot Berriot committed
428
429

        auth = actors.SYSTEM_ACTORS["library"].get_request_auth()
430
431
432
433
434
435
        remote_response = session.get_session().get(
            self.audio_url,
            auth=auth,
            stream=True,
            timeout=20,
            verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
Eliot Berriot's avatar
Eliot Berriot committed
436
            headers={"Content-Type": "application/activity+json"},
437
438
439
440
        )
        with remote_response as r:
            remote_response.raise_for_status()
            extension = music_utils.get_ext_from_type(self.audio_mimetype)
Eliot Berriot's avatar
Eliot Berriot committed
441
442
            title = " - ".join([self.title, self.album_title, self.artist_name])
            filename = "{}.{}".format(title, extension)
443
444
445
446
            tmp_file = tempfile.TemporaryFile()
            for chunk in r.iter_content(chunk_size=512):
                tmp_file.write(chunk)
            self.audio_file.save(filename, tmp_file)
447
448
449

    def get_metadata(self, key):
        return self.metadata.get(key)