Merge branch 'domaind-dedicated-table' into 'develop'

Domaind dedicated table

See merge request funkwhale/funkwhale!504

api/config/settings/common.py

@@ -69,6 +69,8 @@ else:
         FUNKWHALE_HOSTNAME = _parsed.netloc
         FUNKWHALE_PROTOCOL = _parsed.scheme
 
+FUNKWHALE_PROTOCOL = FUNKWHALE_PROTOCOL.lower()
+FUNKWHALE_HOSTNAME = FUNKWHALE_HOSTNAME.lower()
 FUNKWHALE_URL = "{}://{}".format(FUNKWHALE_PROTOCOL, FUNKWHALE_HOSTNAME)
 FUNKWHALE_SPA_HTML_ROOT = env(
     "FUNKWHALE_SPA_HTML_ROOT", default=FUNKWHALE_URL + "/front/"
@@ -83,7 +85,7 @@ APP_NAME = "Funkwhale"
 
 # XXX: deprecated, see #186
 FEDERATION_ENABLED = env.bool("FEDERATION_ENABLED", default=True)
-FEDERATION_HOSTNAME = env("FEDERATION_HOSTNAME", default=FUNKWHALE_HOSTNAME)
+FEDERATION_HOSTNAME = env("FEDERATION_HOSTNAME", default=FUNKWHALE_HOSTNAME).lower()
 # XXX: deprecated, see #186
 FEDERATION_COLLECTION_PAGE_SIZE = env.int("FEDERATION_COLLECTION_PAGE_SIZE", default=50)
 # XXX: deprecated, see #186

api/funkwhale_api/common/channels.py

@@ -5,7 +5,7 @@ from asgiref.sync import async_to_sync
 from channels.layers import get_channel_layer
 from django.core.serializers.json import DjangoJSONEncoder
 
-logger = logging.getLogger(__file__)
+logger = logging.getLogger(__name__)
 channel_layer = get_channel_layer()
 group_add = async_to_sync(channel_layer.group_add)
 

api/funkwhale_api/common/scripts/django_permissions_to_user_permissions.py

@@ -10,7 +10,6 @@ from funkwhale_api.users import models
 mapping = {
     "dynamic_preferences.change_globalpreferencemodel": "settings",
     "music.add_importbatch": "library",
-    "federation.change_library": "federation",
 }
 
 

api/funkwhale_api/federation/activity.py

@@ -42,23 +42,39 @@ ACTIVITY_TYPES = [
     "View",
 ]
 
-
-OBJECT_TYPES = [
-    "Article",
-    "Audio",
-    "Collection",
-    "Document",
-    "Event",
-    "Image",
-    "Note",
-    "OrderedCollection",
-    "Page",
-    "Place",
-    "Profile",
-    "Relationship",
-    "Tombstone",
-    "Video",
-] + ACTIVITY_TYPES
+FUNKWHALE_OBJECT_TYPES = [
+    ("Domain", "Domain"),
+    ("Artist", "Artist"),
+    ("Album", "Album"),
+    ("Track", "Track"),
+    ("Library", "Library"),
+]
+OBJECT_TYPES = (
+    [
+        "Application",
+        "Article",
+        "Audio",
+        "Collection",
+        "Document",
+        "Event",
+        "Group",
+        "Image",
+        "Note",
+        "Object",
+        "OrderedCollection",
+        "Organization",
+        "Page",
+        "Person",
+        "Place",
+        "Profile",
+        "Relationship",
+        "Service",
+        "Tombstone",
+        "Video",
+    ]
+    + ACTIVITY_TYPES
+    + FUNKWHALE_OBJECT_TYPES
+)
 
 
 BROADCAST_TO_USER_ACTIVITIES = ["Follow", "Accept"]
@@ -386,15 +402,3 @@ def get_actors_from_audience(urls):
     if not final_query:
         return models.Actor.objects.none()
     return models.Actor.objects.filter(final_query)
-
-
-def get_inbox_urls(actor_queryset):
-    """
-    Given an actor queryset, returns a deduplicated set containing
-    all inbox or shared inbox urls where we should deliver our payloads for
-    those actors
-    """
-    values = actor_queryset.values("inbox_url", "shared_inbox_url")
-
-    urls = set([actor["shared_inbox_url"] or actor["inbox_url"] for actor in values])
-    return sorted(urls)

api/funkwhale_api/federation/admin.py

@@ -24,6 +24,12 @@ def redeliver_activities(modeladmin, request, queryset):
 redeliver_activities.short_description = "Redeliver"
 
 
+@admin.register(models.Domain)
+class DomainAdmin(admin.ModelAdmin):
+    list_display = ["name", "creation_date"]
+    search_fields = ["name"]
+
+
 @admin.register(models.Activity)
 class ActivityAdmin(admin.ModelAdmin):
     list_display = ["type", "fid", "url", "actor", "creation_date"]

api/funkwhale_api/federation/factories.py

@@ -66,24 +66,39 @@ def create_user(actor):
     return user_factories.UserFactory(actor=actor)
 
 
+@registry.register
+class Domain(factory.django.DjangoModelFactory):
+    name = factory.Faker("domain_name")
+
+    class Meta:
+        model = "federation.Domain"
+        django_get_or_create = ("name",)
+
+
 @registry.register
 class ActorFactory(factory.DjangoModelFactory):
     public_key = None
     private_key = None
     preferred_username = factory.Faker("user_name")
     summary = factory.Faker("paragraph")
-    domain = factory.Faker("domain_name")
+    domain = factory.SubFactory(Domain)
     fid = factory.LazyAttribute(
-        lambda o: "https://{}/users/{}".format(o.domain, o.preferred_username)
+        lambda o: "https://{}/users/{}".format(o.domain.name, o.preferred_username)
     )
     followers_url = factory.LazyAttribute(
-        lambda o: "https://{}/users/{}followers".format(o.domain, o.preferred_username)
+        lambda o: "https://{}/users/{}followers".format(
+            o.domain.name, o.preferred_username
+        )
     )
     inbox_url = factory.LazyAttribute(
-        lambda o: "https://{}/users/{}/inbox".format(o.domain, o.preferred_username)
+        lambda o: "https://{}/users/{}/inbox".format(
+            o.domain.name, o.preferred_username
+        )
     )
     outbox_url = factory.LazyAttribute(
-        lambda o: "https://{}/users/{}/outbox".format(o.domain, o.preferred_username)
+        lambda o: "https://{}/users/{}/outbox".format(
+            o.domain.name, o.preferred_username
+        )
     )
 
     class Meta:
@@ -95,7 +110,9 @@ class ActorFactory(factory.DjangoModelFactory):
             return
         from funkwhale_api.users.factories import UserFactory
 
-        self.domain = settings.FEDERATION_HOSTNAME
+        self.domain = models.Domain.objects.get_or_create(
+            name=settings.FEDERATION_HOSTNAME
+        )[0]
         self.save(update_fields=["domain"])
         if not create:
             if extracted and hasattr(extracted, "pk"):

api/funkwhale_api/federation/migrations/0013_auto_20181226_1935.py

+# Generated by Django 2.0.9 on 2018-12-26 19:35
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("federation", "0012_auto_20180920_1803")]
+
+    operations = [
+        migrations.AlterField(
+            model_name="actor",
+            name="private_key",
+            field=models.TextField(blank=True, max_length=5000, null=True),
+        ),
+        migrations.AlterField(
+            model_name="actor",
+            name="public_key",
+            field=models.TextField(blank=True, max_length=5000, null=True),
+        ),
+    ]

api/funkwhale_api/federation/migrations/0014_auto_20181205_0958.py

+# Generated by Django 2.0.9 on 2018-12-05 09:58
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("federation", "0013_auto_20181226_1935")]
+
+    operations = [
+        migrations.CreateModel(
+            name="Domain",
+            fields=[
+                (
+                    "name",
+                    models.CharField(max_length=255, primary_key=True, serialize=False),
+                ),
+                (
+                    "creation_date",
+                    models.DateTimeField(default=django.utils.timezone.now),
+                ),
+            ],
+        ),
+        migrations.AlterField(
+            model_name="actor",
+            name="domain",
+            field=models.CharField(max_length=1000, null=True),
+        ),
+        migrations.RenameField("actor", "domain", "old_domain"),
+        migrations.AddField(
+            model_name="actor",
+            name="domain",
+            field=models.ForeignKey(
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="actors",
+                to="federation.Domain",
+            ),
+        ),
+        migrations.AlterUniqueTogether(name="actor", unique_together=set()),
+        migrations.AlterUniqueTogether(
+            name="actor", unique_together={("domain", "preferred_username")}
+        ),
+    ]

api/funkwhale_api/federation/migrations/0015_populate_domains.py

+# Generated by Django 2.0.9 on 2018-11-14 08:55
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+
+
+def populate_domains(apps, schema_editor):
+    Domain = apps.get_model("federation", "Domain")
+    Actor = apps.get_model("federation", "Actor")
+
+    domains = set(
+        [v.lower() for v in Actor.objects.values_list("old_domain", flat=True)]
+    )
+    for domain in sorted(domains):
+        print("Populating domain {}...".format(domain))
+        first_actor = (
+            Actor.objects.order_by("creation_date")
+            .exclude(creation_date=None)
+            .filter(old_domain__iexact=domain)
+            .first()
+        )
+
+        if first_actor:
+            first_seen = first_actor.creation_date
+        else:
+            first_seen = django.utils.timezone.now()
+
+        Domain.objects.update_or_create(
+            name=domain, defaults={"creation_date": first_seen}
+        )
+
+    for domain in Domain.objects.all():
+        Actor.objects.filter(old_domain__iexact=domain.name).update(domain=domain)
+
+
+def skip(apps, schema_editor):
+    pass
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("federation", "0014_auto_20181205_0958")]
+
+    operations = [
+        migrations.RunPython(populate_domains, skip),
+        migrations.AlterField(
+            model_name="actor",
+            name="domain",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="actors",
+                to="federation.Domain",
+            ),
+        ),
+    ]

api/funkwhale_api/federation/migrations/0016_auto_20181227_1605.py

+# Generated by Django 2.0.9 on 2018-12-27 16:05
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+import funkwhale_api.federation.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("federation", "0015_populate_domains")]
+
+    operations = [
+        migrations.AddField(
+            model_name="domain",
+            name="nodeinfo",
+            field=django.contrib.postgres.fields.jsonb.JSONField(
+                default=funkwhale_api.federation.models.empty_dict, max_length=50000
+            ),
+        ),
+        migrations.AddField(
+            model_name="domain",
+            name="nodeinfo_fetch_date",
+            field=models.DateTimeField(blank=True, default=None, null=True),
+        ),
+    ]

api/funkwhale_api/federation/models.py

@@ -62,6 +62,81 @@ class ActorQuerySet(models.QuerySet):
         return qs
 
 
+class DomainQuerySet(models.QuerySet):
+    def external(self):
+        return self.exclude(pk=settings.FEDERATION_HOSTNAME)
+
+    def with_last_activity_date(self):
+        activities = Activity.objects.filter(
+            actor__domain=models.OuterRef("pk")
+        ).order_by("-creation_date")
+
+        return self.annotate(
+            last_activity_date=models.Subquery(activities.values("creation_date")[:1])
+        )
+
+    def with_actors_count(self):
+        return self.annotate(actors_count=models.Count("actors", distinct=True))
+
+    def with_outbox_activities_count(self):
+        return self.annotate(
+            outbox_activities_count=models.Count("actors__outbox_activities")
+        )
+
+
+class Domain(models.Model):
+    name = models.CharField(primary_key=True, max_length=255)
+    creation_date = models.DateTimeField(default=timezone.now)
+    nodeinfo_fetch_date = models.DateTimeField(default=None, null=True, blank=True)
+    nodeinfo = JSONField(default=empty_dict, max_length=50000, blank=True)
+
+    objects = DomainQuerySet.as_manager()
+
+    def __str__(self):
+        return self.name
+
+    def save(self, **kwargs):
+        lowercase_fields = ["name"]
+        for field in lowercase_fields:
+            v = getattr(self, field, None)
+            if v:
+                setattr(self, field, v.lower())
+
+        super().save(**kwargs)
+
+    def get_stats(self):
+        from funkwhale_api.music import models as music_models
+
+        data = Domain.objects.filter(pk=self.pk).aggregate(
+            actors=models.Count("actors", distinct=True),
+            outbox_activities=models.Count("actors__outbox_activities", distinct=True),
+            libraries=models.Count("actors__libraries", distinct=True),
+            uploads=models.Count("actors__libraries__uploads", distinct=True),
+            received_library_follows=models.Count(
+                "actors__libraries__received_follows", distinct=True
+            ),
+            emitted_library_follows=models.Count(
+                "actors__library_follows", distinct=True
+            ),
+        )
+        data["artists"] = music_models.Artist.objects.filter(
+            from_activity__actor__domain_id=self.pk
+        ).count()
+        data["albums"] = music_models.Album.objects.filter(
+            from_activity__actor__domain_id=self.pk
+        ).count()
+        data["tracks"] = music_models.Track.objects.filter(
+            from_activity__actor__domain_id=self.pk
+        ).count()
+
+        uploads = music_models.Upload.objects.filter(library__actor__domain_id=self.pk)
+        data["media_total_size"] = uploads.aggregate(v=models.Sum("size"))["v"] or 0
+        data["media_downloaded_size"] = (
+            uploads.with_file().aggregate(v=models.Sum("size"))["v"] or 0
+        )
+        return data
+
+
 class Actor(models.Model):
     ap_type = "Actor"
 
@@ -74,7 +149,7 @@ class Actor(models.Model):
     shared_inbox_url = models.URLField(max_length=500, null=True, blank=True)
     type = models.CharField(choices=TYPE_CHOICES, default="Person", max_length=25)
     name = models.CharField(max_length=200, null=True, blank=True)
-    domain = models.CharField(max_length=1000)
+    domain = models.ForeignKey(Domain, on_delete=models.CASCADE, related_name="actors")
     summary = models.CharField(max_length=500, null=True, blank=True)
     preferred_username = models.CharField(max_length=200, null=True, blank=True)
     public_key = models.TextField(max_length=5000, null=True, blank=True)
@@ -110,36 +185,9 @@ class Actor(models.Model):
     def __str__(self):
         return "{}@{}".format(self.preferred_username, self.domain)
 
-    def save(self, **kwargs):
-        lowercase_fields = ["domain"]
-        for field in lowercase_fields:
-            v = getattr(self, field, None)
-            if v:
-                setattr(self, field, v.lower())
-
-        super().save(**kwargs)
-
     @property
     def is_local(self):
-        return self.domain == settings.FEDERATION_HOSTNAME
-
-    @property
-    def is_system(self):
-        from . import actors
-
-        return all(
-            [
-                settings.FEDERATION_HOSTNAME == self.domain,
-                self.preferred_username in actors.SYSTEM_ACTORS,
-            ]
-        )
-
-    @property
-    def system_conf(self):
-        from . import actors
-
-        if self.is_system:
-            return actors.SYSTEM_ACTORS[self.preferred_username]
+        return self.domain_id == settings.FEDERATION_HOSTNAME
 
     def get_approved_followers(self):
         follows = self.received_follows.filter(approved=True)

api/funkwhale_api/federation/serializers.py

@@ -114,7 +114,7 @@ class ActorSerializer(serializers.Serializer):
         if maf is not None:
             kwargs["manually_approves_followers"] = maf
         domain = urllib.parse.urlparse(kwargs["fid"]).netloc
-        kwargs["domain"] = domain
+        kwargs["domain"] = models.Domain.objects.get_or_create(pk=domain)[0]
         for endpoint, url in self.initial_data.get("endpoints", {}).items():
             if endpoint == "sharedInbox":
                 kwargs["shared_inbox_url"] = url
@@ -888,3 +888,12 @@ class CollectionSerializer(serializers.Serializer):
         if self.context.get("include_ap_context", True):
             d["@context"] = AP_CONTEXT
         return d
+
+
+class NodeInfoLinkSerializer(serializers.Serializer):
+    href = serializers.URLField()
+    rel = serializers.URLField()
+
+
+class NodeInfoSerializer(serializers.Serializer):
+    links = serializers.ListField(child=NodeInfoLinkSerializer(), min_length=1)

api/funkwhale_api/federation/tasks.py

 import datetime
 import logging
 import os
+import requests
 
 from django.conf import settings
 from django.db.models import Q, F
@@ -14,6 +15,7 @@ from funkwhale_api.music import models as music_models
 from funkwhale_api.taskapp import celery
 
 from . import models, signing
+from . import serializers
 from . import routes
 
 logger = logging.getLogger(__name__)
@@ -147,3 +149,40 @@ def deliver_to_remote(delivery):
         delivery.attempts = F("attempts") + 1
         delivery.is_delivered = True
         delivery.save(update_fields=["last_attempt_date", "attempts", "is_delivered"])
+
+
+def fetch_nodeinfo(domain_name):
+    s = session.get_session()
+    wellknown_url = "https://{}/.well-known/nodeinfo".format(domain_name)
+    response = s.get(
+        url=wellknown_url, timeout=5, verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL
+    )
+    response.raise_for_status()
+    serializer = serializers.NodeInfoSerializer(data=response.json())
+    serializer.is_valid(raise_exception=True)
+    nodeinfo_url = None
+    for link in serializer.validated_data["links"]:
+        if link["rel"] == "http://nodeinfo.diaspora.software/ns/schema/2.0":
+            nodeinfo_url = link["href"]
+            break
+
+    response = s.get(
+        url=nodeinfo_url, timeout=5, verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL
+    )
+    response.raise_for_status()
+    return response.json()
+
+
+@celery.app.task(name="federation.update_domain_nodeinfo")
+@celery.require_instance(
+    models.Domain.objects.external(), "domain", id_kwarg_name="domain_name"
+)
+def update_domain_nodeinfo(domain):
+    now = timezone.now()
+    try:
+        nodeinfo = {"status": "ok", "payload": fetch_nodeinfo(domain.name)}
+    except (requests.RequestException, serializers.serializers.ValidationError) as e:
+        nodeinfo = {"status": "error", "error": str(e)}
+    domain.nodeinfo_fetch_date = now
+    domain.nodeinfo = nodeinfo
+    domain.save(update_fields=["nodeinfo", "nodeinfo_fetch_date"])

api/funkwhale_api/manage/filters.py

 from django_filters import rest_framework as filters
 
 from funkwhale_api.common import fields
+from funkwhale_api.federation import models as federation_models
 from funkwhale_api.music import models as music_models
 from funkwhale_api.users import models as users_models
 
@@ -20,6 +21,14 @@ class ManageUploadFilterSet(filters.FilterSet):
         fields = ["q", "track__album", "track__artist", "track"]
 
 
+class ManageDomainFilterSet(filters.FilterSet):
+    q = fields.SearchFilter(search_fields=["name"])
+
+    class Meta:
+        model = federation_models.Domain
+        fields = ["name"]
+
+
 class ManageUserFilterSet(filters.FilterSet):
     q = fields.SearchFilter(search_fields=["username", "email", "name"])
 
@@ -31,10 +40,9 @@ class ManageUserFilterSet(filters.FilterSet):
             "privacy_level",
             "is_staff",
             "is_superuser",
-            "permission_upload",
             "permission_library",
             "permission_settings",
-            "permission_federation",
+            "permission_moderation",
         ]
 
 

api/funkwhale_api/manage/serializers.py

@@ -3,6 +3,7 @@ from django.db import transaction
 from rest_framework import serializers
 
 from funkwhale_api.common import serializers as common_serializers
+from funkwhale_api.federation import models as federation_models
 from funkwhale_api.music import models as music_models
 from funkwhale_api.users import models as users_models
 
@@ -168,3 +169,30 @@ class ManageInvitationActionSerializer(common_serializers.ActionSerializer):
     @transaction.atomic
     def handle_delete(self, objects):
         return objects.delete()
+
+