Commit 54c0987b authored by Eliot Berriot's avatar Eliot Berriot
Browse files

Merge branch 'domaind-dedicated-table' into 'develop'

Domaind dedicated table

See merge request funkwhale/funkwhale!504
parents 79f92ff5 942e9a15
...@@ -69,6 +69,8 @@ else: ...@@ -69,6 +69,8 @@ else:
FUNKWHALE_HOSTNAME = _parsed.netloc FUNKWHALE_HOSTNAME = _parsed.netloc
FUNKWHALE_PROTOCOL = _parsed.scheme FUNKWHALE_PROTOCOL = _parsed.scheme
FUNKWHALE_PROTOCOL = FUNKWHALE_PROTOCOL.lower()
FUNKWHALE_HOSTNAME = FUNKWHALE_HOSTNAME.lower()
FUNKWHALE_URL = "{}://{}".format(FUNKWHALE_PROTOCOL, FUNKWHALE_HOSTNAME) FUNKWHALE_URL = "{}://{}".format(FUNKWHALE_PROTOCOL, FUNKWHALE_HOSTNAME)
FUNKWHALE_SPA_HTML_ROOT = env( FUNKWHALE_SPA_HTML_ROOT = env(
"FUNKWHALE_SPA_HTML_ROOT", default=FUNKWHALE_URL + "/front/" "FUNKWHALE_SPA_HTML_ROOT", default=FUNKWHALE_URL + "/front/"
...@@ -83,7 +85,7 @@ APP_NAME = "Funkwhale" ...@@ -83,7 +85,7 @@ APP_NAME = "Funkwhale"
# XXX: deprecated, see #186 # XXX: deprecated, see #186
FEDERATION_ENABLED = env.bool("FEDERATION_ENABLED", default=True) FEDERATION_ENABLED = env.bool("FEDERATION_ENABLED", default=True)
FEDERATION_HOSTNAME = env("FEDERATION_HOSTNAME", default=FUNKWHALE_HOSTNAME) FEDERATION_HOSTNAME = env("FEDERATION_HOSTNAME", default=FUNKWHALE_HOSTNAME).lower()
# XXX: deprecated, see #186 # XXX: deprecated, see #186
FEDERATION_COLLECTION_PAGE_SIZE = env.int("FEDERATION_COLLECTION_PAGE_SIZE", default=50) FEDERATION_COLLECTION_PAGE_SIZE = env.int("FEDERATION_COLLECTION_PAGE_SIZE", default=50)
# XXX: deprecated, see #186 # XXX: deprecated, see #186
......
...@@ -5,7 +5,7 @@ from asgiref.sync import async_to_sync ...@@ -5,7 +5,7 @@ from asgiref.sync import async_to_sync
from channels.layers import get_channel_layer from channels.layers import get_channel_layer
from django.core.serializers.json import DjangoJSONEncoder from django.core.serializers.json import DjangoJSONEncoder
logger = logging.getLogger(__file__) logger = logging.getLogger(__name__)
channel_layer = get_channel_layer() channel_layer = get_channel_layer()
group_add = async_to_sync(channel_layer.group_add) group_add = async_to_sync(channel_layer.group_add)
......
...@@ -10,7 +10,6 @@ from funkwhale_api.users import models ...@@ -10,7 +10,6 @@ from funkwhale_api.users import models
mapping = { mapping = {
"dynamic_preferences.change_globalpreferencemodel": "settings", "dynamic_preferences.change_globalpreferencemodel": "settings",
"music.add_importbatch": "library", "music.add_importbatch": "library",
"federation.change_library": "federation",
} }
......
...@@ -42,23 +42,39 @@ ACTIVITY_TYPES = [ ...@@ -42,23 +42,39 @@ ACTIVITY_TYPES = [
"View", "View",
] ]
FUNKWHALE_OBJECT_TYPES = [
OBJECT_TYPES = [ ("Domain", "Domain"),
"Article", ("Artist", "Artist"),
"Audio", ("Album", "Album"),
"Collection", ("Track", "Track"),
"Document", ("Library", "Library"),
"Event", ]
"Image", OBJECT_TYPES = (
"Note", [
"OrderedCollection", "Application",
"Page", "Article",
"Place", "Audio",
"Profile", "Collection",
"Relationship", "Document",
"Tombstone", "Event",
"Video", "Group",
] + ACTIVITY_TYPES "Image",
"Note",
"Object",
"OrderedCollection",
"Organization",
"Page",
"Person",
"Place",
"Profile",
"Relationship",
"Service",
"Tombstone",
"Video",
]
+ ACTIVITY_TYPES
+ FUNKWHALE_OBJECT_TYPES
)
BROADCAST_TO_USER_ACTIVITIES = ["Follow", "Accept"] BROADCAST_TO_USER_ACTIVITIES = ["Follow", "Accept"]
...@@ -386,15 +402,3 @@ def get_actors_from_audience(urls): ...@@ -386,15 +402,3 @@ def get_actors_from_audience(urls):
if not final_query: if not final_query:
return models.Actor.objects.none() return models.Actor.objects.none()
return models.Actor.objects.filter(final_query) return models.Actor.objects.filter(final_query)
def get_inbox_urls(actor_queryset):
"""
Given an actor queryset, returns a deduplicated set containing
all inbox or shared inbox urls where we should deliver our payloads for
those actors
"""
values = actor_queryset.values("inbox_url", "shared_inbox_url")
urls = set([actor["shared_inbox_url"] or actor["inbox_url"] for actor in values])
return sorted(urls)
...@@ -24,6 +24,12 @@ def redeliver_activities(modeladmin, request, queryset): ...@@ -24,6 +24,12 @@ def redeliver_activities(modeladmin, request, queryset):
redeliver_activities.short_description = "Redeliver" redeliver_activities.short_description = "Redeliver"
@admin.register(models.Domain)
class DomainAdmin(admin.ModelAdmin):
list_display = ["name", "creation_date"]
search_fields = ["name"]
@admin.register(models.Activity) @admin.register(models.Activity)
class ActivityAdmin(admin.ModelAdmin): class ActivityAdmin(admin.ModelAdmin):
list_display = ["type", "fid", "url", "actor", "creation_date"] list_display = ["type", "fid", "url", "actor", "creation_date"]
......
...@@ -66,24 +66,39 @@ def create_user(actor): ...@@ -66,24 +66,39 @@ def create_user(actor):
return user_factories.UserFactory(actor=actor) return user_factories.UserFactory(actor=actor)
@registry.register
class Domain(factory.django.DjangoModelFactory):
name = factory.Faker("domain_name")
class Meta:
model = "federation.Domain"
django_get_or_create = ("name",)
@registry.register @registry.register
class ActorFactory(factory.DjangoModelFactory): class ActorFactory(factory.DjangoModelFactory):
public_key = None public_key = None
private_key = None private_key = None
preferred_username = factory.Faker("user_name") preferred_username = factory.Faker("user_name")
summary = factory.Faker("paragraph") summary = factory.Faker("paragraph")
domain = factory.Faker("domain_name") domain = factory.SubFactory(Domain)
fid = factory.LazyAttribute( fid = factory.LazyAttribute(
lambda o: "https://{}/users/{}".format(o.domain, o.preferred_username) lambda o: "https://{}/users/{}".format(o.domain.name, o.preferred_username)
) )
followers_url = factory.LazyAttribute( followers_url = factory.LazyAttribute(
lambda o: "https://{}/users/{}followers".format(o.domain, o.preferred_username) lambda o: "https://{}/users/{}followers".format(
o.domain.name, o.preferred_username
)
) )
inbox_url = factory.LazyAttribute( inbox_url = factory.LazyAttribute(
lambda o: "https://{}/users/{}/inbox".format(o.domain, o.preferred_username) lambda o: "https://{}/users/{}/inbox".format(
o.domain.name, o.preferred_username
)
) )
outbox_url = factory.LazyAttribute( outbox_url = factory.LazyAttribute(
lambda o: "https://{}/users/{}/outbox".format(o.domain, o.preferred_username) lambda o: "https://{}/users/{}/outbox".format(
o.domain.name, o.preferred_username
)
) )
class Meta: class Meta:
...@@ -95,7 +110,9 @@ class ActorFactory(factory.DjangoModelFactory): ...@@ -95,7 +110,9 @@ class ActorFactory(factory.DjangoModelFactory):
return return
from funkwhale_api.users.factories import UserFactory from funkwhale_api.users.factories import UserFactory
self.domain = settings.FEDERATION_HOSTNAME self.domain = models.Domain.objects.get_or_create(
name=settings.FEDERATION_HOSTNAME
)[0]
self.save(update_fields=["domain"]) self.save(update_fields=["domain"])
if not create: if not create:
if extracted and hasattr(extracted, "pk"): if extracted and hasattr(extracted, "pk"):
......
# Generated by Django 2.0.9 on 2018-12-26 19:35
from django.db import migrations, models
import django.db.models.deletion
import django.utils.timezone
class Migration(migrations.Migration):
dependencies = [("federation", "0012_auto_20180920_1803")]
operations = [
migrations.AlterField(
model_name="actor",
name="private_key",
field=models.TextField(blank=True, max_length=5000, null=True),
),
migrations.AlterField(
model_name="actor",
name="public_key",
field=models.TextField(blank=True, max_length=5000, null=True),
),
]
# Generated by Django 2.0.9 on 2018-12-05 09:58
from django.db import migrations, models
import django.db.models.deletion
import django.utils.timezone
class Migration(migrations.Migration):
dependencies = [("federation", "0013_auto_20181226_1935")]
operations = [
migrations.CreateModel(
name="Domain",
fields=[
(
"name",
models.CharField(max_length=255, primary_key=True, serialize=False),
),
(
"creation_date",
models.DateTimeField(default=django.utils.timezone.now),
),
],
),
migrations.AlterField(
model_name="actor",
name="domain",
field=models.CharField(max_length=1000, null=True),
),
migrations.RenameField("actor", "domain", "old_domain"),
migrations.AddField(
model_name="actor",
name="domain",
field=models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="actors",
to="federation.Domain",
),
),
migrations.AlterUniqueTogether(name="actor", unique_together=set()),
migrations.AlterUniqueTogether(
name="actor", unique_together={("domain", "preferred_username")}
),
]
# Generated by Django 2.0.9 on 2018-11-14 08:55
from django.db import migrations, models
import django.db.models.deletion
import django.utils.timezone
def populate_domains(apps, schema_editor):
Domain = apps.get_model("federation", "Domain")
Actor = apps.get_model("federation", "Actor")
domains = set(
[v.lower() for v in Actor.objects.values_list("old_domain", flat=True)]
)
for domain in sorted(domains):
print("Populating domain {}...".format(domain))
first_actor = (
Actor.objects.order_by("creation_date")
.exclude(creation_date=None)
.filter(old_domain__iexact=domain)
.first()
)
if first_actor:
first_seen = first_actor.creation_date
else:
first_seen = django.utils.timezone.now()
Domain.objects.update_or_create(
name=domain, defaults={"creation_date": first_seen}
)
for domain in Domain.objects.all():
Actor.objects.filter(old_domain__iexact=domain.name).update(domain=domain)
def skip(apps, schema_editor):
pass
class Migration(migrations.Migration):
dependencies = [("federation", "0014_auto_20181205_0958")]
operations = [
migrations.RunPython(populate_domains, skip),
migrations.AlterField(
model_name="actor",
name="domain",
field=models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="actors",
to="federation.Domain",
),
),
]
# Generated by Django 2.0.9 on 2018-12-27 16:05
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
import funkwhale_api.federation.models
class Migration(migrations.Migration):
dependencies = [("federation", "0015_populate_domains")]
operations = [
migrations.AddField(
model_name="domain",
name="nodeinfo",
field=django.contrib.postgres.fields.jsonb.JSONField(
default=funkwhale_api.federation.models.empty_dict, max_length=50000
),
),
migrations.AddField(
model_name="domain",
name="nodeinfo_fetch_date",
field=models.DateTimeField(blank=True, default=None, null=True),
),
]
...@@ -62,6 +62,81 @@ class ActorQuerySet(models.QuerySet): ...@@ -62,6 +62,81 @@ class ActorQuerySet(models.QuerySet):
return qs return qs
class DomainQuerySet(models.QuerySet):
def external(self):
return self.exclude(pk=settings.FEDERATION_HOSTNAME)
def with_last_activity_date(self):
activities = Activity.objects.filter(
actor__domain=models.OuterRef("pk")
).order_by("-creation_date")
return self.annotate(
last_activity_date=models.Subquery(activities.values("creation_date")[:1])
)
def with_actors_count(self):
return self.annotate(actors_count=models.Count("actors", distinct=True))
def with_outbox_activities_count(self):
return self.annotate(
outbox_activities_count=models.Count("actors__outbox_activities")
)
class Domain(models.Model):
name = models.CharField(primary_key=True, max_length=255)
creation_date = models.DateTimeField(default=timezone.now)
nodeinfo_fetch_date = models.DateTimeField(default=None, null=True, blank=True)
nodeinfo = JSONField(default=empty_dict, max_length=50000, blank=True)
objects = DomainQuerySet.as_manager()
def __str__(self):
return self.name
def save(self, **kwargs):
lowercase_fields = ["name"]
for field in lowercase_fields:
v = getattr(self, field, None)
if v:
setattr(self, field, v.lower())
super().save(**kwargs)
def get_stats(self):
from funkwhale_api.music import models as music_models
data = Domain.objects.filter(pk=self.pk).aggregate(
actors=models.Count("actors", distinct=True),
outbox_activities=models.Count("actors__outbox_activities", distinct=True),
libraries=models.Count("actors__libraries", distinct=True),
uploads=models.Count("actors__libraries__uploads", distinct=True),
received_library_follows=models.Count(
"actors__libraries__received_follows", distinct=True
),
emitted_library_follows=models.Count(
"actors__library_follows", distinct=True
),
)
data["artists"] = music_models.Artist.objects.filter(
from_activity__actor__domain_id=self.pk
).count()
data["albums"] = music_models.Album.objects.filter(
from_activity__actor__domain_id=self.pk
).count()
data["tracks"] = music_models.Track.objects.filter(
from_activity__actor__domain_id=self.pk
).count()
uploads = music_models.Upload.objects.filter(library__actor__domain_id=self.pk)
data["media_total_size"] = uploads.aggregate(v=models.Sum("size"))["v"] or 0
data["media_downloaded_size"] = (
uploads.with_file().aggregate(v=models.Sum("size"))["v"] or 0
)
return data
class Actor(models.Model): class Actor(models.Model):
ap_type = "Actor" ap_type = "Actor"
...@@ -74,7 +149,7 @@ class Actor(models.Model): ...@@ -74,7 +149,7 @@ class Actor(models.Model):
shared_inbox_url = models.URLField(max_length=500, null=True, blank=True) shared_inbox_url = models.URLField(max_length=500, null=True, blank=True)
type = models.CharField(choices=TYPE_CHOICES, default="Person", max_length=25) type = models.CharField(choices=TYPE_CHOICES, default="Person", max_length=25)
name = models.CharField(max_length=200, null=True, blank=True) name = models.CharField(max_length=200, null=True, blank=True)
domain = models.CharField(max_length=1000) domain = models.ForeignKey(Domain, on_delete=models.CASCADE, related_name="actors")
summary = models.CharField(max_length=500, null=True, blank=True) summary = models.CharField(max_length=500, null=True, blank=True)
preferred_username = models.CharField(max_length=200, null=True, blank=True) preferred_username = models.CharField(max_length=200, null=True, blank=True)
public_key = models.TextField(max_length=5000, null=True, blank=True) public_key = models.TextField(max_length=5000, null=True, blank=True)
...@@ -110,36 +185,9 @@ class Actor(models.Model): ...@@ -110,36 +185,9 @@ class Actor(models.Model):
def __str__(self): def __str__(self):
return "{}@{}".format(self.preferred_username, self.domain) return "{}@{}".format(self.preferred_username, self.domain)
def save(self, **kwargs):
lowercase_fields = ["domain"]
for field in lowercase_fields:
v = getattr(self, field, None)
if v:
setattr(self, field, v.lower())
super().save(**kwargs)
@property @property
def is_local(self): def is_local(self):
return self.domain == settings.FEDERATION_HOSTNAME return self.domain_id == settings.FEDERATION_HOSTNAME
@property
def is_system(self):
from . import actors
return all(
[
settings.FEDERATION_HOSTNAME == self.domain,
self.preferred_username in actors.SYSTEM_ACTORS,
]
)
@property
def system_conf(self):
from . import actors
if self.is_system:
return actors.SYSTEM_ACTORS[self.preferred_username]
def get_approved_followers(self): def get_approved_followers(self):
follows = self.received_follows.filter(approved=True) follows = self.received_follows.filter(approved=True)
......
...@@ -114,7 +114,7 @@ class ActorSerializer(serializers.Serializer): ...@@ -114,7 +114,7 @@ class ActorSerializer(serializers.Serializer):
if maf is not None: if maf is not None:
kwargs["manually_approves_followers"] = maf kwargs["manually_approves_followers"] = maf
domain = urllib.parse.urlparse(kwargs["fid"]).netloc domain = urllib.parse.urlparse(kwargs["fid"]).netloc
kwargs["domain"] = domain kwargs["domain"] = models.Domain.objects.get_or_create(pk=domain)[0]
for endpoint, url in self.initial_data.get("endpoints", {}).items(): for endpoint, url in self.initial_data.get("endpoints", {}).items():
if endpoint == "sharedInbox": if endpoint == "sharedInbox":
kwargs["shared_inbox_url"] = url kwargs["shared_inbox_url"] = url
...@@ -888,3 +888,12 @@ class CollectionSerializer(serializers.Serializer): ...@@ -888,3 +888,12 @@ class CollectionSerializer(serializers.Serializer):
if self.context.get("include_ap_context", True): if self.context.get("include_ap_context", True):
d["@context"] = AP_CONTEXT d["@context"] = AP_CONTEXT
return d return d
class NodeInfoLinkSerializer(serializers.Serializer):
href = serializers.URLField()
rel = serializers.URLField()
class NodeInfoSerializer(serializers.Serializer):
links = serializers.ListField(child=NodeInfoLinkSerializer(), min_length=1)
import datetime import datetime
import logging import logging
import os import os
import requests
from django.conf import settings from django.conf import settings
from django.db.models import Q, F from django.db.models import Q, F
...@@ -14,6 +15,7 @@ from funkwhale_api.music import models as music_models ...@@ -14,6 +15,7 @@ from funkwhale_api.music import models as music_models
from funkwhale_api.taskapp import celery from funkwhale_api.taskapp import celery
from . import models, signing from . import models, signing
from . import serializers
from . import routes from . import routes
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
...@@ -147,3 +149,40 @@ def deliver_to_remote(delivery): ...@@ -147,3 +149,40 @@ def deliver_to_remote(delivery):
delivery.attempts = F("attempts") + 1 delivery.attempts = F("attempts") + 1
delivery.is_delivered = True delivery.is_delivered = True
delivery.save(update_fields=["last_attempt_date", "attempts", "is_delivered"]) delivery.save(update_fields=["last_attempt_date", "attempts", "is_delivered"])