Verified Commit 7ac3bb98 authored by Eliot Berriot's avatar Eliot Berriot
Browse files

Moved actor domain to a dedicated table

parent 060543f6
...@@ -69,6 +69,8 @@ else: ...@@ -69,6 +69,8 @@ else:
FUNKWHALE_HOSTNAME = _parsed.netloc FUNKWHALE_HOSTNAME = _parsed.netloc
FUNKWHALE_PROTOCOL = _parsed.scheme FUNKWHALE_PROTOCOL = _parsed.scheme
FUNKWHALE_PROTOCOL = FUNKWHALE_PROTOCOL.lower()
FUNKWHALE_HOSTNAME = FUNKWHALE_HOSTNAME.lower()
FUNKWHALE_URL = "{}://{}".format(FUNKWHALE_PROTOCOL, FUNKWHALE_HOSTNAME) FUNKWHALE_URL = "{}://{}".format(FUNKWHALE_PROTOCOL, FUNKWHALE_HOSTNAME)
FUNKWHALE_SPA_HTML_ROOT = env( FUNKWHALE_SPA_HTML_ROOT = env(
"FUNKWHALE_SPA_HTML_ROOT", default=FUNKWHALE_URL + "/front/" "FUNKWHALE_SPA_HTML_ROOT", default=FUNKWHALE_URL + "/front/"
...@@ -83,7 +85,7 @@ APP_NAME = "Funkwhale" ...@@ -83,7 +85,7 @@ APP_NAME = "Funkwhale"
# XXX: deprecated, see #186 # XXX: deprecated, see #186
FEDERATION_ENABLED = env.bool("FEDERATION_ENABLED", default=True) FEDERATION_ENABLED = env.bool("FEDERATION_ENABLED", default=True)
FEDERATION_HOSTNAME = env("FEDERATION_HOSTNAME", default=FUNKWHALE_HOSTNAME) FEDERATION_HOSTNAME = env("FEDERATION_HOSTNAME", default=FUNKWHALE_HOSTNAME).lower()
# XXX: deprecated, see #186 # XXX: deprecated, see #186
FEDERATION_COLLECTION_PAGE_SIZE = env.int("FEDERATION_COLLECTION_PAGE_SIZE", default=50) FEDERATION_COLLECTION_PAGE_SIZE = env.int("FEDERATION_COLLECTION_PAGE_SIZE", default=50)
# XXX: deprecated, see #186 # XXX: deprecated, see #186
......
...@@ -24,6 +24,12 @@ def redeliver_activities(modeladmin, request, queryset): ...@@ -24,6 +24,12 @@ def redeliver_activities(modeladmin, request, queryset):
redeliver_activities.short_description = "Redeliver" redeliver_activities.short_description = "Redeliver"
@admin.register(models.Domain)
class DomainAdmin(admin.ModelAdmin):
list_display = ["name", "creation_date"]
search_fields = ["name"]
@admin.register(models.Activity) @admin.register(models.Activity)
class ActivityAdmin(admin.ModelAdmin): class ActivityAdmin(admin.ModelAdmin):
list_display = ["type", "fid", "url", "actor", "creation_date"] list_display = ["type", "fid", "url", "actor", "creation_date"]
......
...@@ -66,24 +66,39 @@ def create_user(actor): ...@@ -66,24 +66,39 @@ def create_user(actor):
return user_factories.UserFactory(actor=actor) return user_factories.UserFactory(actor=actor)
@registry.register
class Domain(factory.django.DjangoModelFactory):
name = factory.Faker("domain_name")
class Meta:
model = "federation.Domain"
django_get_or_create = ("name",)
@registry.register @registry.register
class ActorFactory(factory.DjangoModelFactory): class ActorFactory(factory.DjangoModelFactory):
public_key = None public_key = None
private_key = None private_key = None
preferred_username = factory.Faker("user_name") preferred_username = factory.Faker("user_name")
summary = factory.Faker("paragraph") summary = factory.Faker("paragraph")
domain = factory.Faker("domain_name") domain = factory.SubFactory(Domain)
fid = factory.LazyAttribute( fid = factory.LazyAttribute(
lambda o: "https://{}/users/{}".format(o.domain, o.preferred_username) lambda o: "https://{}/users/{}".format(o.domain.name, o.preferred_username)
) )
followers_url = factory.LazyAttribute( followers_url = factory.LazyAttribute(
lambda o: "https://{}/users/{}followers".format(o.domain, o.preferred_username) lambda o: "https://{}/users/{}followers".format(
o.domain.name, o.preferred_username
)
) )
inbox_url = factory.LazyAttribute( inbox_url = factory.LazyAttribute(
lambda o: "https://{}/users/{}/inbox".format(o.domain, o.preferred_username) lambda o: "https://{}/users/{}/inbox".format(
o.domain.name, o.preferred_username
)
) )
outbox_url = factory.LazyAttribute( outbox_url = factory.LazyAttribute(
lambda o: "https://{}/users/{}/outbox".format(o.domain, o.preferred_username) lambda o: "https://{}/users/{}/outbox".format(
o.domain.name, o.preferred_username
)
) )
class Meta: class Meta:
...@@ -95,7 +110,9 @@ class ActorFactory(factory.DjangoModelFactory): ...@@ -95,7 +110,9 @@ class ActorFactory(factory.DjangoModelFactory):
return return
from funkwhale_api.users.factories import UserFactory from funkwhale_api.users.factories import UserFactory
self.domain = settings.FEDERATION_HOSTNAME self.domain = models.Domain.objects.get_or_create(
name=settings.FEDERATION_HOSTNAME
)[0]
self.save(update_fields=["domain"]) self.save(update_fields=["domain"])
if not create: if not create:
if extracted and hasattr(extracted, "pk"): if extracted and hasattr(extracted, "pk"):
......
# Generated by Django 2.0.9 on 2018-12-26 19:35
from django.db import migrations, models
import django.db.models.deletion
import django.utils.timezone
class Migration(migrations.Migration):
dependencies = [("federation", "0012_auto_20180920_1803")]
operations = [
migrations.AlterField(
model_name="actor",
name="private_key",
field=models.TextField(blank=True, max_length=5000, null=True),
),
migrations.AlterField(
model_name="actor",
name="public_key",
field=models.TextField(blank=True, max_length=5000, null=True),
),
]
# Generated by Django 2.0.9 on 2018-12-05 09:58
from django.db import migrations, models
import django.db.models.deletion
import django.utils.timezone
class Migration(migrations.Migration):
dependencies = [("federation", "0013_auto_20181226_1935")]
operations = [
migrations.CreateModel(
name="Domain",
fields=[
(
"name",
models.CharField(max_length=255, primary_key=True, serialize=False),
),
(
"creation_date",
models.DateTimeField(default=django.utils.timezone.now),
),
],
),
migrations.AlterField(
model_name="actor",
name="domain",
field=models.CharField(max_length=1000, null=True),
),
migrations.RenameField("actor", "domain", "old_domain"),
migrations.AddField(
model_name="actor",
name="domain",
field=models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="actors",
to="federation.Domain",
),
),
migrations.AlterUniqueTogether(name="actor", unique_together=set()),
migrations.AlterUniqueTogether(
name="actor", unique_together={("domain", "preferred_username")}
),
]
# Generated by Django 2.0.9 on 2018-11-14 08:55
from django.db import migrations, models
import django.db.models.deletion
import django.utils.timezone
def populate_domains(apps, schema_editor):
Domain = apps.get_model("federation", "Domain")
Actor = apps.get_model("federation", "Actor")
domains = set(
[v.lower() for v in Actor.objects.values_list("old_domain", flat=True)]
)
for domain in sorted(domains):
print("Populating domain {}...".format(domain))
first_actor = (
Actor.objects.order_by("creation_date")
.exclude(creation_date=None)
.filter(old_domain__iexact=domain)
.first()
)
if first_actor:
first_seen = first_actor.creation_date
else:
first_seen = django.utils.timezone.now()
Domain.objects.update_or_create(
name=domain, defaults={"creation_date": first_seen}
)
for domain in Domain.objects.all():
Actor.objects.filter(old_domain__iexact=domain.name).update(domain=domain)
def skip(apps, schema_editor):
pass
class Migration(migrations.Migration):
dependencies = [("federation", "0014_auto_20181205_0958")]
operations = [
migrations.RunPython(populate_domains, skip),
migrations.AlterField(
model_name="actor",
name="domain",
field=models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="actors",
to="federation.Domain",
),
),
]
...@@ -62,6 +62,23 @@ class ActorQuerySet(models.QuerySet): ...@@ -62,6 +62,23 @@ class ActorQuerySet(models.QuerySet):
return qs return qs
class Domain(models.Model):
name = models.CharField(primary_key=True, max_length=255)
creation_date = models.DateTimeField(default=timezone.now)
def __str__(self):
return self.name
def save(self, **kwargs):
lowercase_fields = ["name"]
for field in lowercase_fields:
v = getattr(self, field, None)
if v:
setattr(self, field, v.lower())
super().save(**kwargs)
class Actor(models.Model): class Actor(models.Model):
ap_type = "Actor" ap_type = "Actor"
...@@ -74,7 +91,7 @@ class Actor(models.Model): ...@@ -74,7 +91,7 @@ class Actor(models.Model):
shared_inbox_url = models.URLField(max_length=500, null=True, blank=True) shared_inbox_url = models.URLField(max_length=500, null=True, blank=True)
type = models.CharField(choices=TYPE_CHOICES, default="Person", max_length=25) type = models.CharField(choices=TYPE_CHOICES, default="Person", max_length=25)
name = models.CharField(max_length=200, null=True, blank=True) name = models.CharField(max_length=200, null=True, blank=True)
domain = models.CharField(max_length=1000) domain = models.ForeignKey(Domain, on_delete=models.CASCADE, related_name="actors")
summary = models.CharField(max_length=500, null=True, blank=True) summary = models.CharField(max_length=500, null=True, blank=True)
preferred_username = models.CharField(max_length=200, null=True, blank=True) preferred_username = models.CharField(max_length=200, null=True, blank=True)
public_key = models.TextField(max_length=5000, null=True, blank=True) public_key = models.TextField(max_length=5000, null=True, blank=True)
...@@ -110,36 +127,9 @@ class Actor(models.Model): ...@@ -110,36 +127,9 @@ class Actor(models.Model):
def __str__(self): def __str__(self):
return "{}@{}".format(self.preferred_username, self.domain) return "{}@{}".format(self.preferred_username, self.domain)
def save(self, **kwargs):
lowercase_fields = ["domain"]
for field in lowercase_fields:
v = getattr(self, field, None)
if v:
setattr(self, field, v.lower())
super().save(**kwargs)
@property @property
def is_local(self): def is_local(self):
return self.domain == settings.FEDERATION_HOSTNAME return self.domain_id == settings.FEDERATION_HOSTNAME
@property
def is_system(self):
from . import actors
return all(
[
settings.FEDERATION_HOSTNAME == self.domain,
self.preferred_username in actors.SYSTEM_ACTORS,
]
)
@property
def system_conf(self):
from . import actors
if self.is_system:
return actors.SYSTEM_ACTORS[self.preferred_username]
def get_approved_followers(self): def get_approved_followers(self):
follows = self.received_follows.filter(approved=True) follows = self.received_follows.filter(approved=True)
......
...@@ -114,7 +114,8 @@ class ActorSerializer(serializers.Serializer): ...@@ -114,7 +114,8 @@ class ActorSerializer(serializers.Serializer):
if maf is not None: if maf is not None:
kwargs["manually_approves_followers"] = maf kwargs["manually_approves_followers"] = maf
domain = urllib.parse.urlparse(kwargs["fid"]).netloc domain = urllib.parse.urlparse(kwargs["fid"]).netloc
kwargs["domain"] = domain kwargs["domain"] = models.Domain.objects.get_or_create(
pk=domain)[0]
for endpoint, url in self.initial_data.get("endpoints", {}).items(): for endpoint, url in self.initial_data.get("endpoints", {}).items():
if endpoint == "sharedInbox": if endpoint == "sharedInbox":
kwargs["shared_inbox_url"] = url kwargs["shared_inbox_url"] = url
......
...@@ -252,7 +252,9 @@ def get_actor_data(user): ...@@ -252,7 +252,9 @@ def get_actor_data(user):
username = federation_utils.slugify_username(user.username) username = federation_utils.slugify_username(user.username)
return { return {
"preferred_username": username, "preferred_username": username,
"domain": settings.FEDERATION_HOSTNAME, "domain": federation_models.Domain.objects.get_or_create(
name=settings.FEDERATION_HOSTNAME
)[0],
"type": "Person", "type": "Person",
"name": user.username, "name": user.username,
"manually_approves_followers": False, "manually_approves_followers": False,
......
...@@ -12,12 +12,16 @@ from faker.providers import internet as internet_provider ...@@ -12,12 +12,16 @@ from faker.providers import internet as internet_provider
import factory import factory
import pytest import pytest
from django.core.management import call_command
from django.contrib.auth.models import AnonymousUser from django.contrib.auth.models import AnonymousUser
from django.core.cache import cache as django_cache, caches from django.core.cache import cache as django_cache, caches
from django.core.files import uploadedfile from django.core.files import uploadedfile
from django.utils import timezone from django.utils import timezone
from django.test import client from django.test import client
from django.db import connection
from django.db.migrations.executor import MigrationExecutor
from django.db.models import QuerySet from django.db.models import QuerySet
from dynamic_preferences.registries import global_preferences_registry from dynamic_preferences.registries import global_preferences_registry
from rest_framework import fields as rest_fields from rest_framework import fields as rest_fields
from rest_framework.test import APIClient, APIRequestFactory from rest_framework.test import APIClient, APIRequestFactory
...@@ -400,3 +404,9 @@ def spa_html(r_mock, settings): ...@@ -400,3 +404,9 @@ def spa_html(r_mock, settings):
@pytest.fixture @pytest.fixture
def no_api_auth(preferences): def no_api_auth(preferences):
preferences["common__api_authentication_required"] = False preferences["common__api_authentication_required"] = False
@pytest.fixture()
def migrator(transactional_db):
yield MigrationExecutor(connection)
call_command("migrate", interactive=False)
def test_domain_14_migration(migrator):
a, f, t = ("federation", "0014_auto_20181205_0958", "0015_populate_domains")
migrator.migrate([(a, f)])
old_apps = migrator.loader.project_state([(a, f)]).apps
Actor = old_apps.get_model(a, "Actor")
a1 = Actor.objects.create(
fid="http://test1.com", preferred_username="test1", old_domain="dOmaiN1.com"
)
a2 = Actor.objects.create(
fid="http://test2.com", preferred_username="test2", old_domain="domain1.com"
)
a3 = Actor.objects.create(
fid="http://test3.com", preferred_username="test2", old_domain="domain2.com"
)
migrator.loader.build_graph()
migrator.migrate([(a, t)])
new_apps = migrator.loader.project_state([(a, t)]).apps
Actor = new_apps.get_model(a, "Actor")
Domain = new_apps.get_model(a, "Domain")
a1 = Actor.objects.get(pk=a1.pk)
a2 = Actor.objects.get(pk=a2.pk)
a3 = Actor.objects.get(pk=a3.pk)
assert Domain.objects.count() == 2
assert a1.domain == Domain.objects.get(pk="domain1.com")
assert a2.domain == Domain.objects.get(pk="domain1.com")
assert a3.domain == Domain.objects.get(pk="domain2.com")
assert Domain.objects.get(pk="domain1.com").creation_date == a1.creation_date
assert Domain.objects.get(pk="domain2.com").creation_date == a3.creation_date
...@@ -54,3 +54,16 @@ def test_actor_get_quota(factories): ...@@ -54,3 +54,16 @@ def test_actor_get_quota(factories):
expected = {"total": 10, "pending": 1, "skipped": 2, "errored": 3, "finished": 4} expected = {"total": 10, "pending": 1, "skipped": 2, "errored": 3, "finished": 4}
assert library.actor.get_current_usage() == expected assert library.actor.get_current_usage() == expected
@pytest.mark.parametrize(
"value, expected",
[
("Domain.com", "domain.com"),
("hello-WORLD.com", "hello-world.com"),
("posés.com", "posés.com"),
],
)
def test_domain_name_saved_properly(value, expected, factories):
domain = factories["federation.Domain"](name=value)
assert domain.name == expected
...@@ -43,7 +43,7 @@ def test_actor_serializer_from_ap(db): ...@@ -43,7 +43,7 @@ def test_actor_serializer_from_ap(db):
assert actor.public_key == payload["publicKey"]["publicKeyPem"] assert actor.public_key == payload["publicKey"]["publicKeyPem"]
assert actor.preferred_username == payload["preferredUsername"] assert actor.preferred_username == payload["preferredUsername"]
assert actor.name == payload["name"] assert actor.name == payload["name"]
assert actor.domain == "test.federation" assert actor.domain.pk == "test.federation"
assert actor.summary == payload["summary"] assert actor.summary == payload["summary"]
assert actor.type == "Person" assert actor.type == "Person"
assert actor.manually_approves_followers == payload["manuallyApprovesFollowers"] assert actor.manually_approves_followers == payload["manuallyApprovesFollowers"]
...@@ -71,7 +71,7 @@ def test_actor_serializer_only_mandatory_field_from_ap(db): ...@@ -71,7 +71,7 @@ def test_actor_serializer_only_mandatory_field_from_ap(db):
assert actor.followers_url == payload["followers"] assert actor.followers_url == payload["followers"]
assert actor.following_url == payload["following"] assert actor.following_url == payload["following"]
assert actor.preferred_username == payload["preferredUsername"] assert actor.preferred_username == payload["preferredUsername"]
assert actor.domain == "test.federation" assert actor.domain.pk == "test.federation"
assert actor.type == "Person" assert actor.type == "Person"
assert actor.manually_approves_followers is None assert actor.manually_approves_followers is None
...@@ -110,7 +110,7 @@ def test_actor_serializer_to_ap(): ...@@ -110,7 +110,7 @@ def test_actor_serializer_to_ap():
public_key=expected["publicKey"]["publicKeyPem"], public_key=expected["publicKey"]["publicKeyPem"],
preferred_username=expected["preferredUsername"], preferred_username=expected["preferredUsername"],
name=expected["name"], name=expected["name"],
domain="test.federation", domain=models.Domain(pk="test.federation"),
summary=expected["summary"], summary=expected["summary"],
type="Person", type="Person",
manually_approves_followers=False, manually_approves_followers=False,
...@@ -135,7 +135,7 @@ def test_webfinger_serializer(): ...@@ -135,7 +135,7 @@ def test_webfinger_serializer():
actor = models.Actor( actor = models.Actor(
fid=expected["links"][0]["href"], fid=expected["links"][0]["href"],
preferred_username="service", preferred_username="service",
domain="test.federation", domain=models.Domain(pk="test.federation"),
) )
serializer = serializers.ActorWebfingerSerializer(actor) serializer = serializers.ActorWebfingerSerializer(actor)
...@@ -898,7 +898,7 @@ def test_local_actor_serializer_to_ap(factories): ...@@ -898,7 +898,7 @@ def test_local_actor_serializer_to_ap(factories):
public_key=expected["publicKey"]["publicKeyPem"], public_key=expected["publicKey"]["publicKeyPem"],
preferred_username=expected["preferredUsername"], preferred_username=expected["preferredUsername"],
name=expected["name"], name=expected["name"],
domain="test.federation", domain=models.Domain.objects.create(pk="test.federation"),
summary=expected["summary"], summary=expected["summary"],
type="Person", type="Person",
manually_approves_followers=False, manually_approves_followers=False,
......
...@@ -137,7 +137,7 @@ def test_creating_actor_from_user(factories, settings): ...@@ -137,7 +137,7 @@ def test_creating_actor_from_user(factories, settings):
actor = models.create_actor(user) actor = models.create_actor(user)
assert actor.preferred_username == "Hello_M_world" # slugified assert actor.preferred_username == "Hello_M_world" # slugified
assert actor.domain == settings.FEDERATION_HOSTNAME assert actor.domain.pk == settings.FEDERATION_HOSTNAME
assert actor.type == "Person" assert actor.type == "Person"
assert actor.name == user.username assert actor.name == user.username
assert actor.manually_approves_followers is False assert actor.manually_approves_followers is False
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment