api/funkwhale_api/federation/authentication.py

@@ -19,6 +19,7 @@ class SignatureAuthentication(authentication.BaseAuthentication):
         try:
             actor = actors.get_actor(key_id.split("#")[0])
         except Exception as e:
+            raise
             raise exceptions.AuthenticationFailed(str(e))
 
         if not actor.public_key:

api/funkwhale_api/federation/jsonld.py

+import collections
+import pyld
+
+from rest_framework import serializers
+from django.core import validators
+
+from . import ns
+
+_cache = {}
+
+
+def document_loader(url):
+    loader = pyld.jsonld.requests_document_loader()
+    doc = ns.get_by_url(url)
+    if doc:
+        return doc
+    if url in _cache:
+        return _cache[url]
+    resp = loader(url)
+    _cache[url] = resp
+    return resp
+
+
+def expand(document):
+    return pyld.jsonld.expand(document)
+
+
+pyld.jsonld.set_document_loader(document_loader)
+
+
+class TypeField(serializers.ListField):
+    def __init__(self, *args, **kwargs):
+        kwargs.setdefault("child", serializers.URLField())
+        super().__init__(*args, **kwargs)
+
+    def to_internal_value(self, value):
+        try:
+            return value[0]
+        except IndexError:
+            self.fail("invalid", value)
+
+    def to_representation(self, value):
+        return value
+
+
+class JsonLdField(serializers.ListField):
+    def __init__(self, *args, **kwargs):
+        # by default, pyld will expand all fields to list such as
+        # [{"@value": "something"}]
+        # we may not always want this behaviour
+        self.single = kwargs.pop("single", False)
+        self.value_only = kwargs.pop("value_only", False)
+        self.id_only = kwargs.pop("id_only", False)
+        self.base_field = kwargs.pop("base_field")
+        super().__init__(*args, **kwargs)
+        self.child = self.base_field
+        self.child.parent = self
+        if self.single:
+            self.validators = [
+                v
+                for v in self.validators
+                if not isinstance(
+                    v, (validators.MinLengthValidator, validators.MaxLengthValidator)
+                )
+            ]
+
+    def to_internal_value(self, value):
+        self.source_attrs = [self.field_name]
+        value = super().to_internal_value(value)
+        if self.id_only:
+            value = [v["@id"] for v in value]
+        elif self.value_only:
+            value = [v["@value"] for v in value]
+        if value and self.single:
+            return value[0]
+        return value
+
+    def to_representation(self, value):
+        return self.base_field.to_representation(value)
+
+
+class Serializer(serializers.Serializer):
+    jsonld_fields = []
+    type_source = None
+    id_source = None
+    include_id = True
+    include_type = True
+    valid_types = []
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if self.include_id:
+            self.fields["@id"] = serializers.URLField(
+                required=True, **{"source": self.id_source} if self.id_source else {}
+            )
+        if self.include_type:
+            self.fields["@type"] = TypeField(
+                required=True,
+                **{"source": self.type_source} if self.type_source else {}
+            )
+        for field_name, field, kw in self.jsonld_fields:
+            self.fields[field_name] = JsonLdField(
+                required=field.required,
+                base_field=field,
+                min_length=1 if field.required else 0,
+                **kw
+            )
+
+    def run_validation(self, initial_data):
+        initial_data.setdefault("@context", ns.CONTEXTS)
+        import ipdb
+
+        ipdb.set_trace()
+        try:
+            document = expand(initial_data)[0]
+        except IndexError:
+            raise serializers.ValidationError(
+                "Cannot parse json-ld, maybe you are missing context."
+            )
+
+        return super().run_validation(document)
+
+    def validate(self, validated_data):
+        validated_data = super().validate(validated_data)
+        if self.valid_types and "@type" in validated_data:
+            if validated_data["@type"] not in self.valid_types:
+                raise serializers.ValidationError(
+                    {
+                        "@type": "Invalid type. Allowed types are: {}".format(
+                            ", ".join(self.valid_types)
+                        )
+                    }
+                )
+        return validated_data
+
+    def to_representation(self, data):
+        data = super().to_representation(data)
+        compacted = pyld.jsonld.compact(data, {"@context": ns.CONTEXTS})
+        # While compacted repr is json-ld valid, it's probably not was is
+        # expected by most implementations around in the fediverse,
+        # so we remove the semicolons completely and the prefix
+        return remove_semicolons(compacted)
+
+
+def remove_semicolons(struct: dict, skip=["@context"]):
+    """
+    given a JSON-LD  strut such as {
+        'as:inbox': 'https://:test'
+    },
+    returns  {
+        'inbox': 'https://:test'
+    },
+    """
+    new_data = collections.OrderedDict()
+    for k, v in struct.items():
+        if k in skip:
+            new_data[k] = v
+            continue
+        parts = k.split(":")
+        if len(parts) > 1:
+            k = ":".join(parts[1:])
+        if isinstance(v, dict):
+            v = remove_semicolons(v)
+        elif isinstance(v, (list, tuple)):
+            new_v = []
+            for e in v:
+                if isinstance(e, dict):
+                    new_v.append(remove_semicolons(e))
+                else:
+                    new_v.append(e)
+            v = new_v
+
+        new_data[k] = v
+    return new_data

api/funkwhale_api/federation/models.py

@@ -11,6 +11,9 @@ from django.utils import timezone
 from funkwhale_api.common import session
 from funkwhale_api.music import utils as music_utils
 
+from . import ns
+
+
 TYPE_CHOICES = [
     ("Person", "Person"),
     ("Application", "Application"),
@@ -97,6 +100,21 @@ class Actor(models.Model):
         follows = self.received_follows.filter(approved=True)
         return self.followers.filter(pk__in=follows.values_list("actor", flat=True))
 
+    def public_key_repr(self):
+        return {
+            "owner": self.url,
+            "publicKeyPem": self.public_key,
+            "@id": "{}#main-key".format(self.url),
+        }
+
+    @property
+    def endpoints(self):
+        return {"sharedInbox": self.shared_inbox_url}
+
+    @property
+    def type_ap(self):
+        return ns.SHORT["as:{}".format(self.type)]
+
 
 class Follow(models.Model):
     ap_type = "Follow"

api/funkwhale_api/federation/ns.py

+import collections
+
+NS = {
+    "ldp": {
+        "contextUrl": None,
+        "documentUrl": "http://www.w3.org/ns/ldp",
+        "document": {
+            "@context": {
+                "ldp": "http://www.w3.org/ns/ldp#",
+                "id": "@id",
+                "type": "@type",
+                "Container": "ldp:Container",
+                "BasicContainer": "ldp:BasicContainer",
+                "DirectContainer": "ldp:DirectContainer",
+                "IndirectContainer": "ldp:IndirectContainer",
+                "hasMemberRelation": {"@id": "ldp:hasMemberRelation", "@type": "@id"},
+                "isMemberOfRelation": {"@id": "ldp:isMemberOfRelation", "@type": "@id"},
+                "membershipResource": {"@id": "ldp:membershipResource", "@type": "@id"},
+                "insertedContentRelation": {
+                    "@id": "ldp:insertedContentRelation",
+                    "@type": "@id",
+                },
+                "contains": {"@id": "ldp:contains", "@type": "@id"},
+                "member": {"@id": "ldp:member", "@type": "@id"},
+                "constrainedBy": {"@id": "ldp:constrainedBy", "@type": "@id"},
+                "Resource": "ldp:Resource",
+                "RDFSource": "ldp:RDFSource",
+                "NonRDFSource": "ldp:NonRDFSource",
+                "MemberSubject": "ldp:MemberSubject",
+                "PreferContainment": "ldp:PreferContainment",
+                "PreferMembership": "ldp:PreferMembership",
+                "PreferMinimalContainer": "ldp:PreferMinimalContainer",
+                "PageSortCriterion": "ldp:PageSortCriterion",
+                "pageSortCriteria": {
+                    "@id": "ldp:pageSortCriteria",
+                    "@type": "@id",
+                    "@container": "@list",
+                },
+                "pageSortPredicate": {"@id": "ldp:pageSortPredicate", "@type": "@id"},
+                "pageSortOrder": {"@id": "ldp:pageSortOrder", "@type": "@id"},
+                "pageSortCollation": {"@id": "ldp:pageSortCollation", "@type": "@id"},
+                "Ascending": "ldp:Ascending",
+                "Descending": "ldp:Descending",
+                "Page": "ldp:Page",
+                "pageSequence": {"@id": "ldp:pageSequence", "@type": "@id"},
+                "inbox": {"@id": "ldp:inbox", "@type": "@id"},
+            }
+        },
+    },
+    "as": {
+        "contextUrl": None,
+        "documentUrl": "https://www.w3.org/ns/activitystreams",
+        "document": {
+            "@context": {
+                "source": "as:source",
+                "summary": "as:summary",
+                "target": {"@type": "@id", "@id": "as:target"},
+                "Block": "as:Block",
+                "shares": {"@type": "@id", "@id": "as:shares"},
+                "liked": {"@type": "@id", "@id": "as:liked"},
+                "location": {"@type": "@id", "@id": "as:location"},
+                "name": "as:name",
+                "endpoints": {"@type": "@id", "@id": "as:endpoints"},
+                "following": {"@type": "@id", "@id": "as:following"},
+                "Accept": "as:Accept",
+                "Object": "as:Object",
+                "instrument": {"@type": "@id", "@id": "as:instrument"},
+                "Mention": "as:Mention",
+                "context": {"@type": "@id", "@id": "as:context"},
+                "Travel": "as:Travel",
+                "Remove": "as:Remove",
+                "Profile": "as:Profile",
+                "preview": {"@type": "@id", "@id": "as:preview"},
+                "TentativeReject": "as:TentativeReject",
+                "startIndex": {
+                    "@type": "xsd:nonNegativeInteger",
+                    "@id": "as:startIndex",
+                },
+                "endTime": {"@type": "xsd:dateTime", "@id": "as:endTime"},
+                "current": {"@type": "@id", "@id": "as:current"},
+                "altitude": {"@type": "xsd:float", "@id": "as:altitude"},
+                "startTime": {"@type": "xsd:dateTime", "@id": "as:startTime"},
+                "CollectionPage": "as:CollectionPage",
+                "bto": {"@type": "@id", "@id": "as:bto"},
+                "orderedItems": {
+                    "@container": "@list",
+                    "@type": "@id",
+                    "@id": "as:items",
+                },
+                "attributedTo": {"@type": "@id", "@id": "as:attributedTo"},
+                "next": {"@type": "@id", "@id": "as:next"},
+                "prev": {"@type": "@id", "@id": "as:prev"},
+                "Leave": "as:Leave",
+                "signClientKey": {"@type": "@id", "@id": "as:signClientKey"},
+                "Image": "as:Image",
+                "anyOf": {"@type": "@id", "@id": "as:anyOf"},
+                "icon": {"@type": "@id", "@id": "as:icon"},
+                "@vocab": "_:",
+                "bcc": {"@type": "@id", "@id": "as:bcc"},
+                "latitude": {"@type": "xsd:float", "@id": "as:latitude"},
+                "rel": "as:rel",
+                "mediaType": "as:mediaType",
+                "Create": "as:Create",
+                "Document": "as:Document",
+                "result": {"@type": "@id", "@id": "as:result"},
+                "subject": {"@type": "@id", "@id": "as:subject"},
+                "uploadMedia": {"@type": "@id", "@id": "as:uploadMedia"},
+                "oneOf": {"@type": "@id", "@id": "as:oneOf"},
+                "streams": {"@type": "@id", "@id": "as:streams"},
+                "Like": "as:Like",
+                "Delete": "as:Delete",
+                "object": {"@type": "@id", "@id": "as:object"},
+                "Tombstone": "as:Tombstone",
+                "replies": {"@type": "@id", "@id": "as:replies"},
+                "followers": {"@type": "@id", "@id": "as:followers"},
+                "inReplyTo": {"@type": "@id", "@id": "as:inReplyTo"},
+                "content": "as:content",
+                "relationship": {"@type": "@id", "@id": "as:relationship"},
+                "duration": {"@type": "xsd:duration", "@id": "as:duration"},
+                "Event": "as:Event",
+                "longitude": {"@type": "xsd:float", "@id": "as:longitude"},
+                "Organization": "as:Organization",
+                "contentMap": {"@container": "@language", "@id": "as:content"},
+                "describes": {"@type": "@id", "@id": "as:describes"},
+                "oauthAuthorizationEndpoint": {
+                    "@type": "@id",
+                    "@id": "as:oauthAuthorizationEndpoint",
+                },
+                "Update": "as:Update",
+                "Listen": "as:Listen",
+                "Ignore": "as:Ignore",
+                "Article": "as:Article",
+                "cc": {"@type": "@id", "@id": "as:cc"},
+                "Page": "as:Page",
+                "Audio": "as:Audio",
+                "items": {"@type": "@id", "@id": "as:items"},
+                "xsd": "http://www.w3.org/2001/XMLSchema#",
+                "Reject": "as:Reject",
+                "Person": "as:Person",
+                "nameMap": {"@container": "@language", "@id": "as:name"},
+                "outbox": {"@type": "@id", "@id": "as:outbox"},
+                "actor": {"@type": "@id", "@id": "as:actor"},
+                "Move": "as:Move",
+                "preferredUsername": "as:preferredUsername",
+                "IsFollowedBy": "as:IsFollowedBy",
+                "Follow": "as:Follow",
+                "origin": {"@type": "@id", "@id": "as:origin"},
+                "Link": "as:Link",
+                "units": "as:units",
+                "width": {"@type": "xsd:nonNegativeInteger", "@id": "as:width"},
+                "likes": {"@type": "@id", "@id": "as:likes"},
+                "IsFollowing": "as:IsFollowing",
+                "Public": {"@type": "@id", "@id": "as:Public"},
+                "Service": "as:Service",
+                "Note": "as:Note",
+                "IsContact": "as:IsContact",
+                "Join": "as:Join",
+                "Place": "as:Place",
+                "attachment": {"@type": "@id", "@id": "as:attachment"},
+                "IsMember": "as:IsMember",
+                "OrderedCollectionPage": "as:OrderedCollectionPage",
+                "Invite": "as:Invite",
+                "ldp": "http://www.w3.org/ns/ldp#",
+                "to": {"@type": "@id", "@id": "as:to"},
+                "Activity": "as:Activity",
+                "generator": {"@type": "@id", "@id": "as:generator"},
+                "type": "@type",
+                "tag": {"@type": "@id", "@id": "as:tag"},
+                "hreflang": "as:hreflang",
+                "Dislike": "as:Dislike",
+                "Question": "as:Question",
+                "radius": {"@type": "xsd:float", "@id": "as:radius"},
+                "closed": {"@type": "xsd:dateTime", "@id": "as:closed"},
+                "sharedInbox": {"@type": "@id", "@id": "as:sharedInbox"},
+                "View": "as:View",
+                "Flag": "as:Flag",
+                "updated": {"@type": "xsd:dateTime", "@id": "as:updated"},
+                "Offer": "as:Offer",
+                "as": "https://www.w3.org/ns/activitystreams#",
+                "deleted": {"@type": "xsd:dateTime", "@id": "as:deleted"},
+                "last": {"@type": "@id", "@id": "as:last"},
+                "audience": {"@type": "@id", "@id": "as:audience"},
+                "summaryMap": {"@container": "@language", "@id": "as:summary"},
+                "proxyUrl": {"@type": "@id", "@id": "as:proxyUrl"},
+                "IntransitiveActivity": "as:IntransitiveActivity",
+                "OrderedCollection": "as:OrderedCollection",
+                "Group": "as:Group",
+                "oauthTokenEndpoint": {"@type": "@id", "@id": "as:oauthTokenEndpoint"},
+                "Add": "as:Add",
+                "Video": "as:Video",
+                "Read": "as:Read",
+                "provideClientKey": {"@type": "@id", "@id": "as:provideClientKey"},
+                "first": {"@type": "@id", "@id": "as:first"},
+                "TentativeAccept": "as:TentativeAccept",
+                "formerType": {"@type": "@id", "@id": "as:formerType"},
+                "href": {"@type": "@id", "@id": "as:href"},
+                "Collection": "as:Collection",
+                "accuracy": {"@type": "xsd:float", "@id": "as:accuracy"},
+                "height": {"@type": "xsd:nonNegativeInteger", "@id": "as:height"},
+                "Announce": "as:Announce",
+                "Relationship": "as:Relationship",
+                "id": "@id",
+                "published": {"@type": "xsd:dateTime", "@id": "as:published"},
+                "url": {"@type": "@id", "@id": "as:url"},
+                "totalItems": {
+                    "@type": "xsd:nonNegativeInteger",
+                    "@id": "as:totalItems",
+                },
+                "Application": "as:Application",
+                "image": {"@type": "@id", "@id": "as:image"},
+                "Arrive": "as:Arrive",
+                "partOf": {"@type": "@id", "@id": "as:partOf"},
+                "Undo": "as:Undo",
+                "inbox": {"@type": "@id", "@id": "ldp:inbox"},
+                # extensions
+                "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
+            }
+        },
+    },
+    "sec": {
+        "contextUrl": None,
+        "documentUrl": "https://w3id.org/security/v1",
+        "document": {
+            "@context": {
+                "password": "sec:password",
+                "salt": "sec:salt",
+                "sec": "https://w3id.org/security#",
+                "privateKeyPem": "sec:privateKeyPem",
+                "LinkedDataSignature2016": "sec:LinkedDataSignature2016",
+                "iterationCount": "sec:iterationCount",
+                "canonicalizationAlgorithm": "sec:canonicalizationAlgorithm",
+                "cipherData": "sec:cipherData",
+                "revoked": {"@type": "xsd:dateTime", "@id": "sec:revoked"},
+                "CryptographicKey": "sec:Key",
+                "xsd": "http://www.w3.org/2001/XMLSchema#",
+                "publicKey": {"@type": "@id", "@id": "sec:publicKey"},
+                "domain": "sec:domain",
+                "signature": "sec:signature",
+                "authenticationTag": "sec:authenticationTag",
+                "dc": "http://purl.org/dc/terms/",
+                "nonce": "sec:nonce",
+                "publicKeyPem": "sec:publicKeyPem",
+                "owner": {"@type": "@id", "@id": "sec:owner"},
+                "publicKeyService": {"@type": "@id", "@id": "sec:publicKeyService"},
+                "expiration": {"@type": "xsd:dateTime", "@id": "sec:expiration"},
+                "digestAlgorithm": "sec:digestAlgorithm",
+                "signatureAlgorithm": "sec:signingAlgorithm",
+                "normalizationAlgorithm": "sec:normalizationAlgorithm",
+                "privateKey": {"@type": "@id", "@id": "sec:privateKey"},
+                "creator": {"@type": "@id", "@id": "dc:creator"},
+                "cipherAlgorithm": "sec:cipherAlgorithm",
+                "id": "@id",
+                "Ed25519Signature2018": "sec:Ed25519Signature2018",
+                "encryptionKey": "sec:encryptionKey",
+                "created": {"@type": "xsd:dateTime", "@id": "dc:created"},
+                "signatureValue": "sec:signatureValue",
+                "cipherKey": "sec:cipherKey",
+                "type": "@type",
+                "digestValue": "sec:digestValue",
+                "publicKeyBase58": "sec:publicKeyBase58",
+                "expires": {"@type": "xsd:dateTime", "@id": "sec:expiration"},
+                "EncryptedMessage": "sec:EncryptedMessage",
+                "EcdsaKoblitzSignature2016": "sec:EcdsaKoblitzSignature2016",
+                "LinkedDataSignature2015": "sec:LinkedDataSignature2015",
+                "GraphSignature2012": "sec:GraphSignature2012",
+                "initializationVector": "sec:initializationVector",
+            }
+        },
+    },
+}
+
+
+SHORT = {}
+VALIDATORS = collections.defaultdict(list)
+
+for ns, data in NS.items():
+    context = data["document"]["@context"]
+    base = context[ns]
+    for key, value in data["document"]["@context"].items():
+        if key == ns:
+            continue
+
+        try:
+            SHORT[value] = base + key
+        except TypeError:
+            if value["@id"] in SHORT:
+                continue
+
+            if not value["@id"].startswith("{}:".format(ns)):
+                # this is an alias
+                SHORT[":".join([ns, key])] = base + key
+            else:
+                SHORT[value["@id"]] = base + key
+
+FULL_TO_SHORT = {v: k for k, v in SHORT.items()}
+CONTEXTS = [v["documentUrl"] for k, v in NS.items()]
+ADDITIONAL_CONTEXT = {"manuallyApprovesFollowers": "as:manuallyApprovesFollowers"}
+CONTEXTS.append(ADDITIONAL_CONTEXT)
+
+
+def get_by_url(url: str):
+    for v in NS.values():
+        if v["documentUrl"] == url:
+            return v
+
+
+def register_validator(SHORT: list):
+    def decorator(validator):
+        for shortcut in SHORT:
+            VALIDATORS[shortcut].append(validator)
+        return validator
+
+    return decorator

api/funkwhale_api/federation/serializers.py

@@ -10,7 +10,7 @@ from funkwhale_api.common import utils as funkwhale_utils
 from funkwhale_api.music import models as music_models
 from funkwhale_api.music import tasks as music_tasks
 
-from . import activity, filters, models, utils
+from . import activity, filters, models, utils, ns, jsonld
 
 AP_CONTEXT = [
     "https://www.w3.org/ns/activitystreams",
@@ -21,74 +21,135 @@ AP_CONTEXT = [
 logger = logging.getLogger(__name__)
 
 
-class ActorSerializer(serializers.Serializer):
-    id = serializers.URLField(max_length=500)
-    outbox = serializers.URLField(max_length=500)
-    inbox = serializers.URLField(max_length=500)
-    type = serializers.ChoiceField(choices=models.TYPE_CHOICES)
-    preferredUsername = serializers.CharField()
-    manuallyApprovesFollowers = serializers.NullBooleanField(required=False)
-    name = serializers.CharField(required=False, max_length=200)
-    summary = serializers.CharField(max_length=None, required=False)
-    followers = serializers.URLField(max_length=500, required=False, allow_null=True)
-    following = serializers.URLField(max_length=500, required=False, allow_null=True)
-    publicKey = serializers.JSONField(required=False)
-
-    def to_representation(self, instance):
-        ret = {
-            "id": instance.url,
-            "outbox": instance.outbox_url,
-            "inbox": instance.inbox_url,
-            "preferredUsername": instance.preferred_username,
-            "type": instance.type,
-        }
-        if instance.name:
-            ret["name"] = instance.name
-        if instance.followers_url:
-            ret["followers"] = instance.followers_url
-        if instance.following_url:
-            ret["following"] = instance.following_url
-        if instance.summary:
-            ret["summary"] = instance.summary
-        if instance.manually_approves_followers is not None:
-            ret["manuallyApprovesFollowers"] = instance.manually_approves_followers
-
-        ret["@context"] = AP_CONTEXT
-        if instance.public_key:
-            ret["publicKey"] = {
-                "owner": instance.url,
-                "publicKeyPem": instance.public_key,
-                "id": "{}#main-key".format(instance.url),
-            }
-        ret["endpoints"] = {}
-        if instance.shared_inbox_url:
-            ret["endpoints"]["sharedInbox"] = instance.shared_inbox_url
-        return ret
+class EndpointsField(jsonld.Serializer):
+    include_id = False
+    include_type = False
+    jsonld_fields = [
+        (
+            ns.SHORT["as:sharedInbox"],
+            serializers.URLField(),
+            {"source": "sharedInbox", "id_only": True, "single": True},
+        )
+    ]
+
+
+class PublicKeyField(jsonld.Serializer):
+    include_type = False
+    jsonld_fields = [
+        (
+            ns.SHORT["sec:owner"],
+            serializers.URLField(),
+            {"source": "owner", "single": True, "id_only": True},
+        ),
+        (
+            ns.SHORT["sec:publicKeyPem"],
+            serializers.CharField(),
+            {"source": "publicKeyPem", "single": True, "value_only": True},
+        ),
+    ]
+
+
+class ActorSerializer(jsonld.Serializer):
+    id_source = "url"
+    type_source = "type_ap"
+    valid_types = [
+        ns.SHORT["as:Application"],
+        ns.SHORT["as:Group"],
+        ns.SHORT["as:Organization"],
+        ns.SHORT["as:Person"],
+        ns.SHORT["as:Service"],
+    ]
+    jsonld_fields = [
+        (
+            ns.SHORT["as:outbox"],
+            serializers.URLField(max_length=500),
+            {"source": "outbox_url", "id_only": True, "single": True},
+        ),
+        (
+            ns.SHORT["ldp:inbox"],
+            serializers.URLField(max_length=500),
+            {"source": "inbox_url", "id_only": True, "single": True},
+        ),
+        (
+            ns.SHORT["as:preferredUsername"],
+            serializers.CharField(),
+            {"source": "preferred_username", "value_only": True, "single": True},
+        ),
+        (
+            ns.SHORT["as:manuallyApprovesFollowers"],
+            serializers.NullBooleanField(required=False),
+            {
+                "source": "manually_approves_followers",
+                "value_only": True,
+                "single": True,
+            },
+        ),
+        (
+            ns.SHORT["as:name"],
+            serializers.CharField(required=False, max_length=200),
+            {"source": "name", "value_only": True, "single": True},
+        ),
+        (
+            ns.SHORT["as:summary"],
+            serializers.CharField(max_length=None, required=False),
+            {"source": "summary", "value_only": True, "single": True},
+        ),
+        (
+            ns.SHORT["as:followers"],
+            serializers.URLField(max_length=500, required=False, allow_null=True),
+            {"source": "followers_url", "id_only": True, "single": True},
+        ),
+        (
+            ns.SHORT["as:following"],
+            serializers.URLField(max_length=500, required=False, allow_null=True),
+            {"source": "following_url", "id_only": True, "single": True},
+        ),
+        (
+            ns.SHORT["sec:publicKey"],
+            serializers.JSONField(required=False),
+            {"source": "public_key_repr", "value_only": True, "single": True},
+        ),
+        (
+            ns.SHORT["as:endpoints"],
+            EndpointsField(required=False),
+            {"source": "endpoints", "single": True},
+        ),
+        (
+            ns.SHORT["sec:publicKey"],
+            PublicKeyField(required=False),
+            {"source": "public_key_repr", "single": True},
+        ),
+    ]
 
     def prepare_missing_fields(self):
         kwargs = {
-            "url": self.validated_data["id"],
-            "outbox_url": self.validated_data["outbox"],
-            "inbox_url": self.validated_data["inbox"],
-            "following_url": self.validated_data.get("following"),
-            "followers_url": self.validated_data.get("followers"),
-            "summary": self.validated_data.get("summary"),
-            "type": self.validated_data["type"],
-            "name": self.validated_data.get("name"),
-            "preferred_username": self.validated_data["preferredUsername"],
+            "url": self.validated_data["url"],
+            "outbox_url": self.validated_data[ns.SHORT["as:outbox"]],
+            "inbox_url": self.validated_data[ns.SHORT["ldp:inbox"]],
+            "following_url": self.validated_data.get(ns.SHORT["as:following"]),
+            "followers_url": self.validated_data.get(ns.SHORT["as:followers"]),
+            "summary": self.validated_data.get(ns.SHORT["as:summary"]),
+            "type": self.validated_data["type_ap"].split("#")[-1],
+            "name": self.validated_data.get(ns.SHORT["as:name"]),
+            "preferred_username": self.validated_data[ns.SHORT["as:preferredUsername"]],
+            "manually_approves_followers": self.validated_data.get(
+                ns.SHORT["as:manuallyApprovesFollowers"]
+            ),
         }
-        maf = self.validated_data.get("manuallyApprovesFollowers")
-        if maf is not None:
-            kwargs["manually_approves_followers"] = maf
         domain = urllib.parse.urlparse(kwargs["url"]).netloc
         kwargs["domain"] = domain
-        for endpoint, url in self.initial_data.get("endpoints", {}).items():
-            if endpoint == "sharedInbox":
-                kwargs["shared_inbox_url"] = url
+        for endpoint, url in self.validated_data.get(
+            ns.SHORT["as:endpoints"], {}
+        ).items():
+            if endpoint == ns.SHORT["as:sharedInbox"]:
+                kwargs["shared_inbox_url"] = url[0]["@id"]
                 break
+        public_key_data = self.validated_data.get(ns.SHORT["sec:publicKey"], {})
         try:
-            kwargs["public_key"] = self.initial_data["publicKey"]["publicKeyPem"]
-        except KeyError:
+            kwargs["public_key"] = public_key_data[ns.SHORT["sec:publicKeyPem"]][0][
+                "@value"
+            ]
+        except (KeyError, IndexError):
             pass
         return kwargs
 
@@ -124,22 +185,22 @@ class APIActorSerializer(serializers.ModelSerializer):
 
 
 class LibraryActorSerializer(ActorSerializer):
-    url = serializers.ListField(child=serializers.JSONField())
+    # url = serializers.ListField(child=serializers.JSONField())
+    jsonld_fields = ActorSerializer.jsonld_fields + [
+        (
+            ns.SHORT["as:url"],
+            jsonld.Serializer(required=True),
+            {"source": "library_url"},
+        )
+    ]
 
     def validate(self, validated_data):
-        try:
-            urls = validated_data["url"]
-        except KeyError:
-            raise serializers.ValidationError("Missing URL field")
-
-        for u in urls:
-            try:
-                if u["name"] != "library":
-                    continue
-                validated_data["library_url"] = u["href"]
-                break
-            except KeyError:
+        validated_data = super().validate(validated_data)
+        for url in validated_data[ns.SHORT["as:url"]]:
+            if url[ns.SHORT["as:name"]][0]["@value"] != "library":
                 continue
+            validated_data["library_url"] = url[ns.SHORT["as:href"]][0]["@id"]
+            break
 
         return validated_data
 
@@ -306,18 +367,37 @@ class APILibraryTrackSerializer(serializers.ModelSerializer):
         return "not_imported"
 
 
-class FollowSerializer(serializers.Serializer):
-    id = serializers.URLField(max_length=500)
-    object = serializers.URLField(max_length=500)
-    actor = serializers.URLField(max_length=500)
-    type = serializers.ChoiceField(choices=["Follow"])
+class FollowSerializer(jsonld.Serializer):
+    valid_types = [ns.SHORT["as:Follow"]]
+    jsonld_fields = [
+        (
+            ns.SHORT["as:actor"],
+            ActorSerializer(required=True),
+            {"source": "actor", "single": True},
+        ),
+        (
+            ns.SHORT["as:object"],
+            ActorSerializer(required=True),
+            {"source": "object", "single": True},
+        ),
+    ]
+
+    def validate(self, validated_data):
+        validated_data = super().validate(validated_data)
+        validated_data[ns.SHORT["as:object"]] = self.validate_object(
+            validated_data[ns.SHORT["as:object"]]
+        )
+        validated_data[ns.SHORT["as:actor"]] = self.validate_actor(
+            validated_data[ns.SHORT["as:actor"]]
+        )
+        return validated_data
 
     def validate_object(self, v):
         expected = self.context.get("follow_target")
         if expected and expected.url != v:
             raise serializers.ValidationError("Invalid target")
         try:
-            return models.Actor.objects.get(url=v)
+            return models.Actor.objects.get(url=v["@id"])
         except models.Actor.DoesNotExist:
             raise serializers.ValidationError("Target not found")
 
@@ -326,20 +406,20 @@ class FollowSerializer(serializers.Serializer):
         if expected and expected.url != v:
             raise serializers.ValidationError("Invalid actor")
         try:
-            return models.Actor.objects.get(url=v)
+            return models.Actor.objects.get(url=v["@id"])
         except models.Actor.DoesNotExist:
             raise serializers.ValidationError("Actor not found")
 
     def save(self, **kwargs):
         return models.Follow.objects.get_or_create(
-            actor=self.validated_data["actor"],
-            target=self.validated_data["object"],
+            actor=self.validated_data[ns.SHORT["as:actor"]],
+            target=self.validated_data[ns.SHORT["as:object"]],
             **kwargs,  # noqa
         )[0]
 
     def to_representation(self, instance):
         return {
-            "@context": AP_CONTEXT,
+            "@context": ns.CONTEXTS,
             "actor": instance.actor.url,
             "id": instance.get_federation_url(),
             "object": instance.target.url,
@@ -364,30 +444,45 @@ class APIFollowSerializer(serializers.ModelSerializer):
         ]
 
 
-class AcceptFollowSerializer(serializers.Serializer):
-    id = serializers.URLField(max_length=500)
-    actor = serializers.URLField(max_length=500)
-    object = FollowSerializer()
-    type = serializers.ChoiceField(choices=["Accept"])
+class AcceptFollowSerializer(jsonld.Serializer):
+    valid_types = [ns.SHORT["as:Accept"]]
+    jsonld_fields = [
+        (
+            ns.SHORT["as:actor"],
+            ActorSerializer(required=True),
+            {"source": "actor", "single": True},
+        ),
+        (
+            ns.SHORT["as:object"],
+            FollowSerializer(required=True),
+            {"source": "object", "single": True},
+        ),
+    ]
 
     def validate_actor(self, v):
         expected = self.context.get("follow_target")
         if expected and expected.url != v:
             raise serializers.ValidationError("Invalid actor")
         try:
-            return models.Actor.objects.get(url=v)
+            return models.Actor.objects.get(url=v["@id"])
         except models.Actor.DoesNotExist:
             raise serializers.ValidationError("Actor not found")
 
     def validate(self, validated_data):
+        import ipdb; ipdb.set_trace()
+        validated_data = super().validate(validated_data)
+        validated_data[ns.SHORT["as:actor"]] = self.validate_actor(
+            validated_data[ns.SHORT["as:actor"]]
+        )
+        follow_target = validated_data[ns.SHORT["as:object"]][ns.SHORT["as:object"]]
+        follow_actor = validated_data[ns.SHORT["as:object"]][ns.SHORT["as:actor"]]
         # we ensure the accept actor actually match the follow target
-        if validated_data["actor"] != validated_data["object"]["object"]:
+        if validated_data[ns.SHORT["as:actor"]].url != follow_target["@id"]:
             raise serializers.ValidationError("Actor mismatch")
         try:
             validated_data["follow"] = (
                 models.Follow.objects.filter(
-                    target=validated_data["actor"],
-                    actor=validated_data["object"]["actor"],
+                    target=validated_data[ns.SHORT["as:actor"]], actor=follow_actor
                 )
                 .exclude(approved=True)
                 .get()

api/requirements/base.txt

@@ -66,3 +66,4 @@ cryptography>=2,<3
 # clone until the branch is merged and released upstream
 git+https://github.com/EliotBerriot/requests-http-signature.git@signature-header-support
 django-cleanup==2.1.0
+pyld>=1,<1.1

api/tests/federation/test_authentication.py

-from funkwhale_api.federation import authentication, keys
+from funkwhale_api.federation import authentication, keys, ns
 
 
 def test_authenticate(factories, mocker, api_request):
@@ -7,10 +7,13 @@ def test_authenticate(factories, mocker, api_request):
     mocker.patch(
         "funkwhale_api.federation.actors.get_actor_data",
         return_value={
+            "@context": ns.CONTEXTS,
             "id": actor_url,
             "type": "Person",
             "outbox": "https://test.com",
             "inbox": "https://test.com",
+            "following": "https://test.com",
+            "followers": "https://test.com",
             "preferredUsername": "test",
             "publicKey": {
                 "publicKeyPem": public.decode("utf-8"),

api/tests/federation/test_jsonld.py

+import pytest
+from rest_framework import serializers
+from funkwhale_api.federation import jsonld
+from funkwhale_api.federation import ns
+
+
+def test_expand(requests_mock):
+    document = {
+        "@context": ns.NS["as"]["documentUrl"],
+        "type": "Person",
+        "summary": "Hello world",
+        "id": "http://test.document",
+    }
+
+    expanded = jsonld.expand(document)
+
+    assert expanded == [
+        {
+            "@type": [ns.SHORT["as:Person"]],
+            ns.SHORT["as:summary"]: [{"@value": "Hello world"}],
+            "@id": document["id"],
+        }
+    ]
+
+
+def test_register_validator():
+    def validator(value):
+        pass
+
+    ns.register_validator(["as:test", "hello:world"])(validator)
+
+    assert ns.VALIDATORS["as:test"] == [validator]
+    assert ns.VALIDATORS["hello:world"] == [validator]
+
+
+@pytest.mark.parametrize(
+    "field_conf,expected",
+    [
+        ({}, [{"@value": "Hello world"}]),
+        ({"single": True}, {"@value": "Hello world"}),
+        ({"value_only": True}, ["Hello world"]),
+        ({"value_only": True, "single": True}, "Hello world"),
+    ],
+)
+def test_json_ld_serializer_validation_value(field_conf, expected):
+    document = {
+        "@context": ns.NS["as"]["documentUrl"],
+        "type": "Person",
+        "summary": "Hello world",
+        "id": "http://test.document",
+    }
+
+    class Serializer(jsonld.Serializer):
+        jsonld_fields = [(ns.SHORT["as:summary"], serializers.CharField(), field_conf)]
+
+    serializer = Serializer(data=document)
+    assert serializer.is_valid(raise_exception=True)
+
+    assert serializer.validated_data == {
+        "@type": ns.SHORT["as:Person"],
+        ns.SHORT["as:summary"]: expected,
+        "@id": document["id"],
+    }
+
+
+@pytest.mark.parametrize(
+    "field_conf,expected",
+    [
+        ({}, [{"@id": "http://test"}]),
+        ({"single": True}, {"@id": "http://test"}),
+        ({"id_only": True}, ["http://test"]),
+        ({"id_only": True, "single": True}, "http://test"),
+    ],
+)
+def test_json_ld_serializer_validation_id(field_conf, expected):
+    document = {
+        "@context": ns.NS["as"]["documentUrl"],
+        "type": "Person",
+        "outbox": "http://test",
+        "id": "http://test.document",
+    }
+
+    class Serializer(jsonld.Serializer):
+        jsonld_fields = [(ns.SHORT["as:outbox"], serializers.URLField(), field_conf)]
+
+    serializer = Serializer(data=document)
+    assert serializer.is_valid(raise_exception=True)
+
+    assert serializer.validated_data == {
+        "@type": ns.SHORT["as:Person"],
+        ns.SHORT["as:outbox"]: expected,
+        "@id": document["id"],
+    }
+
+
+def test_json_ld_validates_type_failing():
+    document = {
+        "@context": ns.NS["as"]["documentUrl"],
+        "type": "Disallowed",
+        "outbox": "http://test",
+        "id": "http://test.document",
+    }
+
+    class Serializer(jsonld.Serializer):
+        valid_types = [ns.SHORT["as:Person"]]
+
+    serializer = Serializer(data=document)
+    assert serializer.is_valid() is False
+    assert "@type" in serializer.errors
+
+
+def test_json_ld_validates_type_success():
+    document = {
+        "@context": ns.NS["as"]["documentUrl"],
+        "type": "Person",
+        "outbox": "http://test",
+        "id": "http://test.document",
+    }
+
+    class Serializer(jsonld.Serializer):
+        valid_types = [ns.SHORT["as:Person"]]
+
+    serializer = Serializer(data=document)
+    assert serializer.is_valid(raise_exception=True) is True
+
+
+def test_json_ld_serializer_to_representation():
+    class Serializer(jsonld.Serializer):
+        jsonld_fields = [
+            (ns.SHORT["as:summary"], serializers.CharField(), {"source": "my_summary"})
+        ]
+        type_source = "my_type_field"
+        id_source = "my_id_field"
+
+    serializer = Serializer(
+        {
+            "my_type_field": ns.SHORT["as:Person"],
+            "my_id_field": "http://hello.world",
+            "my_summary": "Hello",
+        }
+    )
+
+    expected = {
+        "type": "Person",
+        "summary": "Hello",
+        "id": "http://hello.world",
+        "@context": [
+            "http://www.w3.org/ns/ldp",
+            "https://www.w3.org/ns/activitystreams",
+            "https://w3id.org/security/v1",
+            ns.ADDITIONAL_CONTEXT,
+        ],
+    }
+    assert serializer.data == expected
+
+
+def test_remove_semicolon():
+    struct = {
+        "@context": {"this:is": "skipped"},
+        "as:inbox": "https://test1",
+        "some:other": {"prop:erty": ["hello", {"wo:rld": "hi"}]},
+    }
+    expected = {
+        "@context": {"this:is": "skipped"},
+        "inbox": "https://test1",
+        "other": {"erty": ["hello", {"rld": "hi"}]},
+    }
+    result = jsonld.remove_semicolons(struct)
+
+    assert result == expected

api/tests/federation/test_serializers.py

@@ -2,11 +2,12 @@ import arrow
 import pytest
 from django.core.paginator import Paginator
 
-from funkwhale_api.federation import actors, models, serializers, utils
+from funkwhale_api.federation import actors, ns, models, serializers, utils
 
 
 def test_actor_serializer_from_ap(db):
     payload = {
+        "@context": ns.CONTEXTS,
         "id": "https://test.federation/user",
         "type": "Person",
         "following": "https://test.federation/user/following",
@@ -48,6 +49,7 @@ def test_actor_serializer_from_ap(db):
 
 def test_actor_serializer_only_mandatory_field_from_ap(db):
     payload = {
+        "@context": ns.CONTEXTS,
         "id": "https://test.federation/user",
         "type": "Person",
         "following": "https://test.federation/user/following",
@@ -75,11 +77,7 @@ def test_actor_serializer_only_mandatory_field_from_ap(db):
 
 def test_actor_serializer_to_ap():
     expected = {
-        "@context": [
-            "https://www.w3.org/ns/activitystreams",
-            "https://w3id.org/security/v1",
-            {},
-        ],
+        "@context": ns.CONTEXTS,
         "id": "https://test.federation/user",
         "type": "Person",
         "following": "https://test.federation/user/following",
@@ -144,11 +142,7 @@ def test_follow_serializer_to_ap(factories):
     serializer = serializers.FollowSerializer(follow)
 
     expected = {
-        "@context": [
-            "https://www.w3.org/ns/activitystreams",
-            "https://w3id.org/security/v1",
-            {},
-        ],
+        "@context": ns.CONTEXTS,
         "id": follow.get_federation_url(),
         "type": "Follow",
         "actor": follow.actor.url,
@@ -163,6 +157,7 @@ def test_follow_serializer_save(factories):
     target = factories["federation.Actor"]()
 
     data = {
+        "@context": ns.CONTEXTS,
         "id": "https://test.follow",
         "type": "Follow",
         "actor": actor.url,
@@ -186,6 +181,7 @@ def test_follow_serializer_save_validates_on_context(factories):
     impostor = factories["federation.Actor"]()
 
     data = {
+        "@context": ns.CONTEXTS,
         "id": "https://test.follow",
         "type": "Follow",
         "actor": actor.url,
@@ -205,11 +201,7 @@ def test_accept_follow_serializer_representation(factories):
     follow = factories["federation.Follow"](approved=None)
 
     expected = {
-        "@context": [
-            "https://www.w3.org/ns/activitystreams",
-            "https://w3id.org/security/v1",
-            {},
-        ],
+        "@context": ns.CONTEXTS,
         "id": follow.get_federation_url() + "/accept",
         "type": "Accept",
         "actor": follow.target.url,
@@ -225,11 +217,7 @@ def test_accept_follow_serializer_save(factories):
     follow = factories["federation.Follow"](approved=None)
 
     data = {
-        "@context": [
-            "https://www.w3.org/ns/activitystreams",
-            "https://w3id.org/security/v1",
-            {},
-        ],
+        "@context": ns.CONTEXTS,
         "id": follow.get_federation_url() + "/accept",
         "type": "Accept",
         "actor": follow.target.url,
@@ -249,11 +237,7 @@ def test_accept_follow_serializer_validates_on_context(factories):
     follow = factories["federation.Follow"](approved=None)
     impostor = factories["federation.Actor"]()
     data = {
-        "@context": [
-            "https://www.w3.org/ns/activitystreams",
-            "https://w3id.org/security/v1",
-            {},
-        ],
+        "@context": ns.CONTEXTS,
         "id": follow.get_federation_url() + "/accept",
         "type": "Accept",
         "actor": impostor.url,
@@ -273,11 +257,7 @@ def test_undo_follow_serializer_representation(factories):
     follow = factories["federation.Follow"](approved=True)
 
     expected = {
-        "@context": [
-            "https://www.w3.org/ns/activitystreams",
-            "https://w3id.org/security/v1",
-            {},
-        ],
+        "@context": ns.CONTEXTS,
         "id": follow.get_federation_url() + "/undo",
         "type": "Undo",
         "actor": follow.actor.url,
@@ -293,11 +273,7 @@ def test_undo_follow_serializer_save(factories):
     follow = factories["federation.Follow"](approved=True)
 
     data = {
-        "@context": [
-            "https://www.w3.org/ns/activitystreams",
-            "https://w3id.org/security/v1",
-            {},
-        ],
+        "@context": ns.CONTEXTS,
         "id": follow.get_federation_url() + "/undo",
         "type": "Undo",
         "actor": follow.actor.url,
@@ -316,11 +292,7 @@ def test_undo_follow_serializer_validates_on_context(factories):
     follow = factories["federation.Follow"](approved=True)
     impostor = factories["federation.Actor"]()
     data = {
-        "@context": [
-            "https://www.w3.org/ns/activitystreams",
-            "https://w3id.org/security/v1",
-            {},
-        ],
+        "@context": ns.CONTEXTS,
         "id": follow.get_federation_url() + "/undo",
         "type": "Undo",
         "actor": impostor.url,