Skip to content
GitLab
Commit 12c9db39 authored by Eliot Berriot's avatar Eliot Berriot
Browse files

Ensures federation urls can answer to application/ld+json,... 

Ensures federation urls can answer to application/ld+json, application/activity+json and application/json requests
parent c9e7eea6
Expand all Hide whitespace changes
Inline Side-by-side
api/funkwhale_api/common/utils.py
View file @ 12c9db39
......@@ -193,3 +193,11 @@ def replace_prefix(queryset, field, old, new):
models.functions.Substr(field, len(old) + 1, output_field=models.CharField()),
)
return qs.update(**{field: update})
def concat_dicts(*dicts):
n = {}
for d in dicts:
n.update(d)
return n
api/funkwhale_api/federation/activity.py
View file @ 12c9db39
......@@ -9,11 +9,13 @@ from django.db.models import Q
from funkwhale_api.common import channels
from funkwhale_api.common import utils as funkwhale_utils
from . import contexts
recursive_getattr = funkwhale_utils.recursive_getattr
logger = logging.getLogger(__name__)
PUBLIC_ADDRESS = "https://www.w3.org/ns/activitystreams#Public"
PUBLIC_ADDRESS = contexts.AS.Public
ACTIVITY_TYPES = [
"Accept",
......@@ -84,7 +86,10 @@ OBJECT_TYPES = (
BROADCAST_TO_USER_ACTIVITIES = ["Follow", "Accept"]
def should_reject(id, actor_id=None, payload={}):
def should_reject(fid, actor_id=None, payload={}):
if fid is None and actor_id is None:
return False
from funkwhale_api.moderation import models as moderation_models
policies = moderation_models.InstancePolicy.objects.active()
......@@ -102,9 +107,12 @@ def should_reject(id, actor_id=None, payload={}):
else:
policy_type = Q(block_all=True)
query = policies.matching_url_query(id) & policy_type
if actor_id:
if fid:
query = policies.matching_url_query(fid) & policy_type
if fid and actor_id:
query |= policies.matching_url_query(actor_id) & policy_type
elif actor_id:
query = policies.matching_url_query(actor_id) & policy_type
return policies.filter(query).exists()
......@@ -121,7 +129,7 @@ def receive(activity, on_behalf_of):
)
serializer.is_valid(raise_exception=True)
if should_reject(
id=serializer.validated_data["id"],
fid=serializer.validated_data.get("id"),
actor_id=serializer.validated_data["actor"].fid,
payload=activity,
):
......
api/funkwhale_api/federation/contexts.py 0 → 100644
View file @ 12c9db39
CONTEXTS = [
{
"shortId": "LDP",
"contextUrl": None,
"documentUrl": "http://www.w3.org/ns/ldp",
"document": {
"@context": {
"ldp": "http://www.w3.org/ns/ldp#",
"id": "@id",
"type": "@type",
"Container": "ldp:Container",
"BasicContainer": "ldp:BasicContainer",
"DirectContainer": "ldp:DirectContainer",
"IndirectContainer": "ldp:IndirectContainer",
"hasMemberRelation": {"@id": "ldp:hasMemberRelation", "@type": "@id"},
"isMemberOfRelation": {"@id": "ldp:isMemberOfRelation", "@type": "@id"},
"membershipResource": {"@id": "ldp:membershipResource", "@type": "@id"},
"insertedContentRelation": {
"@id": "ldp:insertedContentRelation",
"@type": "@id",
},
"contains": {"@id": "ldp:contains", "@type": "@id"},
"member": {"@id": "ldp:member", "@type": "@id"},
"constrainedBy": {"@id": "ldp:constrainedBy", "@type": "@id"},
"Resource": "ldp:Resource",
"RDFSource": "ldp:RDFSource",
"NonRDFSource": "ldp:NonRDFSource",
"MemberSubject": "ldp:MemberSubject",
"PreferContainment": "ldp:PreferContainment",
"PreferMembership": "ldp:PreferMembership",
"PreferMinimalContainer": "ldp:PreferMinimalContainer",
"PageSortCriterion": "ldp:PageSortCriterion",
"pageSortCriteria": {
"@id": "ldp:pageSortCriteria",
"@type": "@id",
"@container": "@list",
},
"pageSortPredicate": {"@id": "ldp:pageSortPredicate", "@type": "@id"},
"pageSortOrder": {"@id": "ldp:pageSortOrder", "@type": "@id"},
"pageSortCollation": {"@id": "ldp:pageSortCollation", "@type": "@id"},
"Ascending": "ldp:Ascending",
"Descending": "ldp:Descending",
"Page": "ldp:Page",
"pageSequence": {"@id": "ldp:pageSequence", "@type": "@id"},
"inbox": {"@id": "ldp:inbox", "@type": "@id"},
}
},
},
{
"shortId": "AS",
"contextUrl": None,
"documentUrl": "https://www.w3.org/ns/activitystreams",
"document": {
"@context": {
"@vocab": "_:",
"xsd": "http://www.w3.org/2001/XMLSchema#",
"as": "https://www.w3.org/ns/activitystreams#",
"ldp": "http://www.w3.org/ns/ldp#",
"id": "@id",
"type": "@type",
"Accept": "as:Accept",
"Activity": "as:Activity",
"IntransitiveActivity": "as:IntransitiveActivity",
"Add": "as:Add",
"Announce": "as:Announce",
"Application": "as:Application",
"Arrive": "as:Arrive",
"Article": "as:Article",
"Audio": "as:Audio",
"Block": "as:Block",
"Collection": "as:Collection",
"CollectionPage": "as:CollectionPage",
"Relationship": "as:Relationship",
"Create": "as:Create",
"Delete": "as:Delete",
"Dislike": "as:Dislike",
"Document": "as:Document",
"Event": "as:Event",
"Follow": "as:Follow",
"Flag": "as:Flag",
"Group": "as:Group",
"Ignore": "as:Ignore",
"Image": "as:Image",
"Invite": "as:Invite",
"Join": "as:Join",
"Leave": "as:Leave",
"Like": "as:Like",
"Link": "as:Link",
"Mention": "as:Mention",
"Note": "as:Note",
"Object": "as:Object",
"Offer": "as:Offer",
"OrderedCollection": "as:OrderedCollection",
"OrderedCollectionPage": "as:OrderedCollectionPage",
"Organization": "as:Organization",
"Page": "as:Page",
"Person": "as:Person",
"Place": "as:Place",
"Profile": "as:Profile",
"Question": "as:Question",
"Reject": "as:Reject",
"Remove": "as:Remove",
"Service": "as:Service",
"TentativeAccept": "as:TentativeAccept",
"TentativeReject": "as:TentativeReject",
"Tombstone": "as:Tombstone",
"Undo": "as:Undo",
"Update": "as:Update",
"Video": "as:Video",
"View": "as:View",
"Listen": "as:Listen",
"Read": "as:Read",
"Move": "as:Move",
"Travel": "as:Travel",
"IsFollowing": "as:IsFollowing",
"IsFollowedBy": "as:IsFollowedBy",
"IsContact": "as:IsContact",
"IsMember": "as:IsMember",
"subject": {"@id": "as:subject", "@type": "@id"},
"relationship": {"@id": "as:relationship", "@type": "@id"},
"actor": {"@id": "as:actor", "@type": "@id"},
"attributedTo": {"@id": "as:attributedTo", "@type": "@id"},
"attachment": {"@id": "as:attachment", "@type": "@id"},
"bcc": {"@id": "as:bcc", "@type": "@id"},
"bto": {"@id": "as:bto", "@type": "@id"},
"cc": {"@id": "as:cc", "@type": "@id"},
"context": {"@id": "as:context", "@type": "@id"},
"current": {"@id": "as:current", "@type": "@id"},
"first": {"@id": "as:first", "@type": "@id"},
"generator": {"@id": "as:generator", "@type": "@id"},
"icon": {"@id": "as:icon", "@type": "@id"},
"image": {"@id": "as:image", "@type": "@id"},
"inReplyTo": {"@id": "as:inReplyTo", "@type": "@id"},
"items": {"@id": "as:items", "@type": "@id"},
"instrument": {"@id": "as:instrument", "@type": "@id"},
"orderedItems": {
"@id": "as:items",
"@type": "@id",
"@container": "@list",
},
"last": {"@id": "as:last", "@type": "@id"},
"location": {"@id": "as:location", "@type": "@id"},
"next": {"@id": "as:next", "@type": "@id"},
"object": {"@id": "as:object", "@type": "@id"},
"oneOf": {"@id": "as:oneOf", "@type": "@id"},
"anyOf": {"@id": "as:anyOf", "@type": "@id"},
"closed": {"@id": "as:closed", "@type": "xsd:dateTime"},
"origin": {"@id": "as:origin", "@type": "@id"},
"accuracy": {"@id": "as:accuracy", "@type": "xsd:float"},
"prev": {"@id": "as:prev", "@type": "@id"},
"preview": {"@id": "as:preview", "@type": "@id"},
"replies": {"@id": "as:replies", "@type": "@id"},
"result": {"@id": "as:result", "@type": "@id"},
"audience": {"@id": "as:audience", "@type": "@id"},
"partOf": {"@id": "as:partOf", "@type": "@id"},
"tag": {"@id": "as:tag", "@type": "@id"},
"target": {"@id": "as:target", "@type": "@id"},
"to": {"@id": "as:to", "@type": "@id"},
"url": {"@id": "as:url", "@type": "@id"},
"altitude": {"@id": "as:altitude", "@type": "xsd:float"},
"content": "as:content",
"contentMap": {"@id": "as:content", "@container": "@language"},
"name": "as:name",
"nameMap": {"@id": "as:name", "@container": "@language"},
"duration": {"@id": "as:duration", "@type": "xsd:duration"},
"endTime": {"@id": "as:endTime", "@type": "xsd:dateTime"},
"height": {"@id": "as:height", "@type": "xsd:nonNegativeInteger"},
"href": {"@id": "as:href", "@type": "@id"},
"hreflang": "as:hreflang",
"latitude": {"@id": "as:latitude", "@type": "xsd:float"},
"longitude": {"@id": "as:longitude", "@type": "xsd:float"},
"mediaType": "as:mediaType",
"published": {"@id": "as:published", "@type": "xsd:dateTime"},
"radius": {"@id": "as:radius", "@type": "xsd:float"},
"rel": "as:rel",
"startIndex": {
"@id": "as:startIndex",
"@type": "xsd:nonNegativeInteger",
},
"startTime": {"@id": "as:startTime", "@type": "xsd:dateTime"},
"summary": "as:summary",
"summaryMap": {"@id": "as:summary", "@container": "@language"},
"totalItems": {
"@id": "as:totalItems",
"@type": "xsd:nonNegativeInteger",
},
"units": "as:units",
"updated": {"@id": "as:updated", "@type": "xsd:dateTime"},
"width": {"@id": "as:width", "@type": "xsd:nonNegativeInteger"},
"describes": {"@id": "as:describes", "@type": "@id"},
"formerType": {"@id": "as:formerType", "@type": "@id"},
"deleted": {"@id": "as:deleted", "@type": "xsd:dateTime"},
"inbox": {"@id": "ldp:inbox", "@type": "@id"},
"outbox": {"@id": "as:outbox", "@type": "@id"},
"following": {"@id": "as:following", "@type": "@id"},
"followers": {"@id": "as:followers", "@type": "@id"},
"streams": {"@id": "as:streams", "@type": "@id"},
"preferredUsername": "as:preferredUsername",
"endpoints": {"@id": "as:endpoints", "@type": "@id"},
"uploadMedia": {"@id": "as:uploadMedia", "@type": "@id"},
"proxyUrl": {"@id": "as:proxyUrl", "@type": "@id"},
"liked": {"@id": "as:liked", "@type": "@id"},
"oauthAuthorizationEndpoint": {
"@id": "as:oauthAuthorizationEndpoint",
"@type": "@id",
},
"oauthTokenEndpoint": {"@id": "as:oauthTokenEndpoint", "@type": "@id"},
"provideClientKey": {"@id": "as:provideClientKey", "@type": "@id"},
"signClientKey": {"@id": "as:signClientKey", "@type": "@id"},
"sharedInbox": {"@id": "as:sharedInbox", "@type": "@id"},
"Public": {"@id": "as:Public", "@type": "@id"},
"source": "as:source",
"likes": {"@id": "as:likes", "@type": "@id"},
"shares": {"@id": "as:shares", "@type": "@id"},
# Added manually
"manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
}
},
},
{
"shortId": "SEC",
"contextUrl": None,
"documentUrl": "https://w3id.org/security/v1",
"document": {
"@context": {
"id": "@id",
"type": "@type",
"dc": "http://purl.org/dc/terms/",
"sec": "https://w3id.org/security#",
"xsd": "http://www.w3.org/2001/XMLSchema#",
"EcdsaKoblitzSignature2016": "sec:EcdsaKoblitzSignature2016",
"Ed25519Signature2018": "sec:Ed25519Signature2018",
"EncryptedMessage": "sec:EncryptedMessage",
"GraphSignature2012": "sec:GraphSignature2012",
"LinkedDataSignature2015": "sec:LinkedDataSignature2015",
"LinkedDataSignature2016": "sec:LinkedDataSignature2016",
"CryptographicKey": "sec:Key",
"authenticationTag": "sec:authenticationTag",
"canonicalizationAlgorithm": "sec:canonicalizationAlgorithm",
"cipherAlgorithm": "sec:cipherAlgorithm",
"cipherData": "sec:cipherData",
"cipherKey": "sec:cipherKey",
"created": {"@id": "dc:created", "@type": "xsd:dateTime"},
"creator": {"@id": "dc:creator", "@type": "@id"},
"digestAlgorithm": "sec:digestAlgorithm",
"digestValue": "sec:digestValue",
"domain": "sec:domain",
"encryptionKey": "sec:encryptionKey",
"expiration": {"@id": "sec:expiration", "@type": "xsd:dateTime"},
"expires": {"@id": "sec:expiration", "@type": "xsd:dateTime"},
"initializationVector": "sec:initializationVector",
"iterationCount": "sec:iterationCount",
"nonce": "sec:nonce",
"normalizationAlgorithm": "sec:normalizationAlgorithm",
"owner": {"@id": "sec:owner", "@type": "@id"},
"password": "sec:password",
"privateKey": {"@id": "sec:privateKey", "@type": "@id"},
"privateKeyPem": "sec:privateKeyPem",
"publicKey": {"@id": "sec:publicKey", "@type": "@id"},
"publicKeyBase58": "sec:publicKeyBase58",
"publicKeyPem": "sec:publicKeyPem",
"publicKeyWif": "sec:publicKeyWif",
"publicKeyService": {"@id": "sec:publicKeyService", "@type": "@id"},
"revoked": {"@id": "sec:revoked", "@type": "xsd:dateTime"},
"salt": "sec:salt",
"signature": "sec:signature",
"signatureAlgorithm": "sec:signingAlgorithm",
"signatureValue": "sec:signatureValue",
}
},
},
{
"shortId": "FW",
"contextUrl": None,
"documentUrl": "https://funkwhale.audio/ns",
"document": {
"@context": {
"id": "@id",
"type": "@type",
"as": "https://www.w3.org/ns/activitystreams#",
"fw": "https://funkwhale.audio/ns#",
"xsd": "http://www.w3.org/2001/XMLSchema#",
"Album": "fw:Album",
"Track": "fw:Track",
"Artist": "fw:Artist",
"Library": "fw:Library",
"bitrate": {"@id": "fw:bitrate", "@type": "xsd:nonNegativeInteger"},
"size": {"@id": "fw:size", "@type": "xsd:nonNegativeInteger"},
"position": {"@id": "fw:position", "@type": "xsd:nonNegativeInteger"},
"disc": {"@id": "fw:disc", "@type": "xsd:nonNegativeInteger"},
"library": {"@id": "fw:library", "@type": "@id"},
"track": {"@id": "fw:track", "@type": "@id"},
"cover": {"@id": "fw:cover", "@type": "as:Link"},
"album": {"@id": "fw:album", "@type": "@id"},
"artists": {"@id": "fw:artists", "@type": "@id", "@container": "@list"},
"released": {"@id": "fw:released", "@type": "xsd:date"},
"musicbrainzId": "fw:musicbrainzId",
"license": {"@id": "fw:license", "@type": "@id"},
"copyright": "fw:copyright",
}
},
},
]
CONTEXTS_BY_ID = {c["shortId"]: c for c in CONTEXTS}
class NS:
def __init__(self, conf):
self.conf = conf
self.baseUrl = self.conf["document"]["@context"][self.conf["shortId"].lower()]
def __repr__(self):
return "<{}: {}>".format(self.conf["shortId"], self.baseUrl)
def __getattr__(self, key):
if key not in self.conf["document"]["@context"]:
raise AttributeError(
"{} is not a valid property of context {}".format(key, self.baseUrl)
)
return self.baseUrl + key
class NoopContext:
def __getattr__(self, key):
return "_:{}".format(key)
NOOP = NoopContext()
AS = NS(CONTEXTS_BY_ID["AS"])
LDP = NS(CONTEXTS_BY_ID["LDP"])
SEC = NS(CONTEXTS_BY_ID["SEC"])
FW = NS(CONTEXTS_BY_ID["FW"])
api/funkwhale_api/federation/jsonld.py 0 → 100644
View file @ 12c9db39
import aiohttp
import asyncio
import functools
import pyld.jsonld
from django.conf import settings
import pyld.documentloader.requests
from rest_framework import serializers
from rest_framework.fields import empty
from . import contexts
def cached_contexts(loader):
functools.wraps(loader)
def load(url, *args, **kwargs):
for cached in contexts.CONTEXTS:
if url == cached["documentUrl"]:
return cached
return loader(url, *args, **kwargs)
return load
def get_document_loader():
loader = pyld.documentloader.requests.requests_document_loader(
verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL
)
return cached_contexts(loader)
def expand(doc, options=None, insert_fw_context=True):
options = options or {}
options.setdefault("documentLoader", get_document_loader())
if isinstance(doc, str):
doc = options["documentLoader"](doc)["document"]
if insert_fw_context:
fw = contexts.CONTEXTS_BY_ID["FW"]["documentUrl"]
try:
insert_context(fw, doc)
except KeyError:
# probably an already expanded document
pass
result = pyld.jsonld.expand(doc, options=options)
try:
# jsonld.expand returns a list, which is useless for us
return result[0]
except IndexError:
raise ValueError("Impossible to expand this jsonld document")
def insert_context(ctx, doc):
"""
In some situations, we may want to add a default context to an existing document.
This function enable that (this will mutate the original document)
"""
existing = doc["@context"]
if isinstance(existing, list):
if ctx not in existing:
existing.append(ctx)
else:
doc["@context"] = [existing, ctx]
return doc
def get_session():
return aiohttp.ClientSession(raise_for_status=True)
async def fetch_json(url, session, cache=None, lock=None):
async with session.get(url) as response:
response.raise_for_status()
return url, await response.json()
async def fetch_many(*ids, references=None):
"""
Given a list of object ids, will fetch the remote
representations for those objects, expand them
and return a dictionnary with id as the key and expanded document as the values
"""
ids = set(ids)
results = references if references is not None else {}
if not ids:
return results
async with get_session() as session:
tasks = [fetch_json(url, session) for url in ids if url not in results]
tasks_results = await asyncio.gather(*tasks)
for url, payload in tasks_results:
results[url] = payload
return results
DEFAULT_PREPARE_CONFIG = {
"type": {"property": "@type", "keep": "first"},
"id": {"property": "@id"},
}
def dereference(value, references):
"""
Given a payload and a dictonary containing ids and objects, will replace
all the matching objects in the payload by the one in the references dictionary.
"""
def replace(obj, id):
try:
matching = references[id]
except KeyError:
return
# we clear the current dict, and replace its content by the matching obj
obj.clear()
obj.update(matching)
if isinstance(value, dict):
if "@id" in value:
replace(value, value["@id"])
else:
for attr in value.values():
dereference(attr, references)
elif isinstance(value, list):
# we loop on nested objects and trigger dereferencing
for obj in value:
dereference(obj, references)
return value
def get_value(value, keep=None, attr=None):
if keep == "first":
value = value[0]
if attr:
value = value[attr]
elif attr:
value = [obj[attr] for obj in value if attr in obj]
return value
def prepare_for_serializer(payload, config, fallbacks={}):
"""
Json-ld payloads, as returned by expand are quite complex to handle, because
every attr is basically a list of dictionnaries. To make code simpler,
we use this function to clean the payload a little bit, base on the config object.
Config is a dictionnary, with keys being serializer field names, and values
being dictionaries describing how to handle this field.
"""
final_payload = {}
final_config = {}
final_config.update(DEFAULT_PREPARE_CONFIG)
final_config.update(config)
for field, field_config in final_config.items():
try:
value = get_value(
payload[field_config["property"]],
keep=field_config.get("keep"),
attr=field_config.get("attr"),
)
except (IndexError, KeyError):
aliases = field_config.get("aliases", [])
noop = object()
value = noop
if not aliases:
continue
for a in aliases:
try:
value = get_value(
payload[a],
keep=field_config.get("keep"),
attr=field_config.get("attr"),
)
except (IndexError, KeyError):
continue
break
if value is noop:
continue
final_payload[field] = value
for key, choices in fallbacks.items():
if key in final_payload: