Commit e49a4602 authored by Agate's avatar Agate 💬
Browse files

Audio federation

parent 6992c567
......@@ -249,6 +249,7 @@ Then, in separate terminals, you can setup as many different instances as you
need::
export COMPOSE_PROJECT_NAME=node2
export VUE_PORT=1234 # this has to be unique for each instance
docker-compose -f dev.yml run --rm api python manage.py migrate
docker-compose -f dev.yml run --rm api python manage.py createsuperuser
docker-compose -f dev.yml up nginx api front nginx api celeryworker
......
......@@ -14,7 +14,7 @@ router.register(r"settings", GlobalPreferencesViewSet, base_name="settings")
router.register(r"activity", activity_views.ActivityViewSet, "activity")
router.register(r"tags", views.TagViewSet, "tags")
router.register(r"tracks", views.TrackViewSet, "tracks")
router.register(r"track-files", views.TrackFileViewSet, "trackfiles")
router.register(r"uploads", views.UploadViewSet, "uploads")
router.register(r"libraries", views.LibraryViewSet, "libraries")
router.register(r"listen", views.ListenViewSet, "listen")
router.register(r"artists", views.ArtistViewSet, "artists")
......
......@@ -514,8 +514,14 @@ ACCOUNT_USERNAME_BLACKLIST = [
"me",
"ghost",
"_",
"-",
"hello",
"contact",
"inbox",
"outbox",
"shared-inbox",
"shared_inbox",
"actor",
] + env.list("ACCOUNT_USERNAME_BLACKLIST", default=[])
EXTERNAL_REQUESTS_VERIFY_SSL = env.bool("EXTERNAL_REQUESTS_VERIFY_SSL", default=True)
......
......@@ -9,7 +9,9 @@ from funkwhale_api.common import preferences
class ConditionalAuthentication(BasePermission):
def has_permission(self, request, view):
if preferences.get("common__api_authentication_required"):
return request.user and request.user.is_authenticated
return (request.user and request.user.is_authenticated) or (
hasattr(request, "actor") and request.actor
)
return True
......
......@@ -5,6 +5,12 @@ visibility.
Files without any import job will be bounded to a "default" library on the first
superuser account found. This should now happen though.
XXX TODO:
- add followers url on actor
- shared inbox url on actor
- compute hash from files
"""
from funkwhale_api.music import models
......@@ -19,7 +25,7 @@ def main(command, **kwargs):
command.stdout.write(
"* {} users imported music on this instance".format(len(importers))
)
files = models.TrackFile.objects.filter(
files = models.Upload.objects.filter(
library__isnull=True, jobs__isnull=False
).distinct()
command.stdout.write(
......@@ -39,7 +45,7 @@ def main(command, **kwargs):
)
user_files.update(library=library)
files = models.TrackFile.objects.filter(
files = models.Upload.objects.filter(
library__isnull=True, jobs__isnull=True
).distinct()
command.stdout.write(
......
......@@ -64,3 +64,46 @@ class ChunkedPath(object):
new_filename = "".join(chunks[3:]) + ".{}".format(ext)
parts = chunks[:3] + [new_filename]
return os.path.join(self.root, *parts)
def chunk_queryset(source_qs, chunk_size):
"""
From https://github.com/peopledoc/django-chunkator/blob/master/chunkator/__init__.py
"""
pk = None
# In django 1.9, _fields is always present and `None` if 'values()' is used
# In Django 1.8 and below, _fields will only be present if using `values()`
has_fields = hasattr(source_qs, "_fields") and source_qs._fields
if has_fields:
if "pk" not in source_qs._fields:
raise ValueError("The values() call must include the `pk` field")
field = source_qs.model._meta.pk
# set the correct field name:
# for ForeignKeys, we want to use `model_id` field, and not `model`,
# to bypass default ordering on related model
order_by_field = field.attname
source_qs = source_qs.order_by(order_by_field)
queryset = source_qs
while True:
if pk:
queryset = source_qs.filter(pk__gt=pk)
page = queryset[:chunk_size]
page = list(page)
nb_items = len(page)
if nb_items == 0:
return
last_item = page[-1]
# source_qs._fields exists *and* is not none when using "values()"
if has_fields:
pk = last_item["pk"]
else:
pk = last_item.pk
yield page
if nb_items < chunk_size:
return
......@@ -2,11 +2,12 @@ import uuid
import logging
from django.db import transaction, IntegrityError
from django.utils import timezone
from django.db.models import Q
from funkwhale_api.common import channels
from funkwhale_api.common import utils as funkwhale_utils
logger = logging.getLogger(__name__)
PUBLIC_ADDRESS = "https://www.w3.org/ns/activitystreams#Public"
......@@ -83,18 +84,21 @@ def receive(activity, on_behalf_of):
serializer.validated_data.get("id"),
)
return
# we create inbox items for further delivery
items = [
models.InboxItem(activity=copy, actor=r, type="to")
for r in serializer.validated_data["recipients"]["to"]
if hasattr(r, "fid")
]
items += [
models.InboxItem(activity=copy, actor=r, type="cc")
for r in serializer.validated_data["recipients"]["cc"]
if hasattr(r, "fid")
]
models.InboxItem.objects.bulk_create(items)
local_to_recipients = get_actors_from_audience(activity.get("to", []))
local_to_recipients = local_to_recipients.exclude(user=None)
local_cc_recipients = get_actors_from_audience(activity.get("cc", []))
local_cc_recipients = local_cc_recipients.exclude(user=None)
inbox_items = []
for recipients, type in [(local_to_recipients, "to"), (local_cc_recipients, "cc")]:
for r in recipients.values_list("pk", flat=True):
inbox_items.append(models.InboxItem(actor_id=r, type=type, activity=copy))
models.InboxItem.objects.bulk_create(inbox_items)
# at this point, we have the activity in database. Even if we crash, it's
# okay, as we can retry later
funkwhale_utils.on_commit(tasks.dispatch_inbox.delay, activity_id=copy.pk)
......@@ -153,6 +157,16 @@ class InboxRouter(Router):
inbox_items = context.get(
"inbox_items", models.InboxItem.objects.none()
)
inbox_items = (
inbox_items.select_related()
.select_related("actor__user")
.prefetch_related(
"activity__object",
"activity__target",
"activity__related_object",
)
)
for ii in inbox_items:
user = ii.actor.get_user()
if not user:
......@@ -169,7 +183,6 @@ class InboxRouter(Router):
},
},
)
inbox_items.update(is_delivered=True, last_delivery_date=timezone.now())
return
......@@ -185,73 +198,203 @@ class OutboxRouter(Router):
from . import tasks
for route, handler in self.routes:
if match_route(route, routing):
activities_data = []
for e in handler(context):
# a route can yield zero, one or more activity payloads
if e:
activities_data.append(e)
inbox_items_by_activity_uuid = {}
prepared_activities = []
for activity_data in activities_data:
to = activity_data["payload"].pop("to", [])
cc = activity_data["payload"].pop("cc", [])
a = models.Activity(**activity_data)
a.uuid = uuid.uuid4()
to_items, new_to = prepare_inbox_items(to, "to")
cc_items, new_cc = prepare_inbox_items(cc, "cc")
if not to_items and not cc_items:
continue
inbox_items_by_activity_uuid[str(a.uuid)] = to_items + cc_items
if new_to:
a.payload["to"] = new_to
if new_cc:
a.payload["cc"] = new_cc
prepared_activities.append(a)
activities = models.Activity.objects.bulk_create(prepared_activities)
final_inbox_items = []
for a in activities:
try:
prepared_inbox_items = inbox_items_by_activity_uuid[str(a.uuid)]
except KeyError:
continue
for ii in prepared_inbox_items:
ii.activity = a
final_inbox_items.append(ii)
# create all inbox items, in bulk
models.InboxItem.objects.bulk_create(final_inbox_items)
for a in activities:
funkwhale_utils.on_commit(
tasks.dispatch_outbox.delay, activity_id=a.pk
)
return activities
if not match_route(route, routing):
continue
activities_data = []
for e in handler(context):
# a route can yield zero, one or more activity payloads
if e:
activities_data.append(e)
inbox_items_by_activity_uuid = {}
deliveries_by_activity_uuid = {}
prepared_activities = []
for activity_data in activities_data:
activity_data["payload"]["actor"] = activity_data["actor"].fid
to = activity_data["payload"].pop("to", [])
cc = activity_data["payload"].pop("cc", [])
a = models.Activity(**activity_data)
a.uuid = uuid.uuid4()
to_inbox_items, to_deliveries, new_to = prepare_deliveries_and_inbox_items(
to, "to"
)
cc_inbox_items, cc_deliveries, new_cc = prepare_deliveries_and_inbox_items(
cc, "cc"
)
if not any(
[to_inbox_items, to_deliveries, cc_inbox_items, cc_deliveries]
):
continue
deliveries_by_activity_uuid[str(a.uuid)] = to_deliveries + cc_deliveries
inbox_items_by_activity_uuid[str(a.uuid)] = (
to_inbox_items + cc_inbox_items
)
if new_to:
a.payload["to"] = new_to
if new_cc:
a.payload["cc"] = new_cc
prepared_activities.append(a)
activities = models.Activity.objects.bulk_create(prepared_activities)
for activity in activities:
if str(activity.uuid) in deliveries_by_activity_uuid:
for obj in deliveries_by_activity_uuid[str(a.uuid)]:
obj.activity = activity
if str(activity.uuid) in inbox_items_by_activity_uuid:
for obj in inbox_items_by_activity_uuid[str(a.uuid)]:
obj.activity = activity
# create all deliveries and items, in bulk
models.Delivery.objects.bulk_create(
[
obj
for collection in deliveries_by_activity_uuid.values()
for obj in collection
]
)
models.InboxItem.objects.bulk_create(
[
obj
for collection in inbox_items_by_activity_uuid.values()
for obj in collection
]
)
for a in activities:
funkwhale_utils.on_commit(tasks.dispatch_outbox.delay, activity_id=a.pk)
return activities
def recursive_gettattr(obj, key):
"""
Given a dictionary such as {'user': {'name': 'Bob'}} and
a dotted string such as user.name, returns 'Bob'.
If the value is not present, returns None
"""
v = obj
for k in key.split("."):
v = v.get(k)
if v is None:
return
return v
def match_route(route, payload):
for key, value in route.items():
if payload.get(key) != value:
payload_value = recursive_gettattr(payload, key)
if payload_value != value:
return False
return True
def prepare_inbox_items(recipient_list, type):
def prepare_deliveries_and_inbox_items(recipient_list, type):
"""
Given a list of recipients (
either actor instances, public adresses, a dictionnary with a "type" and "target"
keys for followers collections)
returns a list of deliveries, alist of inbox_items and a list
of urls to persist in the activity in place of the initial recipient list.
"""
from . import models
items = []
new_list = [] # we return a list of actors url instead
local_recipients = set()
remote_inbox_urls = set()
urls = []
for r in recipient_list:
if r != PUBLIC_ADDRESS:
item = models.InboxItem(actor=r, type=type)
items.append(item)
new_list.append(r.fid)
else:
new_list.append(r)
return items, new_list
if isinstance(r, models.Actor):
if r.is_local:
local_recipients.add(r)
else:
remote_inbox_urls.add(r.shared_inbox_url or r.inbox_url)
urls.append(r.fid)
elif r == PUBLIC_ADDRESS:
urls.append(r)
elif isinstance(r, dict) and r["type"] == "followers":
received_follows = (
r["target"]
.received_follows.filter(approved=True)
.select_related("actor__user")
)
for follow in received_follows:
actor = follow.actor
if actor.is_local:
local_recipients.add(actor)
else:
remote_inbox_urls.add(actor.shared_inbox_url or actor.inbox_url)
urls.append(r["target"].followers_url)
deliveries = [models.Delivery(inbox_url=url) for url in remote_inbox_urls]
inbox_items = [
models.InboxItem(actor=actor, type=type) for actor in local_recipients
]
return inbox_items, deliveries, urls
def join_queries_or(left, right):
if left:
return left | right
else:
return right
def get_actors_from_audience(urls):
"""
Given a list of urls such as [
"https://hello.world/@bob/followers",
"https://eldritch.cafe/@alice/followers",
"https://funkwhale.demo/libraries/uuid/followers",
]
Returns a queryset of actors that are member of the collections
listed in the given urls. The urls may contain urls referring
to an actor, an actor followers collection or an library followers
collection.
Urls that don't match anything are simply discarded
"""
from . import models
queries = {"followed": None, "actors": []}
for url in urls:
if url == PUBLIC_ADDRESS:
continue
queries["actors"].append(url)
queries["followed"] = join_queries_or(
queries["followed"], Q(target__followers_url=url)
)
final_query = None
if queries["actors"]:
final_query = join_queries_or(final_query, Q(fid__in=queries["actors"]))
if queries["followed"]:
actor_follows = models.Follow.objects.filter(queries["followed"], approved=True)
final_query = join_queries_or(
final_query, Q(pk__in=actor_follows.values_list("actor", flat=True))
)
library_follows = models.LibraryFollow.objects.filter(
queries["followed"], approved=True
)
final_query = join_queries_or(
final_query, Q(pk__in=library_follows.values_list("actor", flat=True))
)
if not final_query:
return models.Actor.objects.none()
return models.Actor.objects.filter(final_query)
def get_inbox_urls(actor_queryset):
"""
Given an actor queryset, returns a deduplicated set containing
all inbox or shared inbox urls where we should deliver our payloads for
those actors
"""
values = actor_queryset.values("inbox_url", "shared_inbox_url")
urls = set([actor["shared_inbox_url"] or actor["inbox_url"] for actor in values])
return sorted(urls)
......@@ -4,23 +4,21 @@ from . import models
from . import tasks
def redeliver_inbox_items(modeladmin, request, queryset):
for id in set(
queryset.filter(activity__actor__user__isnull=False).values_list(
"activity", flat=True
)
):
tasks.dispatch_outbox.delay(activity_id=id)
def redeliver_deliveries(modeladmin, request, queryset):
queryset.update(is_delivered=False)
for delivery in queryset:
tasks.deliver_to_remote.delay(delivery_id=delivery.pk)
redeliver_inbox_items.short_description = "Redeliver"
redeliver_deliveries.short_description = "Redeliver"
def redeliver_activities(modeladmin, request, queryset):
for id in set(
queryset.filter(actor__user__isnull=False).values_list("id", flat=True)
):
tasks.dispatch_outbox.delay(activity_id=id)
for activity in queryset.select_related("actor__user"):
if activity.actor.is_local:
tasks.dispatch_outbox.delay(activity_id=activity.pk)
else:
tasks.dispatch_inbox.delay(activity_id=activity.pk)
redeliver_activities.short_description = "Redeliver"
......@@ -67,14 +65,22 @@ class LibraryFollowAdmin(admin.ModelAdmin):
@admin.register(models.InboxItem)
class InboxItemAdmin(admin.ModelAdmin):
list_display = ["actor", "activity", "type", "is_read"]
list_filter = ["type", "activity__type", "is_read"]
search_fields = ["actor__fid", "activity__fid"]
list_select_related = True
@admin.register(models.Delivery)
class DeliveryAdmin(admin.ModelAdmin):
list_display = [
"actor",
"inbox_url",
"activity",
"type",
"last_delivery_date",
"delivery_attempts",
"last_attempt_date",
"attempts",
"is_delivered",
]
list_filter = ["type"]
search_fields = ["actor__fid", "activity__fid"]
list_filter = ["activity__type", "is_delivered"]
search_fields = ["inbox_url"]
list_select_related = True
actions = [redeliver_inbox_items]
actions = [redeliver_deliveries]
......@@ -16,7 +16,7 @@ class NestedLibraryFollowSerializer(serializers.ModelSerializer):
class LibrarySerializer(serializers.ModelSerializer):
actor = federation_serializers.APIActorSerializer()
files_count = serializers.SerializerMethodField()
uploads_count = serializers.SerializerMethodField()
follow = serializers.SerializerMethodField()
class Meta:
......@@ -28,13 +28,13 @@ class LibrarySerializer(serializers.ModelSerializer):
"name",
"description",
"creation_date",
"files_count",
"uploads_count",
"privacy_level",
"follow",
]
def get_files_count(self, o):
return max(getattr(o, "_files_count", 0), o.files_count)
def get_uploads_count(self, o):
return max(getattr(o, "_uploads_count", 0), o.uploads_count)
def get_follow(self, o):
try:
......
......@@ -87,7 +87,7 @@ class LibraryViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet):
music_models.Library.objects.all()
.order_by("-creation_date")
.select_related("actor")
.annotate(_files_count=Count("files"))
.annotate(_uploads_count=Count("uploads"))
)
serializer_class = api_serializers.LibrarySerializer
permission_classes = [permissions.IsAuthenticated]
......
......@@ -76,6 +76,9 @@ class ActorFactory(factory.DjangoModelFactory):
fid = factory.LazyAttribute(
lambda o: "https://{}/users/{}".format(o.domain, o.preferred_username)
)
followers_url = factory.LazyAttribute(
lambda o: "https://{}/users/{}followers".format(o.domain, o.preferred_username)
)
inbox_url = factory.LazyAttribute(
lambda o: "https://{}/users/{}/inbox".format(o.domain, o.preferred_username)
)
......@@ -134,19 +137,12 @@ class MusicLibraryFactory(factory.django.DjangoModelFactory):
privacy_level = "me"
name = factory.Faker("sentence")
description = factory.Faker("sentence")
files_count = 0
uploads_count = 0
fid = factory.Faker("federation_url")
class Meta:
model = "music.Library"
@factory.post_generation
def fid(self, create, extracted, **kwargs):
if not create:
# Simple build, do nothing.
return
self.fid = extracted or self.get_federation_id()