Commit 2ea21994 authored by Eliot Berriot's avatar Eliot Berriot 💬

Resolve "Per-user libraries" (use !368 instead)

parent b0ca1810
......@@ -14,12 +14,11 @@ router.register(r"settings", GlobalPreferencesViewSet, base_name="settings")
router.register(r"activity", activity_views.ActivityViewSet, "activity")
router.register(r"tags", views.TagViewSet, "tags")
router.register(r"tracks", views.TrackViewSet, "tracks")
router.register(r"trackfiles", views.TrackFileViewSet, "trackfiles")
router.register(r"track-files", views.TrackFileViewSet, "trackfiles")
router.register(r"libraries", views.LibraryViewSet, "libraries")
router.register(r"listen", views.ListenViewSet, "listen")
router.register(r"artists", views.ArtistViewSet, "artists")
router.register(r"albums", views.AlbumViewSet, "albums")
router.register(r"import-batches", views.ImportBatchViewSet, "import-batches")
router.register(r"import-jobs", views.ImportJobViewSet, "import-jobs")
router.register(r"submit", views.SubmitViewSet, "submit")
router.register(r"playlists", playlists_views.PlaylistViewSet, "playlists")
router.register(
r"playlist-tracks", playlists_views.PlaylistTrackViewSet, "playlist-tracks"
......
......@@ -8,9 +8,7 @@ application = ProtocolTypeRouter(
{
# Empty for now (http->django views is added by default)
"websocket": TokenAuthMiddleware(
URLRouter(
[url("^api/v1/instance/activity$", consumers.InstanceActivityConsumer)]
)
URLRouter([url("^api/v1/activity$", consumers.InstanceActivityConsumer)])
)
}
)
......@@ -126,7 +126,6 @@ LOCAL_APPS = (
"funkwhale_api.history",
"funkwhale_api.playlists",
"funkwhale_api.providers.audiofile",
"funkwhale_api.providers.youtube",
"funkwhale_api.providers.acoustid",
"funkwhale_api.subsonic",
)
......@@ -280,7 +279,7 @@ MEDIA_ROOT = env("MEDIA_ROOT", default=str(APPS_DIR("media")))
# See: https://docs.djangoproject.com/en/dev/ref/settings/#media-url
MEDIA_URL = env("MEDIA_URL", default="/media/")
FILE_UPLOAD_PERMISSIONS = 0o644
# URL Configuration
# ------------------------------------------------------------------------------
ROOT_URLCONF = "config.urls"
......@@ -446,7 +445,7 @@ REST_FRAMEWORK = {
"DEFAULT_AUTHENTICATION_CLASSES": (
"funkwhale_api.common.authentication.JSONWebTokenAuthenticationQS",
"funkwhale_api.common.authentication.BearerTokenHeaderAuth",
"rest_framework_jwt.authentication.JSONWebTokenAuthentication",
"funkwhale_api.common.authentication.JSONWebTokenAuthentication",
"rest_framework.authentication.SessionAuthentication",
"rest_framework.authentication.BasicAuthentication",
),
......
......@@ -2,11 +2,13 @@
from __future__ import unicode_literals
from django.conf import settings
from django.conf.urls import include, url
from django.conf.urls import url
from django.urls import include, path
from django.conf.urls.static import static
from django.contrib import admin
from django.views import defaults as default_views
urlpatterns = [
# Django Admin, use {% url 'admin:index' %}
url(settings.ADMIN_URL, admin.site.urls),
......@@ -36,4 +38,6 @@ if settings.DEBUG:
if "debug_toolbar" in settings.INSTALLED_APPS:
import debug_toolbar
urlpatterns += [url(r"^__debug__/", include(debug_toolbar.urls))]
urlpatterns = [
path("api/__debug__/", include(debug_toolbar.urls))
] + urlpatterns
......@@ -56,3 +56,20 @@ class BearerTokenHeaderAuth(authentication.BaseJSONWebTokenAuthentication):
def authenticate_header(self, request):
return '{0} realm="{1}"'.format("Bearer", self.www_authenticate_realm)
def authenticate(self, request):
auth = super().authenticate(request)
if auth:
if not auth[0].actor:
auth[0].create_actor()
return auth
class JSONWebTokenAuthentication(authentication.JSONWebTokenAuthentication):
def authenticate(self, request):
auth = super().authenticate(request)
if auth:
if not auth[0].actor:
auth[0].create_actor()
return auth
import json
import logging
from asgiref.sync import async_to_sync
from channels.layers import get_channel_layer
from django.core.serializers.json import DjangoJSONEncoder
logger = logging.getLogger(__file__)
channel_layer = get_channel_layer()
group_send = async_to_sync(channel_layer.group_send)
group_add = async_to_sync(channel_layer.group_add)
def group_send(group, event):
# we serialize the payload ourselves and deserialize it to ensure it
# works with msgpack. This is dirty, but we'll find a better solution
# later
s = json.dumps(event, cls=DjangoJSONEncoder)
event = json.loads(s)
logger.debug(
"[channels] Dispatching %s to group %s: %s",
event["type"],
group,
{"type": event["data"]["type"]},
)
async_to_sync(channel_layer.group_send)(group, event)
......@@ -16,3 +16,5 @@ class JsonAuthConsumer(JsonWebsocketConsumer):
super().accept()
for group in self.groups:
channels.group_add(group, self.channel_name)
for group in self.scope["user"].get_channels_groups():
channels.group_add(group, self.channel_name)
from . import create_actors
from . import create_image_variations
from . import django_permissions_to_user_permissions
from . import migrate_to_user_libraries
from . import test
......@@ -8,5 +9,6 @@ __all__ = [
"create_actors",
"create_image_variations",
"django_permissions_to_user_permissions",
"migrate_to_user_libraries",
"test",
]
"""
Mirate instance files to a library #463. For each user that imported music on an
instance, we will create a "default" library with related files and an instance-level
visibility.
Files without any import job will be bounded to a "default" library on the first
superuser account found. This should now happen though.
"""
from funkwhale_api.music import models
from funkwhale_api.users.models import User
def main(command, **kwargs):
importer_ids = set(
models.ImportBatch.objects.values_list("submitted_by", flat=True)
)
importers = User.objects.filter(pk__in=importer_ids).order_by("id").select_related()
command.stdout.write(
"* {} users imported music on this instance".format(len(importers))
)
files = models.TrackFile.objects.filter(
library__isnull=True, jobs__isnull=False
).distinct()
command.stdout.write(
"* Reassigning {} files to importers libraries...".format(files.count())
)
for user in importers:
command.stdout.write(
" * Setting up @{}'s 'default' library".format(user.username)
)
library = user.actor.libraries.get_or_create(actor=user.actor, name="default")[
0
]
user_files = files.filter(jobs__batch__submitted_by=user)
total = user_files.count()
command.stdout.write(
" * Reassigning {} files to the user library...".format(total)
)
user_files.update(library=library)
files = models.TrackFile.objects.filter(
library__isnull=True, jobs__isnull=True
).distinct()
command.stdout.write(
"* Handling {} files with no import jobs...".format(files.count())
)
user = User.objects.order_by("id").filter(is_superuser=True).first()
command.stdout.write(" * Setting up @{}'s 'default' library".format(user.username))
library = user.actor.libraries.get_or_create(actor=user.actor, name="default")[0]
total = files.count()
command.stdout.write(
" * Reassigning {} files to the user library...".format(total)
)
files.update(library=library)
command.stdout.write(" * Done!")
import collections
from rest_framework import serializers
from django.core.exceptions import ObjectDoesNotExist
from django.utils.encoding import smart_text
from django.utils.translation import ugettext_lazy as _
class RelatedField(serializers.RelatedField):
default_error_messages = {
"does_not_exist": _("Object with {related_field_name}={value} does not exist."),
"invalid": _("Invalid value."),
}
def __init__(self, related_field_name, serializer, **kwargs):
self.related_field_name = related_field_name
self.serializer = serializer
self.filters = kwargs.pop("filters", None)
kwargs["queryset"] = kwargs.pop(
"queryset", self.serializer.Meta.model.objects.all()
)
super().__init__(**kwargs)
def get_filters(self, data):
filters = {self.related_field_name: data}
if self.filters:
filters.update(self.filters(self.context))
return filters
def to_internal_value(self, data):
try:
queryset = self.get_queryset()
filters = self.get_filters(data)
return queryset.get(**filters)
except ObjectDoesNotExist:
self.fail(
"does_not_exist",
related_field_name=self.related_field_name,
value=smart_text(data),
)
except (TypeError, ValueError):
self.fail("invalid")
def to_representation(self, obj):
return self.serializer.to_representation(obj)
def get_choices(self, cutoff=None):
queryset = self.get_queryset()
if queryset is None:
# Ensure that field.choices returns something sensible
# even when accessed with a read-only field.
return {}
if cutoff is not None:
queryset = queryset[:cutoff]
return collections.OrderedDict(
[
(
self.to_representation(item)[self.related_field_name],
self.display_value(item),
)
for item in queryset
]
)
class Action(object):
def __init__(self, name, allow_all=False, qs_filter=None):
......@@ -21,6 +86,7 @@ class ActionSerializer(serializers.Serializer):
objects = serializers.JSONField(required=True)
filters = serializers.DictField(required=False)
actions = None
pk_field = "pk"
def __init__(self, *args, **kwargs):
self.actions_by_name = {a.name: a for a in self.actions}
......@@ -51,7 +117,9 @@ class ActionSerializer(serializers.Serializer):
if value == "all":
return self.queryset.all().order_by("id")
if type(value) in [list, tuple]:
return self.queryset.filter(pk__in=value).order_by("id")
return self.queryset.filter(
**{"{}__in".format(self.pk_field): value}
).order_by("id")
raise serializers.ValidationError(
"{} is not a valid value for objects. You must provide either a "
......
......@@ -3,9 +3,12 @@ from rest_framework.decorators import list_route
from rest_framework.permissions import IsAuthenticatedOrReadOnly
from rest_framework.response import Response
from django.db.models import Prefetch
from funkwhale_api.activity import record
from funkwhale_api.common import fields, permissions
from funkwhale_api.music.models import Track
from funkwhale_api.music import utils as music_utils
from . import filters, models, serializers
......@@ -19,11 +22,7 @@ class TrackFavoriteViewSet(
filter_class = filters.TrackFavoriteFilter
serializer_class = serializers.UserTrackFavoriteSerializer
queryset = (
models.TrackFavorite.objects.all()
.select_related("track__artist", "track__album__artist", "user")
.prefetch_related("track__files")
)
queryset = models.TrackFavorite.objects.all().select_related("user")
permission_classes = [
permissions.ConditionalAuthentication,
permissions.OwnerPermission,
......@@ -49,9 +48,14 @@ class TrackFavoriteViewSet(
def get_queryset(self):
queryset = super().get_queryset()
return queryset.filter(
queryset = queryset.filter(
fields.privacy_level_query(self.request.user, "user__privacy_level")
)
tracks = Track.objects.annotate_playable_by_actor(
music_utils.get_actor_from_request(self.request)
).select_related("artist", "album__artist")
queryset = queryset.prefetch_related(Prefetch("track", queryset=tracks))
return queryset
def perform_create(self, serializer):
track = Track.objects.get(pk=serializer.data["track"])
......
import uuid
from funkwhale_api.common import utils as funkwhale_utils
PUBLIC_ADDRESS = "https://www.w3.org/ns/activitystreams#Public"
ACTIVITY_TYPES = [
"Accept",
"Add",
......@@ -58,4 +64,145 @@ def accept_follow(follow):
from . import serializers
serializer = serializers.AcceptFollowSerializer(follow)
return deliver(serializer.data, to=[follow.actor.url], on_behalf_of=follow.target)
return deliver(serializer.data, to=[follow.actor.fid], on_behalf_of=follow.target)
def receive(activity, on_behalf_of):
from . import models
from . import serializers
from . import tasks
# we ensure the activity has the bare minimum structure before storing
# it in our database
serializer = serializers.BaseActivitySerializer(
data=activity, context={"actor": on_behalf_of, "local_recipients": True}
)
serializer.is_valid(raise_exception=True)
copy = serializer.save()
# we create inbox items for further delivery
items = [
models.InboxItem(activity=copy, actor=r, type="to")
for r in serializer.validated_data["recipients"]["to"]
if hasattr(r, "fid")
]
items += [
models.InboxItem(activity=copy, actor=r, type="cc")
for r in serializer.validated_data["recipients"]["cc"]
if hasattr(r, "fid")
]
models.InboxItem.objects.bulk_create(items)
# at this point, we have the activity in database. Even if we crash, it's
# okay, as we can retry later
tasks.dispatch_inbox.delay(activity_id=copy.pk)
return copy
class Router:
def __init__(self):
self.routes = []
def connect(self, route, handler):
self.routes.append((route, handler))
def register(self, route):
def decorator(handler):
self.connect(route, handler)
return handler
return decorator
class InboxRouter(Router):
def dispatch(self, payload, context):
"""
Receives an Activity payload and some context and trigger our
business logic
"""
for route, handler in self.routes:
if match_route(route, payload):
return handler(payload, context=context)
class OutboxRouter(Router):
def dispatch(self, routing, context):
"""
Receives a routing payload and some business objects in the context
and may yield data that should be persisted in the Activity model
for further delivery.
"""
from . import models
from . import tasks
for route, handler in self.routes:
if match_route(route, routing):
activities_data = []
for e in handler(context):
# a route can yield zero, one or more activity payloads
if e:
activities_data.append(e)
inbox_items_by_activity_uuid = {}
prepared_activities = []
for activity_data in activities_data:
to = activity_data.pop("to", [])
cc = activity_data.pop("cc", [])
a = models.Activity(**activity_data)
a.uuid = uuid.uuid4()
to_items, new_to = prepare_inbox_items(to, "to")
cc_items, new_cc = prepare_inbox_items(cc, "cc")
if not to_items and not cc_items:
continue
inbox_items_by_activity_uuid[str(a.uuid)] = to_items + cc_items
if new_to:
a.payload["to"] = new_to
if new_cc:
a.payload["cc"] = new_cc
prepared_activities.append(a)
activities = models.Activity.objects.bulk_create(prepared_activities)
activities = [a for a in activities if a]
final_inbox_items = []
for a in activities:
try:
prepared_inbox_items = inbox_items_by_activity_uuid[str(a.uuid)]
except KeyError:
continue
for ii in prepared_inbox_items:
ii.activity = a
final_inbox_items.append(ii)
# create all inbox items, in bulk
models.InboxItem.objects.bulk_create(final_inbox_items)
for a in activities:
funkwhale_utils.on_commit(
tasks.dispatch_outbox.delay, activity_id=a.pk
)
return activities
def match_route(route, payload):
for key, value in route.items():
if payload.get(key) != value:
return False
return True
def prepare_inbox_items(recipient_list, type):
from . import models
items = []
new_list = [] # we return a list of actors url instead
for r in recipient_list:
if r != PUBLIC_ADDRESS:
item = models.InboxItem(actor=r, type=type)
items.append(item)
new_list.append(r.fid)
else:
new_list.append(r)
return items, new_list
......@@ -3,15 +3,11 @@ import logging
import xml
from django.conf import settings
from django.db import transaction
from django.urls import reverse
from django.utils import timezone
from rest_framework.exceptions import PermissionDenied
from funkwhale_api.common import preferences, session
from funkwhale_api.common import utils as funkwhale_utils
from funkwhale_api.music import models as music_models
from funkwhale_api.music import tasks as music_tasks
from . import activity, keys, models, serializers, signing, utils
......@@ -39,9 +35,9 @@ def get_actor_data(actor_url):
raise ValueError("Invalid actor payload: {}".format(response.text))
def get_actor(actor_url):
def get_actor(fid):
try:
actor = models.Actor.objects.get(url=actor_url)
actor = models.Actor.objects.get(fid=fid)
except models.Actor.DoesNotExist:
actor = None
fetch_delta = datetime.timedelta(
......@@ -50,7 +46,7 @@ def get_actor(actor_url):
if actor and actor.last_fetch_date > timezone.now() - fetch_delta:
# cache is hot, we can return as is
return actor
data = get_actor_data(actor_url)
data = get_actor_data(fid)
serializer = serializers.ActorSerializer(data=data)
serializer.is_valid(raise_exception=True)
......@@ -72,7 +68,7 @@ class SystemActor(object):
def get_actor_instance(self):
try:
return models.Actor.objects.get(url=self.get_actor_url())
return models.Actor.objects.get(fid=self.get_actor_id())
except models.Actor.DoesNotExist:
pass
private, public = keys.get_key_pair()
......@@ -83,7 +79,7 @@ class SystemActor(object):
args["public_key"] = public.decode("utf-8")
return models.Actor.objects.create(**args)
def get_actor_url(self):
def get_actor_id(self):
return utils.full_url(
reverse("federation:instance-actors-detail", kwargs={"actor": self.id})
)
......@@ -95,7 +91,7 @@ class SystemActor(object):
"type": "Person",
"name": name.format(host=settings.FEDERATION_HOSTNAME),
"manually_approves_followers": True,
"url": self.get_actor_url(),
"fid": self.get_actor_id(),
"shared_inbox_url": utils.full_url(
reverse("federation:instance-actors-inbox", kwargs={"actor": id})
),
......@@ -178,91 +174,13 @@ class SystemActor(object):
if ac["object"]["type"] != "Follow":
return
if ac["object"]["actor"] != sender.url:
if ac["object"]["actor"] != sender.fid:
# not the same actor, permission issue
return
self.handle_undo_follow(ac, sender)
class LibraryActor(SystemActor):
id = "library"
name = "{host}'s library"
summary = "Bot account to federate with {host}'s library"
additional_attributes = {"manually_approves_followers": True}
def serialize(self):
data = super().serialize()
urls = data.setdefault("url", [])
urls.append(
{
"type": "Link",
"mediaType": "application/activity+json",
"name": "library",
"href": utils.full_url(reverse("federation:music:files-list")),
}
)
return data
@property
def manually_approves_followers(self):
return preferences.get("federation__music_needs_approval")
@transaction.atomic
def handle_create(self, ac, sender):
try:
remote_library = models.Library.objects.get(
actor=sender, federation_enabled=True
)
except models.Library.DoesNotExist:
logger.info("Skipping import, we're not following %s", sender.url)
return
if ac["object"]["type"] != "Collection":
return
if ac["object"]["totalItems"] <= 0:
return
try:
items = ac["object"]["items"]
except KeyError:
logger.warning("No items in collection!")
return
item_serializers = [
serializers.AudioSerializer(data=i, context={"library": remote_library})
for i in items
]
now = timezone.now()
valid_serializers = []
for s in item_serializers:
if s.is_valid():
valid_serializers.append(s)
else:
logger.debug("Skipping invalid item %s, %s", s.initial_data, s.errors)
lts = []
for s in valid_serializers:
lts.append(s.save())
if remote_library.autoimport:
batch = music_models.ImportBatch.objects.create(source="federation")
for lt in lts:
if lt.creation_date < now:
# track was already in the library, we do not trigger
# an import
continue
job = music_models.ImportJob.objects.create(
batch=batch, library_track=lt, mbid=lt.mbid, source=lt.url
)
funkwhale_utils.on_commit(
music_tasks.import_job_run.delay,
import_job_id=job.pk,
use_acoustid=False,
)
class TestActor(SystemActor):
id = "test"
name = "{host}'s test account"