Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found
Select Git revision

Target

Select target project
  • funkwhale/funkwhale
  • Luclu7/funkwhale
  • mbothorel/funkwhale
  • EorlBruder/funkwhale
  • tcit/funkwhale
  • JocelynDelalande/funkwhale
  • eneiluj/funkwhale
  • reg/funkwhale
  • ButterflyOfFire/funkwhale
  • m4sk1n/funkwhale
  • wxcafe/funkwhale
  • andybalaam/funkwhale
  • jcgruenhage/funkwhale
  • pblayo/funkwhale
  • joshuaboniface/funkwhale
  • n3ddy/funkwhale
  • gegeweb/funkwhale
  • tohojo/funkwhale
  • emillumine/funkwhale
  • Te-k/funkwhale
  • asaintgenis/funkwhale
  • anoadragon453/funkwhale
  • Sakada/funkwhale
  • ilianaw/funkwhale
  • l4p1n/funkwhale
  • pnizet/funkwhale
  • dante383/funkwhale
  • interfect/funkwhale
  • akhardya/funkwhale
  • svfusion/funkwhale
  • noplanman/funkwhale
  • nykopol/funkwhale
  • roipoussiere/funkwhale
  • Von/funkwhale
  • aurieh/funkwhale
  • icaria36/funkwhale
  • floreal/funkwhale
  • paulwalko/funkwhale
  • comradekingu/funkwhale
  • FurryJulie/funkwhale
  • Legolars99/funkwhale
  • Vierkantor/funkwhale
  • zachhats/funkwhale
  • heyjake/funkwhale
  • sn0w/funkwhale
  • jvoisin/funkwhale
  • gordon/funkwhale
  • Alexander/funkwhale
  • bignose/funkwhale
  • qasim.ali/funkwhale
  • fakegit/funkwhale
  • Kxze/funkwhale
  • stenstad/funkwhale
  • creak/funkwhale
  • Kaze/funkwhale
  • Tixie/funkwhale
  • IISergII/funkwhale
  • lfuelling/funkwhale
  • nhaddag/funkwhale
  • yoasif/funkwhale
  • ifischer/funkwhale
  • keslerm/funkwhale
  • flupe/funkwhale
  • petitminion/funkwhale
  • ariasuni/funkwhale
  • ollie/funkwhale
  • ngaumont/funkwhale
  • techknowlogick/funkwhale
  • Shleeble/funkwhale
  • theflyingfrog/funkwhale
  • jonatron/funkwhale
  • neobrain/funkwhale
  • eorn/funkwhale
  • KokaKiwi/funkwhale
  • u1-liquid/funkwhale
  • marzzzello/funkwhale
  • sirenwatcher/funkwhale
  • newer027/funkwhale
  • codl/funkwhale
  • Zwordi/funkwhale
  • gisforgabriel/funkwhale
  • iuriatan/funkwhale
  • simon/funkwhale
  • bheesham/funkwhale
  • zeoses/funkwhale
  • accraze/funkwhale
  • meliurwen/funkwhale
  • divadsn/funkwhale
  • Etua/funkwhale
  • sdrik/funkwhale
  • Soran/funkwhale
  • kuba-orlik/funkwhale
  • cristianvogel/funkwhale
  • Forceu/funkwhale
  • jeff/funkwhale
  • der_scheibenhacker/funkwhale
  • owlnical/funkwhale
  • jovuit/funkwhale
  • SilverFox15/funkwhale
  • phw/funkwhale
  • mayhem/funkwhale
  • sridhar/funkwhale
  • stromlin/funkwhale
  • rrrnld/funkwhale
  • nitaibezerra/funkwhale
  • jaller94/funkwhale
  • pcouy/funkwhale
  • eduxstad/funkwhale
  • codingHahn/funkwhale
  • captain/funkwhale
  • polyedre/funkwhale
  • leishenailong/funkwhale
  • ccritter/funkwhale
  • lnceballosz/funkwhale
  • fpiesche/funkwhale
  • Fanyx/funkwhale
  • markusblogde/funkwhale
  • Firobe/funkwhale
  • devilcius/funkwhale
  • freaktechnik/funkwhale
  • blopware/funkwhale
  • cone/funkwhale
  • thanksd/funkwhale
  • vachan-maker/funkwhale
  • bbenti/funkwhale
  • tarator/funkwhale
  • prplecake/funkwhale
  • DMarzal/funkwhale
  • lullis/funkwhale
  • hanacgr/funkwhale
  • albjeremias/funkwhale
  • xeruf/funkwhale
  • llelite/funkwhale
  • RoiArthurB/funkwhale
  • cloo/funkwhale
  • nztvar/funkwhale
  • Keunes/funkwhale
  • petitminion/funkwhale-petitminion
  • m-idler/funkwhale
  • SkyLeite/funkwhale
140 results
Select Git revision
Show changes
Showing
with 507 additions and 349 deletions
import datetime import datetime
import logging import logging
import sys
import time import time
import uuid import uuid
import feedparser
import requests
from django.conf import settings from django.conf import settings
from django.db import transaction from django.db import transaction
from django.db.models import Q from django.db.models import Q
from django.utils import timezone
import feedparser
import requests
import pytz
from rest_framework import serializers
from django.templatetags.static import static from django.templatetags.static import static
from django.urls import reverse from django.urls import reverse
from django.utils import timezone
from drf_spectacular.types import OpenApiTypes
from drf_spectacular.utils import extend_schema_field
from rest_framework import serializers
from funkwhale_api.common import locales, preferences
from funkwhale_api.common import serializers as common_serializers from funkwhale_api.common import serializers as common_serializers
from funkwhale_api.common import utils as common_utils
from funkwhale_api.common import locales
from funkwhale_api.common import preferences
from funkwhale_api.common import session from funkwhale_api.common import session
from funkwhale_api.common import utils as common_utils
from funkwhale_api.federation import actors from funkwhale_api.federation import actors
from funkwhale_api.federation import models as federation_models from funkwhale_api.federation import models as federation_models
from funkwhale_api.federation import serializers as federation_serializers from funkwhale_api.federation import serializers as federation_serializers
from funkwhale_api.federation import utils as federation_utils from funkwhale_api.federation import utils as federation_utils
from funkwhale_api.moderation import mrf from funkwhale_api.moderation import mrf
from funkwhale_api.music import models as music_models from funkwhale_api.music import models as music_models
from funkwhale_api.music import serializers as music_serializers from funkwhale_api.music import tasks
from funkwhale_api.music.serializers import COVER_WRITE_FIELD, CoverField
from funkwhale_api.tags import models as tags_models from funkwhale_api.tags import models as tags_models
from funkwhale_api.tags import serializers as tags_serializers from funkwhale_api.tags import serializers as tags_serializers
from funkwhale_api.users import serializers as users_serializers from funkwhale_api.users import serializers as users_serializers
from . import categories from . import categories, models
from . import models
if sys.version_info < (3, 9):
from backports.zoneinfo import ZoneInfo
else:
from zoneinfo import ZoneInfo
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
...@@ -66,16 +68,16 @@ class ChannelMetadataSerializer(serializers.Serializer): ...@@ -66,16 +68,16 @@ class ChannelMetadataSerializer(serializers.Serializer):
if child not in categories.ITUNES_CATEGORIES[parent]: if child not in categories.ITUNES_CATEGORIES[parent]:
raise serializers.ValidationError( raise serializers.ValidationError(
'"{}" is not a valid subcategory for "{}"'.format(child, parent) f'"{child}" is not a valid subcategory for "{parent}"'
) )
return child return child
class ChannelCreateSerializer(serializers.Serializer): class ChannelCreateSerializer(serializers.Serializer):
name = serializers.CharField(max_length=music_models.MAX_LENGTHS["ARTIST_NAME"]) name = serializers.CharField(max_length=federation_models.MAX_LENGTHS["ACTOR_NAME"])
username = serializers.CharField( username = serializers.CharField(
max_length=music_models.MAX_LENGTHS["ARTIST_NAME"], max_length=federation_models.MAX_LENGTHS["ACTOR_NAME"],
validators=[users_serializers.ASCIIUsernameValidator()], validators=[users_serializers.ASCIIUsernameValidator()],
) )
description = common_serializers.ContentSerializer(allow_null=True) description = common_serializers.ContentSerializer(allow_null=True)
...@@ -84,7 +86,7 @@ class ChannelCreateSerializer(serializers.Serializer): ...@@ -84,7 +86,7 @@ class ChannelCreateSerializer(serializers.Serializer):
choices=music_models.ARTIST_CONTENT_CATEGORY_CHOICES choices=music_models.ARTIST_CONTENT_CATEGORY_CHOICES
) )
metadata = serializers.DictField(required=False) metadata = serializers.DictField(required=False)
cover = music_serializers.COVER_WRITE_FIELD cover = COVER_WRITE_FIELD
def validate(self, validated_data): def validate(self, validated_data):
existing_channels = self.context["actor"].owned_channels.count() existing_channels = self.context["actor"].owned_channels.count()
...@@ -135,7 +137,8 @@ class ChannelCreateSerializer(serializers.Serializer): ...@@ -135,7 +137,8 @@ class ChannelCreateSerializer(serializers.Serializer):
metadata=validated_data["metadata"], metadata=validated_data["metadata"],
) )
channel.actor = models.generate_actor( channel.actor = models.generate_actor(
validated_data["username"], name=validated_data["name"], validated_data["username"],
name=validated_data["name"],
) )
channel.library = music_models.Library.objects.create( channel.library = music_models.Library.objects.create(
...@@ -155,14 +158,14 @@ NOOP = object() ...@@ -155,14 +158,14 @@ NOOP = object()
class ChannelUpdateSerializer(serializers.Serializer): class ChannelUpdateSerializer(serializers.Serializer):
name = serializers.CharField(max_length=music_models.MAX_LENGTHS["ARTIST_NAME"]) name = serializers.CharField(max_length=federation_models.MAX_LENGTHS["ACTOR_NAME"])
description = common_serializers.ContentSerializer(allow_null=True) description = common_serializers.ContentSerializer(allow_null=True)
tags = tags_serializers.TagsListField() tags = tags_serializers.TagsListField()
content_category = serializers.ChoiceField( content_category = serializers.ChoiceField(
choices=music_models.ARTIST_CONTENT_CATEGORY_CHOICES choices=music_models.ARTIST_CONTENT_CATEGORY_CHOICES
) )
metadata = serializers.DictField(required=False) metadata = serializers.DictField(required=False)
cover = music_serializers.COVER_WRITE_FIELD cover = COVER_WRITE_FIELD
def validate(self, validated_data): def validate(self, validated_data):
validated_data = super().validate(validated_data) validated_data = super().validate(validated_data)
...@@ -232,12 +235,35 @@ class ChannelUpdateSerializer(serializers.Serializer): ...@@ -232,12 +235,35 @@ class ChannelUpdateSerializer(serializers.Serializer):
return ChannelSerializer(obj, context=self.context).data return ChannelSerializer(obj, context=self.context).data
class SimpleChannelArtistSerializer(serializers.Serializer):
id = serializers.IntegerField()
fid = serializers.URLField()
mbid = serializers.CharField()
name = serializers.CharField()
creation_date = serializers.DateTimeField()
modification_date = serializers.DateTimeField()
is_local = serializers.BooleanField()
content_category = serializers.CharField()
description = common_serializers.ContentSerializer(allow_null=True, required=False)
cover = CoverField(allow_null=True, required=False)
channel = serializers.UUIDField(allow_null=True, required=False)
tracks_count = serializers.SerializerMethodField(required=False)
tags = serializers.ListField(
child=serializers.CharField(), source="_prefetched_tagged_items", required=False
)
def get_tracks_count(self, o) -> int:
return getattr(o, "_tracks_count", 0)
class ChannelSerializer(serializers.ModelSerializer): class ChannelSerializer(serializers.ModelSerializer):
artist = serializers.SerializerMethodField() artist = SimpleChannelArtistSerializer()
actor = serializers.SerializerMethodField() actor = serializers.SerializerMethodField()
downloads_count = serializers.SerializerMethodField()
attributed_to = federation_serializers.APIActorSerializer() attributed_to = federation_serializers.APIActorSerializer()
rss_url = serializers.CharField(source="get_rss_url") rss_url = serializers.CharField(source="get_rss_url")
url = serializers.SerializerMethodField() url = serializers.SerializerMethodField()
subscriptions_count = serializers.SerializerMethodField()
class Meta: class Meta:
model = models.Channel model = models.Channel
...@@ -250,29 +276,47 @@ class ChannelSerializer(serializers.ModelSerializer): ...@@ -250,29 +276,47 @@ class ChannelSerializer(serializers.ModelSerializer):
"metadata", "metadata",
"rss_url", "rss_url",
"url", "url",
"downloads_count",
"subscriptions_count",
] ]
def get_artist(self, obj):
return music_serializers.serialize_artist_simple(obj.artist)
def to_representation(self, obj): def to_representation(self, obj):
data = super().to_representation(obj) data = super().to_representation(obj)
if self.context.get("subscriptions_count"): if self.context.get("subscriptions_count"):
data["subscriptions_count"] = self.get_subscriptions_count(obj) data["subscriptions_count"] = self.get_subscriptions_count(obj)
return data return data
def get_subscriptions_count(self, obj): @extend_schema_field(OpenApiTypes.INT)
def get_subscriptions_count(self, obj) -> int:
return obj.actor.received_follows.exclude(approved=False).count() return obj.actor.received_follows.exclude(approved=False).count()
def get_downloads_count(self, obj) -> int:
return getattr(obj, "_downloads_count", None) or 0
@extend_schema_field(federation_serializers.APIActorSerializer)
def get_actor(self, obj): def get_actor(self, obj):
if obj.attributed_to == actors.get_service_actor(): if obj.attributed_to == actors.get_service_actor():
return None return None
return federation_serializers.APIActorSerializer(obj.actor).data return federation_serializers.APIActorSerializer(obj.actor).data
@extend_schema_field(OpenApiTypes.URI)
def get_url(self, obj): def get_url(self, obj):
return obj.actor.url return obj.actor.url
class InlineSubscriptionSerializer(serializers.Serializer):
uuid = serializers.UUIDField()
channel = serializers.UUIDField(source="target__channel__uuid")
class AllSubscriptionsSerializer(serializers.Serializer):
results = InlineSubscriptionSerializer(source="*", many=True)
count = serializers.SerializerMethodField()
def get_count(self, o) -> int:
return len(o)
class SubscriptionSerializer(serializers.Serializer): class SubscriptionSerializer(serializers.Serializer):
approved = serializers.BooleanField(read_only=True) approved = serializers.BooleanField(read_only=True)
fid = serializers.URLField(read_only=True) fid = serializers.URLField(read_only=True)
...@@ -305,7 +349,7 @@ def retrieve_feed(url): ...@@ -305,7 +349,7 @@ def retrieve_feed(url):
except requests.exceptions.HTTPError as e: except requests.exceptions.HTTPError as e:
if e.response: if e.response:
raise FeedFetchException( raise FeedFetchException(
"Error while fetching feed: HTTP {}".format(e.response.status_code) f"Error while fetching feed: HTTP {e.response.status_code}"
) )
raise FeedFetchException("Error while fetching feed: unknown error") raise FeedFetchException("Error while fetching feed: unknown error")
except requests.exceptions.Timeout: except requests.exceptions.Timeout:
...@@ -313,9 +357,9 @@ def retrieve_feed(url): ...@@ -313,9 +357,9 @@ def retrieve_feed(url):
except requests.exceptions.ConnectionError: except requests.exceptions.ConnectionError:
raise FeedFetchException("Error while fetching feed: connection error") raise FeedFetchException("Error while fetching feed: connection error")
except requests.RequestException as e: except requests.RequestException as e:
raise FeedFetchException("Error while fetching feed: {}".format(e)) raise FeedFetchException(f"Error while fetching feed: {e}")
except Exception as e: except Exception as e:
raise FeedFetchException("Error while fetching feed: {}".format(e)) raise FeedFetchException(f"Error while fetching feed: {e}")
return response return response
...@@ -334,7 +378,7 @@ def get_channel_from_rss_url(url, raise_exception=False): ...@@ -334,7 +378,7 @@ def get_channel_from_rss_url(url, raise_exception=False):
parsed_feed = feedparser.parse(response.text) parsed_feed = feedparser.parse(response.text)
serializer = RssFeedSerializer(data=parsed_feed["feed"]) serializer = RssFeedSerializer(data=parsed_feed["feed"])
if not serializer.is_valid(raise_exception=raise_exception): if not serializer.is_valid(raise_exception=raise_exception):
raise FeedFetchException("Invalid xml content: {}".format(serializer.errors)) raise FeedFetchException(f"Invalid xml content: {serializer.errors}")
# second mrf check with validated data # second mrf check with validated data
urls_to_check = set() urls_to_check = set()
...@@ -364,9 +408,7 @@ def get_channel_from_rss_url(url, raise_exception=False): ...@@ -364,9 +408,7 @@ def get_channel_from_rss_url(url, raise_exception=False):
) )
) )
if parsed_feed.feed.get("rights"): if parsed_feed.feed.get("rights"):
track_defaults["copyright"] = parsed_feed.feed.rights[ track_defaults["copyright"] = parsed_feed.feed.rights
: music_models.MAX_LENGTHS["COPYRIGHT"]
]
for entry in entries[: settings.PODCASTS_RSS_FEED_MAX_ITEMS]: for entry in entries[: settings.PODCASTS_RSS_FEED_MAX_ITEMS]:
logger.debug("Importing feed item %s", entry.id) logger.debug("Importing feed item %s", entry.id)
s = RssFeedItemSerializer(data=entry) s = RssFeedItemSerializer(data=entry)
...@@ -504,7 +546,7 @@ class RssFeedSerializer(serializers.Serializer): ...@@ -504,7 +546,7 @@ class RssFeedSerializer(serializers.Serializer):
else: else:
artist_kwargs = {"pk": None} artist_kwargs = {"pk": None}
actor_kwargs = {"pk": None} actor_kwargs = {"pk": None}
preferred_username = "rssfeed-{}".format(uuid.uuid4()) preferred_username = f"rssfeed-{uuid.uuid4()}"
actor_defaults = { actor_defaults = {
"preferred_username": preferred_username, "preferred_username": preferred_username,
"type": "Application", "type": "Application",
...@@ -526,9 +568,7 @@ class RssFeedSerializer(serializers.Serializer): ...@@ -526,9 +568,7 @@ class RssFeedSerializer(serializers.Serializer):
**artist_kwargs, **artist_kwargs,
defaults={ defaults={
"attributed_to": service_actor, "attributed_to": service_actor,
"name": validated_data["title"][ "name": validated_data["title"],
: music_models.MAX_LENGTHS["ARTIST_NAME"]
],
"content_category": "podcast", "content_category": "podcast",
}, },
) )
...@@ -566,7 +606,8 @@ class RssFeedSerializer(serializers.Serializer): ...@@ -566,7 +606,8 @@ class RssFeedSerializer(serializers.Serializer):
# create/update the channel # create/update the channel
channel, created = models.Channel.objects.update_or_create( channel, created = models.Channel.objects.update_or_create(
pk=existing.pk if existing else None, defaults=channel_defaults, pk=existing.pk if existing else None,
defaults=channel_defaults,
) )
return channel return channel
...@@ -583,7 +624,7 @@ class ItunesDurationField(serializers.CharField): ...@@ -583,7 +624,7 @@ class ItunesDurationField(serializers.CharField):
try: try:
int_parts.append(int(part)) int_parts.append(int(part))
except (ValueError, TypeError): except (ValueError, TypeError):
raise serializers.ValidationError("Invalid duration {}".format(v)) raise serializers.ValidationError(f"Invalid duration {v}")
if len(int_parts) == 2: if len(int_parts) == 2:
hours = 0 hours = 0
...@@ -591,7 +632,7 @@ class ItunesDurationField(serializers.CharField): ...@@ -591,7 +632,7 @@ class ItunesDurationField(serializers.CharField):
elif len(int_parts) == 3: elif len(int_parts) == 3:
hours, minutes, seconds = int_parts hours, minutes, seconds = int_parts
else: else:
raise serializers.ValidationError("Invalid duration {}".format(v)) raise serializers.ValidationError(f"Invalid duration {v}")
return (hours * 3600) + (minutes * 60) + seconds return (hours * 3600) + (minutes * 60) + seconds
...@@ -631,6 +672,7 @@ class RssFeedItemSerializer(serializers.Serializer): ...@@ -631,6 +672,7 @@ class RssFeedItemSerializer(serializers.Serializer):
links = serializers.ListField() links = serializers.ListField()
tags = serializers.ListField(required=False) tags = serializers.ListField(required=False)
summary_detail = serializers.DictField(required=False) summary_detail = serializers.DictField(required=False)
content = serializers.ListField(required=False)
published_parsed = DummyField(required=False) published_parsed = DummyField(required=False)
image = serializers.DictField(required=False) image = serializers.DictField(required=False)
...@@ -643,6 +685,16 @@ class RssFeedItemSerializer(serializers.Serializer): ...@@ -643,6 +685,16 @@ class RssFeedItemSerializer(serializers.Serializer):
"text": content, "text": content,
} }
def validate_content(self, v):
# TODO: Are there RSS feeds that use more than one content item?
content = v[0].get("value")
if not content:
return
return {
"content_type": v[0].get("type", "text/plain"),
"text": content,
}
def validate_image(self, v): def validate_image(self, v):
url = v.get("href") url = v.get("href")
if url: if url:
...@@ -704,7 +756,7 @@ class RssFeedItemSerializer(serializers.Serializer): ...@@ -704,7 +756,7 @@ class RssFeedItemSerializer(serializers.Serializer):
else: else:
existing_track = ( existing_track = (
music_models.Track.objects.filter( music_models.Track.objects.filter(
uuid=expected_uuid, artist__channel=channel uuid=expected_uuid, artist_credit__artist__channel=channel
) )
.select_related("description", "attachment_cover") .select_related("description", "attachment_cover")
.first() .first()
...@@ -719,21 +771,16 @@ class RssFeedItemSerializer(serializers.Serializer): ...@@ -719,21 +771,16 @@ class RssFeedItemSerializer(serializers.Serializer):
{ {
"disc_number": validated_data.get("itunes_season", 1) or 1, "disc_number": validated_data.get("itunes_season", 1) or 1,
"position": validated_data.get("itunes_episode", 1) or 1, "position": validated_data.get("itunes_episode", 1) or 1,
"title": validated_data["title"][ "title": validated_data["title"],
: music_models.MAX_LENGTHS["TRACK_TITLE"]
],
"artist": channel.artist,
} }
) )
if "rights" in validated_data: if "rights" in validated_data:
track_defaults["copyright"] = validated_data["rights"][ track_defaults["copyright"] = validated_data["rights"]
: music_models.MAX_LENGTHS["COPYRIGHT"]
]
if "published_parsed" in validated_data: if "published_parsed" in validated_data:
track_defaults["creation_date"] = datetime.datetime.fromtimestamp( track_defaults["creation_date"] = datetime.datetime.fromtimestamp(
time.mktime(validated_data["published_parsed"]) time.mktime(validated_data["published_parsed"])
).replace(tzinfo=pytz.utc) ).replace(tzinfo=ZoneInfo("UTC"))
upload_defaults = { upload_defaults = {
"source": validated_data["links"]["audio"]["source"], "source": validated_data["links"]["audio"]["source"],
...@@ -757,14 +804,30 @@ class RssFeedItemSerializer(serializers.Serializer): ...@@ -757,14 +804,30 @@ class RssFeedItemSerializer(serializers.Serializer):
# create/update the track # create/update the track
track, created = music_models.Track.objects.update_or_create( track, created = music_models.Track.objects.update_or_create(
**track_kwargs, defaults=track_defaults, **track_kwargs,
defaults=track_defaults,
)
# channel only have one artist so we can safely update artist_credit
defaults = {
"artist": channel.artist,
"credit": channel.artist.name,
"joinphrase": "",
}
query = (
Q(artist=channel.artist) & Q(credit=channel.artist.name) & Q(joinphrase="")
) )
artist_credit = tasks.get_best_candidate_or_create(
music_models.ArtistCredit, query, defaults, ["artist", "joinphrase"]
)
track.artist_credit.set([artist_credit[0]])
# optimisation for reducing SQL queries, because we cannot use select_related with # optimisation for reducing SQL queries, because we cannot use select_related with
# update or create, so we restore the cache by hand # update or create, so we restore the cache by hand
if existing_track: if existing_track:
for field in ["attachment_cover", "description"]: for field in ["attachment_cover", "description"]:
cached_id_value = getattr(existing_track, "{}_id".format(field)) cached_id_value = getattr(existing_track, f"{field}_id")
new_id_value = getattr(track, "{}_id".format(field)) new_id_value = getattr(track, f"{field}_id")
if new_id_value and cached_id_value == new_id_value: if new_id_value and cached_id_value == new_id_value:
setattr(track, field, getattr(existing_track, field)) setattr(track, field, getattr(existing_track, field))
...@@ -777,6 +840,15 @@ class RssFeedItemSerializer(serializers.Serializer): ...@@ -777,6 +840,15 @@ class RssFeedItemSerializer(serializers.Serializer):
if tags: if tags:
tags_models.set_tags(track, *tags) tags_models.set_tags(track, *tags)
# "content" refers to the <content:encoded> node in the RSS feed,
# whereas "summary_detail" refers to the <description> node.
# <description> is intended to be used as a short summary and is often
# encoded merely as plain text, whereas <content:encoded> contains
# the full episode description and is generally encoded as HTML.
#
# For details, see https://www.rssboard.org/rss-profile#element-channel-item-description
summary = validated_data.get("content")
if not summary:
summary = validated_data.get("summary_detail") summary = validated_data.get("summary_detail")
if summary: if summary:
common_utils.attach_content(track, "description", summary) common_utils.attach_content(track, "description", summary)
...@@ -813,7 +885,7 @@ def rss_serialize_item(upload): ...@@ -813,7 +885,7 @@ def rss_serialize_item(upload):
data = { data = {
"title": [{"value": upload.track.title}], "title": [{"value": upload.track.title}],
"itunes:title": [{"value": upload.track.title}], "itunes:title": [{"value": upload.track.title}],
"guid": [{"cdata_value": str(upload.uuid), "isPermalink": "false"}], "guid": [{"cdata_value": str(upload.uuid), "isPermaLink": "false"}],
"pubDate": [{"value": rfc822_date(upload.creation_date)}], "pubDate": [{"value": rfc822_date(upload.creation_date)}],
"itunes:duration": [{"value": rss_duration(upload.duration)}], "itunes:duration": [{"value": rss_duration(upload.duration)}],
"itunes:explicit": [{"value": "no"}], "itunes:explicit": [{"value": "no"}],
...@@ -824,14 +896,19 @@ def rss_serialize_item(upload): ...@@ -824,14 +896,19 @@ def rss_serialize_item(upload):
"enclosure": [ "enclosure": [
{ {
# we enforce MP3, since it's the only format supported everywhere # we enforce MP3, since it's the only format supported everywhere
"url": federation_utils.full_url(upload.get_listen_url(to="mp3")), "url": federation_utils.full_url(
reverse(
"api:v1:stream-detail", kwargs={"uuid": str(upload.track.uuid)}
)
+ ".mp3"
),
"length": upload.size or 0, "length": upload.size or 0,
"type": "audio/mpeg", "type": "audio/mpeg",
} }
], ],
} }
if upload.track.description: if upload.track.description:
data["itunes:subtitle"] = [{"value": upload.track.description.truncate(255)}] data["itunes:subtitle"] = [{"value": upload.track.description.truncate(254)}]
data["itunes:summary"] = [{"cdata_value": upload.track.description.rendered}] data["itunes:summary"] = [{"cdata_value": upload.track.description.rendered}]
data["description"] = [{"value": upload.track.description.as_plain_text}] data["description"] = [{"value": upload.track.description.as_plain_text}]
...@@ -843,7 +920,7 @@ def rss_serialize_item(upload): ...@@ -843,7 +920,7 @@ def rss_serialize_item(upload):
tagged_items = getattr(upload.track, "_prefetched_tagged_items", []) tagged_items = getattr(upload.track, "_prefetched_tagged_items", [])
if tagged_items: if tagged_items:
data["itunes:keywords"] = [ data["itunes:keywords"] = [
{"value": " ".join([ti.tag.name for ti in tagged_items])} {"value": ",".join([ti.tag.name for ti in tagged_items])}
] ]
return data return data
...@@ -893,7 +970,7 @@ def rss_serialize_channel(channel): ...@@ -893,7 +970,7 @@ def rss_serialize_channel(channel):
data["itunes:category"] = [node] data["itunes:category"] = [node]
if channel.artist.description: if channel.artist.description:
data["itunes:subtitle"] = [{"value": channel.artist.description.truncate(255)}] data["itunes:subtitle"] = [{"value": channel.artist.description.truncate(254)}]
data["itunes:summary"] = [{"cdata_value": channel.artist.description.rendered}] data["itunes:summary"] = [{"cdata_value": channel.artist.description.rendered}]
data["description"] = [{"value": channel.artist.description.as_plain_text}] data["description"] = [{"value": channel.artist.description.as_plain_text}]
......
...@@ -3,12 +3,9 @@ import urllib.parse ...@@ -3,12 +3,9 @@ import urllib.parse
from django.conf import settings from django.conf import settings
from django.db.models import Q from django.db.models import Q
from django.urls import reverse from django.urls import reverse
from rest_framework import serializers from rest_framework import serializers
from funkwhale_api.common import preferences from funkwhale_api.common import middleware, preferences, utils
from funkwhale_api.common import middleware
from funkwhale_api.common import utils
from funkwhale_api.federation import utils as federation_utils from funkwhale_api.federation import utils as federation_utils
from funkwhale_api.music import spa_views from funkwhale_api.music import spa_views
...@@ -64,7 +61,7 @@ def channel_detail(query, redirect_to_ap): ...@@ -64,7 +61,7 @@ def channel_detail(query, redirect_to_ap):
"rel": "alternate", "rel": "alternate",
"type": "application/rss+xml", "type": "application/rss+xml",
"href": obj.get_rss_url(), "href": obj.get_rss_url(),
"title": "{} - RSS Podcast Feed".format(obj.artist.name), "title": f"{obj.artist.name} - RSS Podcast Feed",
}, },
) )
...@@ -76,7 +73,7 @@ def channel_detail(query, redirect_to_ap): ...@@ -76,7 +73,7 @@ def channel_detail(query, redirect_to_ap):
"type": "application/json+oembed", "type": "application/json+oembed",
"href": ( "href": (
utils.join_url(settings.FUNKWHALE_URL, reverse("api:v1:oembed")) utils.join_url(settings.FUNKWHALE_URL, reverse("api:v1:oembed"))
+ "?format=json&url={}".format(urllib.parse.quote_plus(obj_url)) + f"?format=json&url={urllib.parse.quote_plus(obj_url)}"
), ),
} }
) )
......
...@@ -7,8 +7,7 @@ from django.utils import timezone ...@@ -7,8 +7,7 @@ from django.utils import timezone
from funkwhale_api.taskapp import celery from funkwhale_api.taskapp import celery
from . import models from . import models, serializers
from . import serializers
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
......
from rest_framework import decorators
from rest_framework import exceptions
from rest_framework import mixins
from rest_framework import permissions as rest_permissions
from rest_framework import response
from rest_framework import viewsets
from django import http from django import http
from django.db import transaction from django.db import transaction
from django.db.models import Count, Prefetch, Q from django.db.models import Count, Prefetch, Q, Sum
from django.utils import timezone from django.utils import timezone
from drf_spectacular.utils import extend_schema, extend_schema_view, inline_serializer
from rest_framework import decorators, exceptions, mixins
from rest_framework import permissions as rest_permissions
from rest_framework import response
from rest_framework import serializers as rest_serializers
from rest_framework import viewsets
from funkwhale_api.common import locales from funkwhale_api.common import locales, permissions, preferences
from funkwhale_api.common import permissions
from funkwhale_api.common import preferences
from funkwhale_api.common import utils as common_utils from funkwhale_api.common import utils as common_utils
from funkwhale_api.common.mixins import MultipleLookupDetailMixin from funkwhale_api.common.mixins import MultipleLookupDetailMixin
from funkwhale_api.federation import actors from funkwhale_api.federation import actors
...@@ -27,19 +24,28 @@ from funkwhale_api.users.oauth import permissions as oauth_permissions ...@@ -27,19 +24,28 @@ from funkwhale_api.users.oauth import permissions as oauth_permissions
from . import categories, filters, models, renderers, serializers from . import categories, filters, models, renderers, serializers
ARTIST_PREFETCH_QS = ( ARTIST_PREFETCH_QS = (
music_models.Artist.objects.select_related("description", "attachment_cover",) music_models.Artist.objects.select_related(
"description",
"attachment_cover",
)
.prefetch_related(music_views.TAG_PREFETCH) .prefetch_related(music_views.TAG_PREFETCH)
.annotate(_tracks_count=Count("tracks")) .annotate(_tracks_count=Count("artist_credit__tracks"))
) )
class ChannelsMixin(object): class ChannelsMixin:
def dispatch(self, request, *args, **kwargs): def dispatch(self, request, *args, **kwargs):
if not preferences.get("audio__channels_enabled"): if not preferences.get("audio__channels_enabled"):
return http.HttpResponse(status=405) return http.HttpResponse(status=405)
return super().dispatch(request, *args, **kwargs) return super().dispatch(request, *args, **kwargs)
@extend_schema_view(
metedata_choices=extend_schema(operation_id="get_channel_metadata_choices"),
subscribe=extend_schema(operation_id="subscribe_channel"),
unsubscribe=extend_schema(operation_id="unsubscribe_channel"),
rss_subscribe=extend_schema(operation_id="subscribe_channel_rss"),
)
class ChannelViewSet( class ChannelViewSet(
ChannelsMixin, ChannelsMixin,
MultipleLookupDetailMixin, MultipleLookupDetailMixin,
...@@ -91,7 +97,17 @@ class ChannelViewSet( ...@@ -91,7 +97,17 @@ class ChannelViewSet(
return serializers.ChannelSerializer return serializers.ChannelSerializer
elif self.action in ["update", "partial_update"]: elif self.action in ["update", "partial_update"]:
return serializers.ChannelUpdateSerializer return serializers.ChannelUpdateSerializer
elif self.action == "create":
return serializers.ChannelCreateSerializer return serializers.ChannelCreateSerializer
return serializers.ChannelSerializer
def get_queryset(self):
queryset = super().get_queryset()
if self.action == "retrieve":
queryset = queryset.annotate(
_downloads_count=Sum("artist__artist_credit__tracks__downloads_count")
)
return queryset
def perform_create(self, serializer): def perform_create(self, serializer):
return serializer.save(attributed_to=self.request.user.actor) return serializer.save(attributed_to=self.request.user.actor)
...@@ -123,6 +139,7 @@ class ChannelViewSet( ...@@ -123,6 +139,7 @@ class ChannelViewSet(
detail=True, detail=True,
methods=["post"], methods=["post"],
permission_classes=[rest_permissions.IsAuthenticated], permission_classes=[rest_permissions.IsAuthenticated],
serializer_class=serializers.SubscriptionSerializer,
) )
def subscribe(self, request, *args, **kwargs): def subscribe(self, request, *args, **kwargs):
object = self.get_object() object = self.get_object()
...@@ -145,6 +162,7 @@ class ChannelViewSet( ...@@ -145,6 +162,7 @@ class ChannelViewSet(
data = serializers.SubscriptionSerializer(subscription).data data = serializers.SubscriptionSerializer(subscription).data
return response.Response(data, status=201) return response.Response(data, status=201)
@extend_schema(responses={204: None})
@decorators.action( @decorators.action(
detail=True, detail=True,
methods=["post", "delete"], methods=["post", "delete"],
...@@ -176,7 +194,6 @@ class ChannelViewSet( ...@@ -176,7 +194,6 @@ class ChannelViewSet(
if object.attributed_to == actors.get_service_actor(): if object.attributed_to == actors.get_service_actor():
# external feed, we redirect to the canonical one # external feed, we redirect to the canonical one
return http.HttpResponseRedirect(object.rss_url) return http.HttpResponseRedirect(object.rss_url)
uploads = ( uploads = (
object.library.uploads.playable_by(None) object.library.uploads.playable_by(None)
.prefetch_related( .prefetch_related(
...@@ -184,7 +201,9 @@ class ChannelViewSet( ...@@ -184,7 +201,9 @@ class ChannelViewSet(
"track", "track",
queryset=music_models.Track.objects.select_related( queryset=music_models.Track.objects.select_related(
"attachment_cover", "description" "attachment_cover", "description"
).prefetch_related(music_views.TAG_PREFETCH,), ).prefetch_related(
music_views.TAG_PREFETCH,
),
), ),
) )
.select_related("track__attachment_cover", "track__description") .select_related("track__attachment_cover", "track__description")
...@@ -193,6 +212,32 @@ class ChannelViewSet( ...@@ -193,6 +212,32 @@ class ChannelViewSet(
data = serializers.rss_serialize_channel_full(channel=object, uploads=uploads) data = serializers.rss_serialize_channel_full(channel=object, uploads=uploads)
return response.Response(data, status=200) return response.Response(data, status=200)
@extend_schema(
responses=inline_serializer(
name="MetedataChoicesSerializer",
fields={
"language": rest_serializers.ListField(
child=inline_serializer(
name="LanguageItem",
fields={
"value": rest_serializers.CharField(),
"label": rest_serializers.CharField(),
},
)
),
"itunes_category": rest_serializers.ListField(
child=inline_serializer(
name="iTunesCategoryItem",
fields={
"value": rest_serializers.CharField(),
"label": rest_serializers.CharField(),
"children": rest_serializers.CharField(),
},
)
),
},
)
)
@decorators.action( @decorators.action(
methods=["get"], methods=["get"],
detail=False, detail=False,
...@@ -224,7 +269,9 @@ class ChannelViewSet( ...@@ -224,7 +269,9 @@ class ChannelViewSet(
if not serializer.is_valid(): if not serializer.is_valid():
return response.Response(serializer.errors, status=400) return response.Response(serializer.errors, status=400)
channel = ( channel = (
models.Channel.objects.filter(rss_url=serializer.validated_data["url"],) models.Channel.objects.filter(
rss_url=serializer.validated_data["url"],
)
.order_by("id") .order_by("id")
.first() .first()
) )
...@@ -235,7 +282,10 @@ class ChannelViewSet( ...@@ -235,7 +282,10 @@ class ChannelViewSet(
serializer.validated_data["url"] serializer.validated_data["url"]
) )
except serializers.FeedFetchException as e: except serializers.FeedFetchException as e:
return response.Response({"detail": str(e)}, status=400,) return response.Response(
{"detail": str(e)},
status=400,
)
subscription = federation_models.Follow(actor=request.user.actor) subscription = federation_models.Follow(actor=request.user.actor)
subscription.fid = subscription.get_federation_id() subscription.fid = subscription.get_federation_id()
...@@ -304,6 +354,10 @@ class SubscriptionsViewSet( ...@@ -304,6 +354,10 @@ class SubscriptionsViewSet(
qs = super().get_queryset() qs = super().get_queryset()
return qs.filter(actor=self.request.user.actor) return qs.filter(actor=self.request.user.actor)
@extend_schema(
responses=serializers.AllSubscriptionsSerializer(),
operation_id="get_all_subscriptions",
)
@decorators.action(methods=["get"], detail=False) @decorators.action(methods=["get"], detail=False)
def all(self, request, *args, **kwargs): def all(self, request, *args, **kwargs):
""" """
...@@ -311,12 +365,7 @@ class SubscriptionsViewSet( ...@@ -311,12 +365,7 @@ class SubscriptionsViewSet(
to have a performant endpoint and avoid lots of queries just to display to have a performant endpoint and avoid lots of queries just to display
subscription status in the UI subscription status in the UI
""" """
subscriptions = list( subscriptions = self.get_queryset().values("uuid", "target__channel__uuid")
self.get_queryset().values_list("uuid", "target__channel__uuid")
)
payload = { payload = serializers.AllSubscriptionsSerializer(subscriptions).data
"results": [{"uuid": str(u[0]), "channel": u[1]} for u in subscriptions],
"count": len(subscriptions),
}
return response.Response(payload, status=200) return response.Response(payload, status=200)
import click
import functools import functools
import click
@click.group() @click.group()
def cli(): def cli():
......
...@@ -6,7 +6,8 @@ from . import base ...@@ -6,7 +6,8 @@ from . import base
def handler_add_tags_from_tracks( def handler_add_tags_from_tracks(
artists=False, albums=False, artists=False,
albums=False,
): ):
result = None result = None
if artists: if artists:
...@@ -19,7 +20,7 @@ def handler_add_tags_from_tracks( ...@@ -19,7 +20,7 @@ def handler_add_tags_from_tracks(
if result is None: if result is None:
click.echo(" No relevant tags found") click.echo(" No relevant tags found")
else: else:
click.echo(" Relevant tags added to {} objects".format(len(result))) click.echo(f" Relevant tags added to {len(result)} objects")
@base.cli.group() @base.cli.group()
......
import click
import sys import sys
from . import base import click
from rest_framework.exceptions import ValidationError
from . import library # noqa from . import library # noqa
from . import media # noqa
from . import plugins # noqa
from . import users # noqa from . import users # noqa
from . import base
from rest_framework.exceptions import ValidationError
def invoke(): def invoke():
...@@ -14,7 +16,7 @@ def invoke(): ...@@ -14,7 +16,7 @@ def invoke():
except ValidationError as e: except ValidationError as e:
click.secho("Invalid data:", fg="red") click.secho("Invalid data:", fg="red")
for field, errors in e.detail.items(): for field, errors in e.detail.items():
click.secho(" {}:".format(field), fg="red") click.secho(f" {field}:", fg="red")
for error in errors: for error in errors:
click.secho(" - {}".format(error), fg="red") click.secho(f" - {error}", fg="red")
sys.exit(1) sys.exit(1)
import click
from django.conf import settings
from django.core.cache import cache
from django.core.files.storage import default_storage
from versatileimagefield import settings as vif_settings
from versatileimagefield.image_warmer import VersatileImageFieldWarmer
from funkwhale_api.common import utils as common_utils
from funkwhale_api.common.models import Attachment
from . import base
@base.cli.group()
def media():
"""Manage media files (avatars, covers, attachments…)"""
pass
@media.command("generate-thumbnails")
@click.option("-d", "--delete", is_flag=True)
def generate_thumbnails(delete):
"""
Generate thumbnails for all images (avatars, covers, etc.).
This can take a long time and generate a lot of I/O depending of the size
of your library.
"""
click.echo("Deleting existing thumbnails…")
if delete:
try:
# FileSystemStorage doesn't support deleting a non-empty directory
# so we reimplemented a method to do so
default_storage.force_delete("__sized__")
except AttributeError:
# backends doesn't support directory deletion
pass
MODELS = [
(Attachment, "file", "attachment_square"),
]
for model, attribute, key_set in MODELS:
click.echo(f"Generating thumbnails for {model._meta.label}.{attribute}")
qs = model.objects.exclude(**{f"{attribute}__isnull": True})
qs = qs.exclude(**{attribute: ""})
cache_key = "*{}{}*".format(
settings.MEDIA_URL, vif_settings.VERSATILEIMAGEFIELD_SIZED_DIRNAME
)
entries = cache.keys(cache_key)
if entries:
click.echo(f" Clearing {len(entries)} cache entries: {cache_key}")
for keys in common_utils.batch(iter(entries)):
cache.delete_many(keys)
warmer = VersatileImageFieldWarmer(
instance_or_queryset=qs,
rendition_key_set=key_set,
image_attr=attribute,
verbose=True,
)
click.echo(" Creating images")
num_created, failed_to_create = warmer.warm()
click.echo(f" {num_created} created, {len(failed_to_create)} in error")
import os
import subprocess
import sys
import click
from django.conf import settings
from . import base
@base.cli.group()
def plugins():
"""Manage plugins"""
pass
@plugins.command("install")
@click.argument("plugin", nargs=-1)
def install(plugin):
"""
Install a plugin from a given URL (zip, pip or git are supported)
"""
if not plugin:
return click.echo("No plugin provided")
click.echo("Installing plugins…")
pip_install(list(plugin), settings.FUNKWHALE_PLUGINS_PATH)
def pip_install(deps, target):
if not deps:
return
pip_path = os.path.join(os.path.dirname(sys.executable), "pip")
subprocess.check_call([pip_path, "install", "-t", target] + deps)
import click import click
from django.db import transaction from django.db import transaction
from funkwhale_api.federation import models as federation_models from funkwhale_api.federation import models as federation_models
from funkwhale_api.users import models from funkwhale_api.users import models, serializers, tasks
from funkwhale_api.users import serializers
from funkwhale_api.users import tasks
from . import base from . import base, utils
from . import utils
class FakeRequest(object): class FakeRequest:
def __init__(self, session={}): def __init__(self, session={}):
self.session = session self.session = session
...@@ -37,22 +33,23 @@ def handler_create_user( ...@@ -37,22 +33,23 @@ def handler_create_user(
utils.logger.debug("Validating user data…") utils.logger.debug("Validating user data…")
serializer.is_valid(raise_exception=True) serializer.is_valid(raise_exception=True)
# Override email validation, we assume accounts created from CLI have a valid email # Override e-mail validation, we assume accounts created from CLI have a valid e-mail
request = FakeRequest(session={"account_verified_email": email}) request = FakeRequest(session={"account_verified_email": email})
utils.logger.debug("Creating user…") utils.logger.debug("Creating user…")
user = serializer.save(request=request) user = serializer.save(request=request)
utils.logger.debug("Setting permissions and other attributes…") utils.logger.debug("Setting permissions and other attributes…")
user.is_staff = is_staff user.is_staff = is_staff or is_superuser # Always set staff if superuser is set
user.upload_quota = upload_quota user.upload_quota = upload_quota
user.is_superuser = is_superuser user.is_superuser = is_superuser
for permission in permissions: for permission in permissions:
if permission in models.PERMISSIONS: if permission in models.PERMISSIONS:
utils.logger.debug("Setting %s permission to True", permission) utils.logger.debug("Setting %s permission to True", permission)
setattr(user, "permission_{}".format(permission), True) setattr(user, f"permission_{permission}", True)
else: else:
utils.logger.warn("Unknown permission %s", permission) utils.logger.warn("Unknown permission %s", permission)
utils.logger.debug("Creating actor…") utils.logger.debug("Creating actor…")
user.actor = models.create_actor(user) user.actor = models.create_actor(user)
models.create_user_libraries(user)
user.save() user.save()
return user return user
...@@ -60,7 +57,7 @@ def handler_create_user( ...@@ -60,7 +57,7 @@ def handler_create_user(
@transaction.atomic @transaction.atomic
def handler_delete_user(usernames, soft=True): def handler_delete_user(usernames, soft=True):
for username in usernames: for username in usernames:
click.echo("Deleting {}…".format(username)) click.echo(f"Deleting {username}")
actor = None actor = None
user = None user = None
try: try:
...@@ -157,13 +154,16 @@ def users(): ...@@ -157,13 +154,16 @@ def users():
type=click.INT, type=click.INT,
) )
@click.option( @click.option(
"--superuser/--no-superuser", default=False, "--superuser/--no-superuser",
default=False,
) )
@click.option( @click.option(
"--staff/--no-staff", default=False, "--staff/--no-staff",
default=False,
) )
@click.option( @click.option(
"--permission", multiple=True, "--permission",
multiple=True,
) )
def create(username, password, email, superuser, staff, permission, upload_quota): def create(username, password, email, superuser, staff, permission, upload_quota):
"""Create a new user""" """Create a new user"""
...@@ -179,9 +179,9 @@ def create(username, password, email, superuser, staff, permission, upload_quota ...@@ -179,9 +179,9 @@ def create(username, password, email, superuser, staff, permission, upload_quota
permissions=permission, permissions=permission,
upload_quota=upload_quota, upload_quota=upload_quota,
) )
click.echo("User {} created!".format(user.username)) click.echo(f"User {user.username} created!")
if generated_password: if generated_password:
click.echo(" Generated password: {}".format(generated_password)) click.echo(f" Generated password: {generated_password}")
@base.delete_command(group=users, id_var="username") @base.delete_command(group=users, id_var="username")
...@@ -210,7 +210,9 @@ def delete(username, hard): ...@@ -210,7 +210,9 @@ def delete(username, hard):
@click.option("--permission-settings/--no-permission-settings", default=None) @click.option("--permission-settings/--no-permission-settings", default=None)
@click.option("--password", default=None, envvar="FUNKWHALE_CLI_USER_UPDATE_PASSWORD") @click.option("--password", default=None, envvar="FUNKWHALE_CLI_USER_UPDATE_PASSWORD")
@click.option( @click.option(
"-q", "--upload-quota", type=click.INT, "-q",
"--upload-quota",
type=click.INT,
) )
def update(username, **kwargs): def update(username, **kwargs):
"""Update attributes for given users""" """Update attributes for given users"""
......
from django.contrib.admin import register as initial_register, site, ModelAdmin # noqa from django.contrib.admin import site # noqa: F401
from django.contrib.admin import ModelAdmin
from django.contrib.admin import register as initial_register
from django.db.models.fields.related import RelatedField from django.db.models.fields.related import RelatedField
from . import models from . import models, tasks
from . import tasks
def register(model): def register(model):
......
from django.apps import AppConfig, apps from django.apps import AppConfig, apps
from django.conf import settings
from . import mutations from config import plugins
from . import utils
from . import mutations, utils
class CommonConfig(AppConfig): class CommonConfig(AppConfig):
...@@ -13,3 +15,6 @@ class CommonConfig(AppConfig): ...@@ -13,3 +15,6 @@ class CommonConfig(AppConfig):
app_names = [app.name for app in apps.app_configs.values()] app_names = [app.name for app in apps.app_configs.values()]
mutations.registry.autodiscover(app_names) mutations.registry.autodiscover(app_names)
utils.monkey_patch_request_build_absolute_uri() utils.monkey_patch_request_build_absolute_uri()
plugins.startup.autodiscover([p + ".funkwhale_ready" for p in settings.PLUGINS])
for p in plugins._plugins.values():
p["settings"] = plugins.load_settings(p["name"], p["settings"])
from urllib.parse import parse_qs
from django.contrib.auth.models import AnonymousUser
from rest_framework import exceptions
from rest_framework_jwt.authentication import BaseJSONWebTokenAuthentication
from funkwhale_api.users.models import User
class TokenHeaderAuth(BaseJSONWebTokenAuthentication):
def get_jwt_value(self, request):
try:
qs = request.get("query_string", b"").decode("utf-8")
parsed = parse_qs(qs)
token = parsed["token"][0]
except KeyError:
raise exceptions.AuthenticationFailed("No token")
if not token:
raise exceptions.AuthenticationFailed("Empty token")
return token
class TokenAuthMiddleware:
def __init__(self, inner):
# Store the ASGI application we were passed
self.inner = inner
def __call__(self, scope):
# XXX: 1.0 remove this, replace with websocket/scopedtoken
auth = TokenHeaderAuth()
try:
user, token = auth.authenticate(scope)
except (User.DoesNotExist, exceptions.AuthenticationFailed):
user = AnonymousUser()
scope["user"] = user
return self.inner(scope)
from django.conf import settings from allauth.account.models import EmailAddress
from django.utils.encoding import smart_text
from django.utils.translation import ugettext as _
from django.core.cache import cache from django.core.cache import cache
from django.utils.translation import gettext as _
from allauth.account.utils import send_email_confirmation
from oauth2_provider.contrib.rest_framework.authentication import ( from oauth2_provider.contrib.rest_framework.authentication import (
OAuth2Authentication as BaseOAuth2Authentication, OAuth2Authentication as BaseOAuth2Authentication,
) )
from rest_framework import exceptions from rest_framework import exceptions
from rest_framework_jwt import authentication
from rest_framework_jwt.settings import api_settings
from funkwhale_api.users import models as users_models
def should_verify_email(user):
if user.is_superuser:
return False
has_unverified_email = not user.has_verified_primary_email
mandatory_verification = settings.ACCOUNT_EMAIL_VERIFICATION != "optional"
return has_unverified_email and mandatory_verification
class UnverifiedEmail(Exception): class UnverifiedEmail(Exception):
...@@ -28,13 +16,17 @@ class UnverifiedEmail(Exception): ...@@ -28,13 +16,17 @@ class UnverifiedEmail(Exception):
def resend_confirmation_email(request, user): def resend_confirmation_email(request, user):
THROTTLE_DELAY = 500 THROTTLE_DELAY = 500
cache_key = "auth:resent-email-confirmation:{}".format(user.pk) cache_key = f"auth:resent-email-confirmation:{user.pk}"
if cache.get(cache_key): if cache.get(cache_key):
return False return False
done = send_email_confirmation(request, user) # We do the sending of the conformation by hand because we don't want to pass the request down
# to the email rendering, which would cause another UnverifiedEmail Exception and restarts the sending
# again and again
email = EmailAddress.objects.get_for_user(user, user.email)
email.send_confirmation()
cache.set(cache_key, True, THROTTLE_DELAY) cache.set(cache_key, True, THROTTLE_DELAY)
return done return True
class OAuth2Authentication(BaseOAuth2Authentication): class OAuth2Authentication(BaseOAuth2Authentication):
...@@ -46,114 +38,31 @@ class OAuth2Authentication(BaseOAuth2Authentication): ...@@ -46,114 +38,31 @@ class OAuth2Authentication(BaseOAuth2Authentication):
resend_confirmation_email(request, e.user) resend_confirmation_email(request, e.user)
class BaseJsonWebTokenAuth(object): class ApplicationTokenAuthentication:
def authenticate(self, request): def authenticate(self, request):
try: try:
return super().authenticate(request) header = request.headers["Authorization"]
except UnverifiedEmail as e: except KeyError:
msg = _("You need to verify your email address.") return
resend_confirmation_email(request, e.user)
raise exceptions.AuthenticationFailed(msg)
def authenticate_credentials(self, payload): if "Bearer" not in header:
""" return
We have to implement this method by hand to ensure we can check that the
User has a verified email, if required
"""
User = authentication.get_user_model()
username = authentication.jwt_get_username_from_payload(payload)
if not username: token = header.split()[-1].strip()
msg = _("Invalid payload.")
raise exceptions.AuthenticationFailed(msg)
try: try:
user = User.objects.get_by_natural_key(username) application = users_models.Application.objects.exclude(user=None).get(
except User.DoesNotExist: token=token
msg = _("Invalid signature.") )
raise exceptions.AuthenticationFailed(msg) except users_models.Application.DoesNotExist:
return
user = users_models.User.objects.all().for_auth().get(id=application.user_id)
if not user.is_active: if not user.is_active:
msg = _("User account is disabled.") msg = _("User account is disabled.")
raise exceptions.AuthenticationFailed(msg) raise exceptions.AuthenticationFailed(msg)
if should_verify_email(user): if user.should_verify_email():
raise UnverifiedEmail(user) raise UnverifiedEmail(user)
return user request.scopes = application.scope.split()
return user, None
class JSONWebTokenAuthenticationQS(
BaseJsonWebTokenAuth, authentication.BaseJSONWebTokenAuthentication
):
www_authenticate_realm = "api"
def get_jwt_value(self, request):
token = request.query_params.get("jwt")
if "jwt" in request.query_params and not token:
msg = _("Invalid Authorization header. No credentials provided.")
raise exceptions.AuthenticationFailed(msg)
return token
def authenticate_header(self, request):
return '{0} realm="{1}"'.format(
api_settings.JWT_AUTH_HEADER_PREFIX, self.www_authenticate_realm
)
class BearerTokenHeaderAuth(
BaseJsonWebTokenAuth, authentication.BaseJSONWebTokenAuthentication
):
"""
For backward compatibility purpose, we used Authorization: JWT <token>
but Authorization: Bearer <token> is probably better.
"""
www_authenticate_realm = "api"
def get_jwt_value(self, request):
auth = authentication.get_authorization_header(request).split()
auth_header_prefix = "bearer"
if not auth:
if api_settings.JWT_AUTH_COOKIE:
return request.COOKIES.get(api_settings.JWT_AUTH_COOKIE)
return None
if smart_text(auth[0].lower()) != auth_header_prefix:
return None
if len(auth) == 1:
msg = _("Invalid Authorization header. No credentials provided.")
raise exceptions.AuthenticationFailed(msg)
elif len(auth) > 2:
msg = _(
"Invalid Authorization header. Credentials string "
"should not contain spaces."
)
raise exceptions.AuthenticationFailed(msg)
return auth[1]
def authenticate_header(self, request):
return '{0} realm="{1}"'.format("Bearer", self.www_authenticate_realm)
def authenticate(self, request):
auth = super().authenticate(request)
if auth:
if not auth[0].actor:
auth[0].create_actor()
return auth
class JSONWebTokenAuthentication(
BaseJsonWebTokenAuth, authentication.JSONWebTokenAuthentication
):
def authenticate(self, request):
auth = super().authenticate(request)
if auth:
if not auth[0].actor:
auth[0].create_actor()
return auth
...@@ -19,6 +19,10 @@ class JsonAuthConsumer(JsonWebsocketConsumer): ...@@ -19,6 +19,10 @@ class JsonAuthConsumer(JsonWebsocketConsumer):
channels.group_add(group, self.channel_name) channels.group_add(group, self.channel_name)
def disconnect(self, close_code): def disconnect(self, close_code):
if self.scope.get("user", False) and self.scope.get("user").pk is not None:
groups = self.scope["user"].get_channels_groups() + self.groups groups = self.scope["user"].get_channels_groups() + self.groups
else:
groups = self.groups
for group in groups: for group in groups:
channels.group_discard(group, self.channel_name) channels.group_discard(group, self.channel_name)
from django.db import transaction from django.db import transaction
from drf_spectacular.utils import OpenApiParameter, extend_schema
from rest_framework import decorators, exceptions, response, status
from rest_framework import decorators from . import filters, models
from rest_framework import exceptions
from rest_framework import response
from rest_framework import status
from . import filters
from . import models
from . import mutations as common_mutations from . import mutations as common_mutations
from . import serializers from . import serializers, signals, tasks, utils
from . import signals
from . import tasks
from . import utils
def action_route(serializer_class): def action_route(serializer_class):
...@@ -87,6 +80,16 @@ def mutations_route(types): ...@@ -87,6 +80,16 @@ def mutations_route(types):
) )
return response.Response(serializer.data, status=status.HTTP_201_CREATED) return response.Response(serializer.data, status=status.HTTP_201_CREATED)
return decorators.action( return extend_schema(
methods=["post"], responses=serializers.APIMutationSerializer()
)(
extend_schema(
methods=["get"],
responses=serializers.APIMutationSerializer(many=True),
parameters=[OpenApiParameter("id", location="query", exclude=True)],
)(
decorators.action(
methods=["get", "post"], detail=True, required_scope="edits" methods=["get", "post"], detail=True, required_scope="edits"
)(mutations) )(mutations)
)
)
from dynamic_preferences import types from dynamic_preferences import types
from dynamic_preferences.registries import global_preferences_registry from dynamic_preferences.registries import global_preferences_registry
from funkwhale_api.common import preferences
common = types.Section("common") common = types.Section("common")
@global_preferences_registry.register @global_preferences_registry.register
class APIAutenticationRequired( class APIAutenticationRequired(types.BooleanPreference):
preferences.DefaultFromSettingMixin, types.BooleanPreference
):
section = common section = common
name = "api_authentication_required" name = "api_authentication_required"
verbose_name = "API Requires authentication" verbose_name = "API Requires authentication"
setting = "API_AUTHENTICATION_REQUIRED" default = True
help_text = ( help_text = (
"If disabled, anonymous users will be able to query the API " "If disabled, anonymous users will be able to query the API "
"and access music data (as well as other data exposed in the API " "and access music data (as well as other data exposed in the API "
......
import factory import factory
from funkwhale_api.factories import registry, NoUpdateOnCreate from funkwhale_api.factories import NoUpdateOnCreate, registry
from funkwhale_api.federation import factories as federation_factories from funkwhale_api.federation import factories as federation_factories
...@@ -35,3 +34,12 @@ class CommonFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory): ...@@ -35,3 +34,12 @@ class CommonFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
class Meta: class Meta:
model = "common.Content" model = "common.Content"
@registry.register
class PluginConfiguration(NoUpdateOnCreate, factory.django.DjangoModelFactory):
code = "test"
conf = {"foo": "bar"}
class Meta:
model = "common.PluginConfiguration"
import django_filters import django_filters
from django import forms from django import forms
from django.conf import settings
from django.core.serializers.json import DjangoJSONEncoder from django.core.serializers.json import DjangoJSONEncoder
from django.db import models from django.db import models
from rest_framework import serializers from rest_framework import serializers
from . import search from . import search
...@@ -26,9 +24,22 @@ def privacy_level_query(user, lookup_field="privacy_level", user_field="user"): ...@@ -26,9 +24,22 @@ def privacy_level_query(user, lookup_field="privacy_level", user_field="user"):
if user.is_anonymous: if user.is_anonymous:
return models.Q(**{lookup_field: "everyone"}) return models.Q(**{lookup_field: "everyone"})
return models.Q( followers_query = models.Q(
**{"{}__in".format(lookup_field): ["instance", "everyone"]} **{
) | models.Q(**{lookup_field: "me", user_field: user}) f"{lookup_field}": "followers",
f"{user_field}__actor__in": user.actor.get_approved_followings(),
}
)
# Federated TrackFavorite don't have an user associated with the trackfavorite.actor
# to do : if we implement the followers privacy_level this will become a problem
no_user_query = models.Q(**{f"{user_field}__isnull": True})
return (
models.Q(**{f"{lookup_field}__in": ["instance", "everyone"]})
| models.Q(**{lookup_field: "me", user_field: user})
| followers_query
| no_user_query
)
class SearchFilter(django_filters.CharFilter): class SearchFilter(django_filters.CharFilter):
...@@ -40,7 +51,7 @@ class SearchFilter(django_filters.CharFilter): ...@@ -40,7 +51,7 @@ class SearchFilter(django_filters.CharFilter):
def filter(self, qs, value): def filter(self, qs, value):
if not value: if not value:
return qs return qs
if settings.USE_FULL_TEXT_SEARCH and self.fts_search_fields: if self.fts_search_fields:
query = search.get_fts_query( query = search.get_fts_query(
value, self.fts_search_fields, model=self.parent.Meta.model value, self.fts_search_fields, model=self.parent.Meta.model
) )
...@@ -59,7 +70,7 @@ class SmartSearchFilter(django_filters.CharFilter): ...@@ -59,7 +70,7 @@ class SmartSearchFilter(django_filters.CharFilter):
return qs return qs
try: try:
cleaned = self.config.clean(value) cleaned = self.config.clean(value)
except (forms.ValidationError): except forms.ValidationError:
return qs.none() return qs.none()
return search.apply(qs, cleaned) return search.apply(qs, cleaned)
...@@ -99,7 +110,7 @@ def get_generic_filter_query(value, relation_name, choices): ...@@ -99,7 +110,7 @@ def get_generic_filter_query(value, relation_name, choices):
obj = related_queryset.get(obj_query) obj = related_queryset.get(obj_query)
except related_queryset.model.DoesNotExist: except related_queryset.model.DoesNotExist:
raise forms.ValidationError("Invalid object") raise forms.ValidationError("Invalid object")
filter_query &= models.Q(**{"{}_id".format(relation_name): obj.id}) filter_query &= models.Q(**{f"{relation_name}_id": obj.id})
return filter_query return filter_query
...@@ -165,7 +176,7 @@ class GenericRelation(serializers.JSONField): ...@@ -165,7 +176,7 @@ class GenericRelation(serializers.JSONField):
id_value = v[id_attr] id_value = v[id_attr]
id_value = id_field.to_internal_value(id_value) id_value = id_field.to_internal_value(id_value)
except (TypeError, KeyError, serializers.ValidationError): except (TypeError, KeyError, serializers.ValidationError):
raise serializers.ValidationError("Invalid {}".format(id_attr)) raise serializers.ValidationError(f"Invalid {id_attr}")
query_getter = conf.get( query_getter = conf.get(
"get_query", lambda attr, value: models.Q(**{attr: value}) "get_query", lambda attr, value: models.Q(**{attr: value})
......
from django import forms from django import forms
from django.db.models import Q from django.db.models import Q
from django.db.models.functions import Lower
from django_filters import widgets
from django_filters import rest_framework as filters from django_filters import rest_framework as filters
from django_filters import widgets
from drf_spectacular.utils import extend_schema_field
from . import fields from . import fields, models, search, utils
from . import models
from . import search
class NoneObject(object): class NoneObject:
def __eq__(self, other): def __eq__(self, other):
return other.__class__ == NoneObject return other.__class__ == NoneObject
...@@ -48,9 +47,10 @@ class CoerceChoiceField(forms.ChoiceField): ...@@ -48,9 +47,10 @@ class CoerceChoiceField(forms.ChoiceField):
try: try:
return [b for a, b in self.choices if v == a][0] return [b for a, b in self.choices if v == a][0]
except IndexError: except IndexError:
raise forms.ValidationError("Invalid value {}".format(value)) raise forms.ValidationError(f"Invalid value {value}")
@extend_schema_field(bool)
class NullBooleanFilter(filters.ChoiceFilter): class NullBooleanFilter(filters.ChoiceFilter):
field_class = CoerceChoiceField field_class = CoerceChoiceField
...@@ -64,9 +64,7 @@ class NullBooleanFilter(filters.ChoiceFilter): ...@@ -64,9 +64,7 @@ class NullBooleanFilter(filters.ChoiceFilter):
return qs return qs
if value == NONE: if value == NONE:
value = None value = None
qs = self.get_method(qs)( qs = self.get_method(qs)(**{f"{self.field_name}__{self.lookup_expr}": value})
**{"%s__%s" % (self.field_name, self.lookup_expr): value}
)
return qs.distinct() if self.distinct else qs return qs.distinct() if self.distinct else qs
...@@ -119,11 +117,9 @@ class MultipleQueryFilter(filters.TypedMultipleChoiceFilter): ...@@ -119,11 +117,9 @@ class MultipleQueryFilter(filters.TypedMultipleChoiceFilter):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
kwargs["widget"] = QueryArrayWidget() kwargs["widget"] = QueryArrayWidget()
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
self.lookup_expr = "in"
def filter_target(value): def filter_target(value):
config = { config = {
"artist": ["artist", "target_id", int], "artist": ["artist", "target_id", int],
"album": ["album", "target_id", int], "album": ["album", "target_id", int],
...@@ -170,14 +166,17 @@ class MutationFilter(filters.FilterSet): ...@@ -170,14 +166,17 @@ class MutationFilter(filters.FilterSet):
fields = ["is_approved", "is_applied", "type"] fields = ["is_approved", "is_applied", "type"]
class EmptyQuerySet(ValueError):
pass
class ActorScopeFilter(filters.CharFilter): class ActorScopeFilter(filters.CharFilter):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
self.actor_field = kwargs.pop("actor_field") self.actor_field = kwargs.pop("actor_field")
self.library_field = kwargs.pop("library_field", None)
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
def filter(self, queryset, value): def filter(self, queryset, value):
from funkwhale_api.federation import models as federation_models
if not value: if not value:
return queryset return queryset
...@@ -186,35 +185,74 @@ class ActorScopeFilter(filters.CharFilter): ...@@ -186,35 +185,74 @@ class ActorScopeFilter(filters.CharFilter):
return queryset.none() return queryset.none()
user = getattr(request, "user", None) user = getattr(request, "user", None)
qs = queryset actor = getattr(user, "actor", None)
if value.lower() == "me": scopes = [v.strip().lower() for v in value.split(",")]
qs = self.filter_me(user=user, queryset=queryset) query = None
elif value.lower() == "all": for scope in scopes:
return queryset try:
elif value.lower().startswith("actor:"): right_query = self.get_query(scope, user, actor)
full_username = value.split("actor:", 1)[1] except ValueError:
return queryset.none()
query = utils.join_queries_or(query, right_query)
return queryset.filter(query).distinct()
def get_query(self, scope, user, actor):
from funkwhale_api.federation import models as federation_models
if scope == "me":
return self.filter_me(actor)
elif scope == "all":
return Q(pk__gte=0)
elif scope == "subscribed":
if not actor or self.library_field is None:
raise EmptyQuerySet()
followed_libraries = federation_models.LibraryFollow.objects.filter(
approved=True, actor=user.actor
).values_list("target_id", flat=True)
if not self.library_field:
predicate = "pk__in"
else:
predicate = f"{self.library_field}__in"
return Q(**{predicate: followed_libraries})
elif scope.startswith("actor:"):
full_username = scope.split("actor:", 1)[1]
username, domain = full_username.split("@") username, domain = full_username.split("@")
try: try:
actor = federation_models.Actor.objects.get( actor = federation_models.Actor.objects.get(
preferred_username=username, domain_id=domain, preferred_username__iexact=username,
domain_id=domain,
) )
except federation_models.Actor.DoesNotExist: except federation_models.Actor.DoesNotExist:
return queryset.none() raise EmptyQuerySet()
return queryset.filter(**{self.actor_field: actor}) return Q(**{self.actor_field: actor})
elif value.lower().startswith("domain:"): elif scope.startswith("domain:"):
domain = value.split("domain:", 1)[1] domain = scope.split("domain:", 1)[1]
return queryset.filter(**{"{}__domain_id".format(self.actor_field): domain}) return Q(**{f"{self.actor_field}__domain_id": domain})
else: else:
return queryset.none() raise EmptyQuerySet()
def filter_me(self, actor):
if not actor:
raise EmptyQuerySet()
return Q(**{self.actor_field: actor})
if self.distinct:
qs = qs.distinct() class CaseInsensitiveNameOrderingFilter(filters.OrderingFilter):
def filter(self, qs, value):
order_by = []
if value is None:
return qs return qs
def filter_me(self, user, queryset): for param in value:
actor = getattr(user, "actor", None) if param == "name":
if not actor: order_by.append(Lower("name"))
return queryset.none() else:
order_by.append(self.get_ordering_value(param))
return queryset.filter(**{self.actor_field: actor}) return qs.order_by(*order_by)