Commit d6659277 authored by petitminion's avatar petitminion
Browse files

Merge remote-tracking branch 'upstream/develop' into develop

parents ef18e5ea 568c5959
Pipeline #16869 failed with stages
in 2 minutes and 46 seconds
......@@ -3,7 +3,7 @@ DJANGO_SETTINGS_MODULE=config.settings.local
DJANGO_SECRET_KEY=dev
C_FORCE_ROOT=true
FUNKWHALE_HOSTNAME=localhost
FUNKWHALE_PROTOCOL=https
FUNKWHALE_PROTOCOL=http
PYTHONDONTWRITEBYTECODE=true
VUE_PORT=8080
MUSIC_DIRECTORY_PATH=/music
......@@ -11,7 +11,7 @@ BROWSABLE_API_ENABLED=True
FORWARDED_PROTO=http
LDAP_ENABLED=False
FUNKWHALE_SPA_HTML_ROOT=http://nginx/front/
PYTHONTRACEMALLOC=1
PYTHONTRACEMALLOC=0
# Uncomment this if you're using traefik/https
# FORCE_HTTPS_URLS=True
......
......@@ -9,6 +9,9 @@ variables:
PYTHONDONTWRITEBYTECODE: "true"
REVIEW_DOMAIN: preview.funkwhale.audio
REVIEW_INSTANCE_URL: https://demo.funkwhale.audio
DOCKER_HOST: tcp://docker:2375/
DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: ""
stages:
- review
......@@ -117,13 +120,31 @@ flake8:
paths:
- "$PIP_CACHE_DIR"
eslint:
interruptible: true
image: node:12-buster
stage: lint
allow_failure: true
before_script:
- cd front
- yarn install
script:
# We search for all files ending with .vue or .js in src which changed in relation to develop
# and lint them. This way we focus on some errors instead of checking the hole repository
- export changedFiles=$(git diff --relative --name-only --diff-filter=d origin/develop -- src/ | grep -E "\.(vue|js)$")
- yarn run eslint --quiet -f table $(echo $changedFiles | tr '\n' ' ')
cache:
key: "$CI_PROJECT_ID__eslint_npm_cache"
paths:
- front/node_modules
test_api:
interruptible: true
services:
- postgres:11
- redis:5
stage: test
image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
image: $CI_REGISTRY/funkwhale/backend-test-docker:latest
cache:
key: "$CI_PROJECT_ID__pip_cache"
paths:
......@@ -133,17 +154,16 @@ test_api:
FUNKWHALE_URL: "https://funkwhale.ci"
DJANGO_SETTINGS_MODULE: config.settings.local
POSTGRES_HOST_AUTH_METHOD: trust
CACHE_URL: "redis://redis:6379/0"
only:
- branches
before_script:
- apk add make git gcc python3-dev musl-dev python3-dev openssl-dev cargo
- apk add postgresql-dev py3-psycopg2 libldap libffi-dev make zlib-dev jpeg-dev openldap-dev
- cd api
- pip3 install -r requirements/base.txt
- pip3 install -r requirements/local.txt
- pip3 install -r requirements/test.txt
script:
- pytest --cov-report xml --cov-report term-missing --cov=funkwhale_api --junitxml=report.xml tests/
- pytest --cov-report xml --cov-report term-missing:skip-covered --cov=funkwhale_api --junitxml=report.xml tests/
tags:
- docker
artifacts:
......@@ -198,23 +218,11 @@ build_front:
only:
- tags@funkwhale/funkwhale
- master@funkwhale/funkwhale
- stable@funkwhale/funkwhale
- develop@funkwhale/funkwhale
tags:
- docker
build_backend:
stage: build
image: bash
before_script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
script:
- docker system prune -af
- cd api
- docker build -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME .
- docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
tags:
- docker-build
pages:
stage: test
image: python:3.6
......@@ -235,58 +243,62 @@ pages:
paths:
- public
only:
- master@funkwhale/funkwhale
- stable@funkwhale/funkwhale
tags:
- docker
docker_release:
stage: deploy
image: bash
image: egon0/docker-with-buildx-and-git:bash
services:
- docker:20-dind
before_script:
- docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
- cp -r front/dist api/frontend
- (if [ "$CI_COMMIT_REF_NAME" == "develop" ] || [ "$CI_COMMIT_REF_NAME" == "master" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
- (if [ "$CI_COMMIT_REF_NAME" == "develop" ] || [ "$CI_COMMIT_REF_NAME" == "stable" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
script:
- if [[ ! -z "$CI_COMMIT_TAG" ]]; then (./docs/get-releases-json.py | scripts/is-docker-latest.py $CI_COMMIT_TAG -) && export DOCKER_LATEST_TAG="-t $IMAGE_LATEST" || export DOCKER_LATEST_TAG=; fi
- if [[ "$CI_COMMIT_REF_NAME" =~ ^[0-9]+(.[0-9]+){1,2}$ ]]; then export stable=1 && export major="$(echo $CI_COMMIT_REF_NAME | cut -d '.' -f 1)" && export minor="$(echo $CI_COMMIT_REF_NAME | cut -d '.' -f 1,2)"; fi
- cd api
- docker build -t $IMAGE $DOCKER_LATEST_TAG .
- docker push $IMAGE
- if [[ ! -z "$DOCKER_LATEST_TAG" ]]; then docker push $IMAGE_LATEST; fi
- if [[ $stable == 1 ]]; then docker tag $IMAGE $IMAGE_NAME:$major && docker push $IMAGE_NAME:$major; fi
- if [[ $stable == 1 ]]; then docker tag $IMAGE $IMAGE_NAME:$minor && docker push $IMAGE_NAME:$minor; fi
only:
- develop@funkwhale/funkwhale
- master@funkwhale/funkwhale
- stable@funkwhale/funkwhale
- tags@funkwhale/funkwhale
tags:
- docker-build
#docker_all_in_one_release:
# stage: deploy
# image: bash
# variables:
# ALL_IN_ONE_REF: master
# ALL_IN_ONE_ARTIFACT_URL: https://github.com/thetarkus/docker-funkwhale/archive/$ALL_IN_ONE_REF.zip
# BUILD_PATH: all_in_one
# before_script:
# - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
# - (if [ "$CI_COMMIT_REF_NAME" == "develop" ] || [ "$CI_COMMIT_REF_NAME" == "master" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
# script:
# - if [[ ! -z "$CI_COMMIT_TAG" ]]; then (./docs/get-releases-json.py | scripts/is-docker-latest.py $CI_COMMIT_TAG -) && export DOCKER_LATEST_TAG="-t $ALL_IN_ONE_IMAGE_LATEST" || export DOCKER_LATEST_TAG=; fi
# - wget $ALL_IN_ONE_ARTIFACT_URL -O all_in_one.zip
# - unzip -o all_in_one.zip -d tmpdir
# - mv tmpdir/docker-funkwhale-$ALL_IN_ONE_REF $BUILD_PATH && rmdir tmpdir
# - cp -r api $BUILD_PATH/src/api
# - cp -r front $BUILD_PATH/src/front
# - cd $BUILD_PATH
# - ./scripts/download-nginx-template.sh src/ $CI_COMMIT_REF_NAME
# - docker build -t $ALL_IN_ONE_IMAGE $DOCKER_LATEST_TAG .
# - docker push $ALL_IN_ONE_IMAGE
# - if [[ ! -z "$DOCKER_LATEST_TAG" ]]; then docker push $ALL_IN_ONE_IMAGE_LATEST; fi
# only:
# - develop@funkwhale/funkwhale
# - master@funkwhale/funkwhale
# - tags@funkwhale/funkwhale
# tags:
# - docker-build
docker_all_in_one_release:
stage: deploy
image: egon0/docker-with-buildx-and-git:bash
services:
- docker:20-dind
variables:
ALL_IN_ONE_REF: main
ALL_IN_ONE_ARTIFACT_URL: https://dev.funkwhale.audio/funkwhale/funkwhale-docker-all-in-one/-/archive/$ALL_IN_ONE_REF/funkwhale-docker-all-in-one-$ALL_IN_ONE_REF.zip
BUILD_PATH: all_in_one
before_script:
- docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
- (if [ "$CI_COMMIT_REF_NAME" == "develop" ] || [ "$CI_COMMIT_REF_NAME" == "master" ] || [ "$CI_COMMIT_REF_NAME" == "stable" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
script:
- if [[ ! -z "$CI_COMMIT_TAG" ]]; then (./docs/get-releases-json.py | scripts/is-docker-latest.py $CI_COMMIT_TAG -) && export DOCKER_LATEST_TAG="-t $ALL_IN_ONE_IMAGE_LATEST" || export DOCKER_LATEST_TAG=; fi
- wget $ALL_IN_ONE_ARTIFACT_URL -O all_in_one.zip
- unzip -o all_in_one.zip -d tmpdir
- mv tmpdir/funkwhale-docker-all-in-one-$ALL_IN_ONE_REF $BUILD_PATH && rmdir tmpdir
- cp -r api $BUILD_PATH/src/api
- cp -r front $BUILD_PATH/src/front
- cd $BUILD_PATH
- ./scripts/download-nginx-template.sh src/ $CI_COMMIT_REF_NAME
- docker build -t $ALL_IN_ONE_IMAGE $DOCKER_LATEST_TAG .
- docker push $ALL_IN_ONE_IMAGE
- if [[ ! -z "$DOCKER_LATEST_TAG" ]]; then docker push $ALL_IN_ONE_IMAGE_LATEST; fi
only:
- develop@funkwhale/funkwhale
- master@funkwhale/funkwhale
- stable@funkwhale/funkwhale
- tags@funkwhale/funkwhale
build_api:
# Simply publish a zip containing api/ directory
......@@ -298,12 +310,13 @@ build_api:
- api
script:
- rm -rf api/tests
- (if [ "$CI_COMMIT_REF_NAME" == "develop" ] || [ "$CI_COMMIT_REF_NAME" == "master" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
- (if [ "$CI_COMMIT_REF_NAME" == "develop" ] || [ "$CI_COMMIT_REF_NAME" == "stable" ] || [ "$CI_COMMIT_REF_NAME" == "master" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
- chmod -R 750 api
- echo Done!
only:
- tags@funkwhale/funkwhale
- master@funkwhale/funkwhale
- stable@funkwhale/funkwhale
- develop@funkwhale/funkwhale
check_api_dependencies:
......
Please avoid merging the base branch into your feature branch. We are working with rebases and those merged tend to cause trouble.
For further questions, join us at Matrix: https://matrix.to/#/#funkwhale-dev:matrix.org
Related issue: #XXX <!-- it's okay to have no issue for small changes -->
This Merge Request includes:
......
......@@ -10,6 +10,29 @@ This changelog is viewable on the web at https://docs.funkwhale.audio/changelog.
.. towncrier
1.1.4 (2021-08-02)
Upgrade instructions are available at
https://docs.funkwhale.audio/admin/upgrading.html
- Pinned version of asgiref to avoid trouble with latest release. For further information, see #1516
1.1.3 (2021-08-02)
Upgrade instructions are available at
https://docs.funkwhale.audio/admin/upgrading.html
Enhancements:
- Test better tagging of Docker Images (#1505)
Bugfixes:
- Fix the scrobbler plugin submitting literal "None" as MusicBrainz ID (#1498)
- Add worker-src to nginx header to prevent issues (#1489)
- Only suggest typed tag once if it already exists
- Implement access control on the moderation views (#1494)
- Prevent open redirect on login (#1492)
1.1.2 (2021-05-19)
------------------
......
......@@ -100,7 +100,7 @@ Visit https://dev.funkwhale.audio/funkwhale/funkwhale and clone the repository u
A note about branches
^^^^^^^^^^^^^^^^^^^^^
Next release development occurs on the "develop" branch, and releases are made on the "master" branch. Therefore, when submitting Merge Requests, ensure you are merging on the develop branch.
Next release development occurs on the "develop" branch, and releases are made on the "stable" branch. Therefore, when submitting Merge Requests, ensure you are merging on the develop branch.
Working with docker
......@@ -750,7 +750,7 @@ To make a new 3.4 release::
export PREVIOUS_RELEASE=3.3 # replace with the previous release number
# ensure you have an up-to-date repo
git checkout develop # use master if you're doing a hotfix release
git checkout develop # use stable if you're doing a hotfix release
git pull
# compile changelog
......@@ -775,11 +775,11 @@ To make a new 3.4 release::
# publish
git push --tags && git push
# if you're doing a hotfix release from master
git checkout develop && git merge master && git push
# if you're doing a hotfix release from stable
git checkout develop && git merge stable && git push
# if you're doing a non-hotfix release, and a real release (not a real release) from develop
git checkout master && git merge develop && git push
git checkout stable && git merge develop && git push
Then, visit https://dev.funkwhale.audio/funkwhale/funkwhale/-/tags, copy-paste the changelog on the corresponding
tag, and announce the good news ;)
......@@ -2,25 +2,24 @@ FROM alpine:3.13 as builder
RUN \
echo 'installing dependencies' && \
apk add --no-cache \
git \
musl-dev \
apk add --no-cache \
git \
musl-dev \
gcc \
postgresql-dev \
python3-dev \
py3-psycopg2 \
libldap \
libffi-dev \
make \
zlib-dev \
jpeg-dev \
python3-dev \
py3-psycopg2 \
libldap \
libffi-dev \
make \
zlib-dev \
jpeg-dev \
openldap-dev \
openssl-dev \
cargo \
libxml2-dev \
libxslt-dev \
&& \
\
ln -s /usr/bin/python3 /usr/bin/python
# create virtual env for next stage
......@@ -53,21 +52,21 @@ ENV PATH="/venv/bin:$PATH"
RUN apk add --no-cache \
libmagic \
bash \
gettext \
python3 \
bash \
gettext \
python3 \
jpeg-dev \
ffmpeg \
libpq \
ffmpeg \
libpq \
libxml2 \
libxslt \
&& \
\
ln -s /usr/bin/python3 /usr/bin/python
COPY . /app
WORKDIR /app
RUN find . -type d -exec chmod 755 {} \+
ENTRYPOINT ["./compose/django/entrypoint.sh"]
CMD ["./compose/django/server.sh"]
COPY . /app
WORKDIR /app
......@@ -19,8 +19,7 @@ if [ -z "$CELERY_BROKER_URL" ]; then
fi
# we copy the frontend files, if any so we can serve them from the outside
if [ -d "frontend" ]; then
mkdir -p /frontend
if [ -d "frontend" ] && [ -d "/frontend" ]; then
cp -r frontend/* /frontend/
export FUNKWHALE_SPA_HTML_ROOT=/frontend/index.html
fi
......
......@@ -8,7 +8,9 @@ application = ProtocolTypeRouter(
{
# Empty for now (http->django views is added by default)
"websocket": AuthMiddlewareStack(
URLRouter([url("^api/v1/activity$", consumers.InstanceActivityConsumer)])
URLRouter(
[url("^api/v1/activity$", consumers.InstanceActivityConsumer.as_asgi())]
)
)
}
)
# -*- coding: utf-8 -*-
__version__ = "1.1.2"
__version__ = "1.1.4"
__version_info__ = tuple(
[
int(num) if num.isdigit() else num
......
......@@ -17,6 +17,7 @@ class PodcastRSSRenderer(renderers.JSONRenderer):
"version": "2.0",
"xmlns:atom": "http://www.w3.org/2005/Atom",
"xmlns:itunes": "http://www.itunes.com/dtds/podcast-1.0.dtd",
"xmlns:content": "http://purl.org/rss/1.0/modules/content/",
"xmlns:media": "http://search.yahoo.com/mrss/",
}
final.update(data)
......
......@@ -636,6 +636,7 @@ class RssFeedItemSerializer(serializers.Serializer):
links = serializers.ListField()
tags = serializers.ListField(required=False)
summary_detail = serializers.DictField(required=False)
content = serializers.ListField(required=False)
published_parsed = DummyField(required=False)
image = serializers.DictField(required=False)
......@@ -648,6 +649,16 @@ class RssFeedItemSerializer(serializers.Serializer):
"text": content,
}
def validate_content(self, v):
# TODO: Are there RSS feeds that use more than one content item?
content = v[0].get("value")
if not content:
return
return {
"content_type": v[0].get("type", "text/plain"),
"text": content,
}
def validate_image(self, v):
url = v.get("href")
if url:
......@@ -782,7 +793,16 @@ class RssFeedItemSerializer(serializers.Serializer):
if tags:
tags_models.set_tags(track, *tags)
summary = validated_data.get("summary_detail")
# "content" refers to the <content:encoded> node in the RSS feed,
# whereas "summary_detail" refers to the <description> node.
# <description> is intended to be used as a short summary and is often
# encoded merely as plain text, whereas <content:encoded> contains
# the full episode description and is generally encoded as HTML.
#
# For details, see https://www.rssboard.org/rss-profile#element-channel-item-description
summary = validated_data.get("content")
if not summary:
summary = validated_data.get("summary_detail")
if summary:
common_utils.attach_content(track, "description", summary)
......
......@@ -297,6 +297,7 @@ SAFE_TAGS = [
"acronym",
"b",
"blockquote",
"br",
"code",
"em",
"i",
......
......@@ -5,6 +5,7 @@ PLUGIN = plugins.get_plugin_config(
name="listenbrainz",
label="ListenBrainz",
description="A plugin that allows you to submit your listens to ListenBrainz.",
homepage="https://docs.funkwhale.audio/users/builtinplugins.html#listenbrainz-plugin", # noqa
version="0.1",
user=True,
conf=[
......
......@@ -7,7 +7,7 @@ PLUGIN = plugins.get_plugin_config(
"A plugin that enables scrobbling to ListenBrainz and Last.fm. "
"It must be configured on the server if you use Last.fm."
),
homepage="https://dev.funkwhale.audio/funkwhale/funkwhale/-/blob/develop/api/funkwhale_api/contrib/scrobbler/README.rst", # noqa
homepage="https://docs.funkwhale.audio/users/builtinplugins.html#scrobbler-plugin", # noqa
version="0.1",
user=True,
conf=[
......
......@@ -357,6 +357,8 @@ class Metadata(Mapping):
self._file = kind(filething)
if self._file is None:
raise ValueError("Cannot parse metadata from {}".format(filething))
if len(self._file) == 0:
raise ValueError("No tags found in {}".format(filething))
self.fallback = self.load_fallback(filething, self._file)
ft = self.get_file_type(self._file)
try:
......
......@@ -75,7 +75,12 @@ def dict_to_xml_tree(root_tag, d, parent=None):
root.append(dict_to_xml_tree(key, value, parent=root))
elif isinstance(value, list):
for obj in value:
root.append(dict_to_xml_tree(key, obj, parent=root))
if isinstance(obj, dict):
el = dict_to_xml_tree(key, obj, parent=root)
else:
el = ET.Element(key)
el.text = str(obj)
root.append(el)
else:
if key == "value":
root.text = str(value)
......
......@@ -49,6 +49,7 @@ def get_artist_data(artist_values):
"id": artist_values["id"],
"name": artist_values["name"],
"albumCount": artist_values["_albums_count"],
"coverArt": "ar-{}".format(artist_values["id"]),
}
......@@ -82,6 +83,8 @@ class GetArtistSerializer(serializers.Serializer):
"albumCount": len(albums),
"album": [],
}
if artist.attachment_cover_id:
payload["coverArt"] = "ar-{}".format(artist.id)
for album in albums:
album_data = {
"id": album.id,
......
......@@ -14,6 +14,7 @@ from rest_framework import permissions as rest_permissions
from rest_framework import renderers, response, viewsets
from rest_framework.decorators import action
from rest_framework.serializers import ValidationError
from config import plugins
import funkwhale_api
from funkwhale_api.activity import record
......@@ -772,6 +773,19 @@ class SubsonicViewSet(viewsets.GenericViewSet):
{"error": {"code": 70, "message": "cover art not found."}}
)
attachment = album.attachment_cover
elif id.startswith("ar-"):
try:
artist_id = int(id.replace("ar-", ""))
artist = (
music_models.Artist.objects.exclude(attachment_cover=None)
.select_related("attachment_cover")
.get(pk=artist_id)
)
except (TypeError, ValueError, music_models.Album.DoesNotExist):
return response.Response(
{"error": {"code": 70, "message": "cover art not found."}}
)
attachment = artist.attachment_cover
elif id.startswith("at-"):
try:
attachment_id = id.replace("at-", "")
......@@ -810,6 +824,11 @@ class SubsonicViewSet(viewsets.GenericViewSet):
)
if serializer.validated_data["submission"]:
listening = serializer.save()
plugins.trigger_hook(
plugins.LISTENING_CREATED,
listening=listening,
confs=plugins.get_confs(request.user),
)
record.send(listening)
return response.Response({})
......@@ -926,7 +945,7 @@ class SubsonicViewSet(viewsets.GenericViewSet):
)
uploads_qs = (
music_models.Upload.objects.playable_by(request.user.actor)
.select_related("track__attachment_cover", "track__description",)
.select_related("track__attachment_cover", "track__description")
.order_by("-track__creation_date")
)
......
django~=3.2.3
setuptools~=57.1.0
setuptools~=57.4.0
# Configuration
django-environ~=0.4.0
......@@ -21,22 +21,22 @@ kombu~=5.1.0
celery~=5.1.2
# Your custom requirements go here
django-cors-headers~=3.7.0
django-cors-headers~=3.8.0
musicbrainzngs~=0.7.1
djangorestframework~=3.12.2
arrow~=1.1.0
persisting-theory~=0.2.0
django-versatileimagefield~=2.1.0
django-versatileimagefield~=2.2.0
django-filter~=2.4.0
django-rest-auth~=0.9.0
ipython~=7.25.0
ipython~=7.27.0
mutagen~=1.45.0
pymemoize~=1.0.0
django-dynamic-preferences~=1.10
python-magic~=0.4.0
channels~=2.4.0
channels~=3.0.3
channels_redis~=3.3.0
uvicorn[standard]~=0.14.0
gunicorn~=20.1.0
......@@ -46,12 +46,12 @@ cryptography~=3.4.7
# clone until the branch is merged and released upstream
git+https://github.com/agateblue/requests-http-signature.git@signature-header-support
django-cleanup~=5.2.0
requests~=2.25.1
requests~=2.26.0
pyOpenSSL~=20.0.1
# for LDAP authentication
python-ldap~=3.3.0
django-auth-ldap~=2.4.0
django-auth-ldap~=3.0.0
pydub~=0.25.1
pyld~=2.0.3
......@@ -71,5 +71,4 @@ feedparser~=6.0.0
watchdog~=2.1.2
## Pin third party dependency to avoid issue with latest version
twisted==20.3.0
asgiref==3.3.4
......@@ -6,7 +6,8 @@ pytest-cov~=2.12.0
pytest-django~=4.4.0
pytest-env~=0.6.0
pytest-mock~=3.6.0
pytest-randomly~=3.8.0
pytest-randomly~=3.10.1
pytest-sugar~=0.9.0
pytest-asyncio~=0.15.1
requests-mock~=1.9.0
faker~=8.9.1
faker~=8.12.1
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment