Verified Commit 9b611df1 authored by Jeff's avatar Jeff
Browse files

merged upstream/develop

parents 779f64b7 474f1667
Pipeline #17096 passed with stages
in 22 minutes and 4 seconds
......@@ -3,7 +3,7 @@ DJANGO_SETTINGS_MODULE=config.settings.local
DJANGO_SECRET_KEY=dev
C_FORCE_ROOT=true
FUNKWHALE_HOSTNAME=localhost
FUNKWHALE_PROTOCOL=https
FUNKWHALE_PROTOCOL=http
PYTHONDONTWRITEBYTECODE=true
VUE_PORT=8080
MUSIC_DIRECTORY_PATH=/music
......@@ -11,7 +11,7 @@ BROWSABLE_API_ENABLED=True
FORWARDED_PROTO=http
LDAP_ENABLED=False
FUNKWHALE_SPA_HTML_ROOT=http://nginx/front/
PYTHONTRACEMALLOC=1
PYTHONTRACEMALLOC=0
# Uncomment this if you're using traefik/https
# FORCE_HTTPS_URLS=True
......
......@@ -9,6 +9,10 @@ variables:
PYTHONDONTWRITEBYTECODE: "true"
REVIEW_DOMAIN: preview.funkwhale.audio
REVIEW_INSTANCE_URL: https://demo.funkwhale.audio
DOCKER_HOST: tcp://docker:2375/
DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: ""
BUILD_PLATFORMS: linux/amd64,linux/arm64,linux/arm/v7
stages:
- review
......@@ -101,6 +105,9 @@ black:
- pip install black==19.10b0
script:
- black --check --diff api/
only:
changes:
- api/**/*
flake8:
interruptible: true
......@@ -116,6 +123,9 @@ flake8:
key: "$CI_PROJECT_ID__flake8_pip_cache"
paths:
- "$PIP_CACHE_DIR"
only:
changes:
- api/**/*
eslint:
interruptible: true
......@@ -129,11 +139,14 @@ eslint:
# We search for all files ending with .vue or .js in src which changed in relation to develop
# and lint them. This way we focus on some errors instead of checking the hole repository
- export changedFiles=$(git diff --relative --name-only --diff-filter=d origin/develop -- src/ | grep -E "\.(vue|js)$")
- yarn run eslint --quiet -f table $changedFiles
- yarn run eslint --quiet -f table $(echo $changedFiles | tr '\n' ' ')
cache:
key: "$CI_PROJECT_ID__eslint_npm_cache"
paths:
- front/node_modules
only:
changes:
- front/**/*
test_api:
interruptible: true
......@@ -141,7 +154,7 @@ test_api:
- postgres:11
- redis:5
stage: test
image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
image: $CI_REGISTRY/funkwhale/backend-test-docker:latest
cache:
key: "$CI_PROJECT_ID__pip_cache"
paths:
......@@ -151,17 +164,19 @@ test_api:
FUNKWHALE_URL: "https://funkwhale.ci"
DJANGO_SETTINGS_MODULE: config.settings.local
POSTGRES_HOST_AUTH_METHOD: trust
CACHE_URL: "redis://redis:6379/0"
only:
- branches
refs:
- branches
changes:
- api/**/*
before_script:
- apk add make git gcc python3-dev musl-dev python3-dev openssl-dev cargo
- apk add postgresql-dev py3-psycopg2 libldap libffi-dev make zlib-dev jpeg-dev openldap-dev
- cd api
- pip3 install -r requirements/base.txt
- pip3 install -r requirements/local.txt
- pip3 install -r requirements/test.txt
script:
- pytest --cov-report xml --cov-report term-missing --cov=funkwhale_api --junitxml=report.xml tests/
- pytest --cov-report xml --cov-report term-missing:skip-covered --cov=funkwhale_api --junitxml=report.xml tests/
tags:
- docker
artifacts:
......@@ -177,7 +192,10 @@ test_front:
before_script:
- cd front
only:
- branches
refs:
- branches
changes:
- front/**/*
script:
- yarn install --check-files
- yarn test:unit
......@@ -216,23 +234,11 @@ build_front:
only:
- tags@funkwhale/funkwhale
- master@funkwhale/funkwhale
- stable@funkwhale/funkwhale
- develop@funkwhale/funkwhale
tags:
- docker
build_backend:
stage: build
image: bash
before_script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
script:
- docker system prune -af
- cd api
- docker build -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME .
- docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
tags:
- docker-build
pages:
stage: test
image: python:3.6
......@@ -253,61 +259,89 @@ pages:
paths:
- public
only:
- master@funkwhale/funkwhale
- stable@funkwhale/funkwhale
tags:
- docker
docker_release:
.docker_publish:
stage: deploy
image: bash
image: egon0/docker-with-buildx-and-git:bash
tags:
- multiarch
services:
- docker:20-dind
before_script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
- cp -r front/dist api/frontend
- (if [ "$CI_COMMIT_REF_NAME" == "develop" ] || [ "$CI_COMMIT_REF_NAME" == "master" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
script:
- if [[ ! -z "$CI_COMMIT_TAG" ]]; then (./docs/get-releases-json.py | scripts/is-docker-latest.py $CI_COMMIT_TAG -) && export DOCKER_LATEST_TAG="-t $IMAGE_LATEST" || export DOCKER_LATEST_TAG=; fi
- if [[ "$CI_COMMIT_REF_NAME" =~ ^[0-9]+(.[0-9]+){1,2}$ ]]; then export stable=1 && export major="$(echo $CI_COMMIT_REF_NAME | cut -d '.' -f 1)" && export minor="$(echo $CI_COMMIT_REF_NAME | cut -d '.' -f 1,2)"; fi
docker_publish_stable_release:
# Publish a docker image for releases
extends: .docker_publish
rules:
- if: $CI_COMMIT_TAG && $CI_COMMIT_REF_NAME =~ /^[0-9]+(.[0-9]+){1,2}$/
script:
# Check if this is the latest release
- ./docs/get-releases-json.py | scripts/is-docker-latest.py $CI_COMMIT_TAG - && export DOCKER_LATEST_TAG="-t $IMAGE_LATEST" || export DOCKER_LATEST_TAG=;
- export major="$(echo $CI_COMMIT_REF_NAME | cut -d '.' -f 1)"
- export minor="$(echo $CI_COMMIT_REF_NAME | cut -d '.' -f 1,2)"; fi
- cd api
- docker buildx create --use --name A$CI_COMMIT_SHORT_SHA
- docker buildx build --platform $BUILD_PLATFORMS --push -t $IMAGE -t $DOCKER_LATEST_TAG -t $IMAGE_NAME:$major -t $IMAGE_NAME:$minor .
docker_publish_unstable_release:
# Publish a docker image for releases
extends: .docker_publish
rules:
- if: $CI_COMMIT_TAG && $CI_COMMIT_REF_NAME !~ /^[0-9]+(.[0-9]+){1,2}$/
script:
# Check if this is the latest release
- cd api
- docker buildx create --use --name A$CI_COMMIT_SHORT_SHA
- docker buildx build --platform $BUILD_PLATFORMS --push -t $IMAGE .
docker_published_non-release:
# Publish a docker image for each commit on develop
extends: .docker_publish
only:
- develop@funkwhale/funkwhale
- stable@funkwhale/funkwhale
script:
- ./scripts/set-api-build-metadata.sh $CI_COMMIT_SHORT_SHA
- cd api
- docker build -t $IMAGE $DOCKER_LATEST_TAG .
- docker push $IMAGE
- if [[ ! -z "$DOCKER_LATEST_TAG" ]]; then docker push $IMAGE_LATEST; fi
- if [[ $stable == 1 ]]; then docker tag $IMAGE $IMAGE_NAME:$major && docker push $IMAGE_NAME:$major; fi
- if [[ $stable == 1 ]]; then docker tag $IMAGE $IMAGE_NAME:$minor && docker push $IMAGE_NAME:$minor; fi
- docker buildx create --use --name A$CI_COMMIT_SHORT_SHA
- docker buildx build --platform $BUILD_PLATFORMS --push -t $IMAGE .
docker_all_in_one_release:
stage: deploy
image: egon0/docker-with-buildx-and-git:bash
services:
- docker:20-dind
variables:
ALL_IN_ONE_REF: main
ALL_IN_ONE_ARTIFACT_URL: https://dev.funkwhale.audio/funkwhale/funkwhale-docker-all-in-one/-/archive/$ALL_IN_ONE_REF/funkwhale-docker-all-in-one-$ALL_IN_ONE_REF.zip
BUILD_PATH: all_in_one
before_script:
- docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
- (if [ "$CI_COMMIT_REF_NAME" == "develop" ] || [ "$CI_COMMIT_REF_NAME" == "master" ] || [ "$CI_COMMIT_REF_NAME" == "stable" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
script:
- if [[ ! -z "$CI_COMMIT_TAG" ]]; then (./docs/get-releases-json.py | scripts/is-docker-latest.py $CI_COMMIT_TAG -) && export DOCKER_LATEST_TAG="-t $ALL_IN_ONE_IMAGE_LATEST" || export DOCKER_LATEST_TAG=; fi
- wget $ALL_IN_ONE_ARTIFACT_URL -O all_in_one.zip
- unzip -o all_in_one.zip -d tmpdir
- mv tmpdir/funkwhale-docker-all-in-one-$ALL_IN_ONE_REF $BUILD_PATH && rmdir tmpdir
- cp -r api $BUILD_PATH/src/api
- cp -r front $BUILD_PATH/src/front
- cd $BUILD_PATH
- ./scripts/download-nginx-template.sh src/ $CI_COMMIT_REF_NAME
- docker build -t $ALL_IN_ONE_IMAGE $DOCKER_LATEST_TAG .
- docker push $ALL_IN_ONE_IMAGE
- if [[ ! -z "$DOCKER_LATEST_TAG" ]]; then docker push $ALL_IN_ONE_IMAGE_LATEST; fi
only:
- develop@funkwhale/funkwhale
- master@funkwhale/funkwhale
- stable@funkwhale/funkwhale
- tags@funkwhale/funkwhale
tags:
- docker-build
#docker_all_in_one_release:
# stage: deploy
# image: bash
# variables:
# ALL_IN_ONE_REF: master
# ALL_IN_ONE_ARTIFACT_URL: https://github.com/thetarkus/docker-funkwhale/archive/$ALL_IN_ONE_REF.zip
# BUILD_PATH: all_in_one
# before_script:
# - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
# - (if [ "$CI_COMMIT_REF_NAME" == "develop" ] || [ "$CI_COMMIT_REF_NAME" == "master" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
# script:
# - if [[ ! -z "$CI_COMMIT_TAG" ]]; then (./docs/get-releases-json.py | scripts/is-docker-latest.py $CI_COMMIT_TAG -) && export DOCKER_LATEST_TAG="-t $ALL_IN_ONE_IMAGE_LATEST" || export DOCKER_LATEST_TAG=; fi
# - wget $ALL_IN_ONE_ARTIFACT_URL -O all_in_one.zip
# - unzip -o all_in_one.zip -d tmpdir
# - mv tmpdir/docker-funkwhale-$ALL_IN_ONE_REF $BUILD_PATH && rmdir tmpdir
# - cp -r api $BUILD_PATH/src/api
# - cp -r front $BUILD_PATH/src/front
# - cd $BUILD_PATH
# - ./scripts/download-nginx-template.sh src/ $CI_COMMIT_REF_NAME
# - docker build -t $ALL_IN_ONE_IMAGE $DOCKER_LATEST_TAG .
# - docker push $ALL_IN_ONE_IMAGE
# - if [[ ! -z "$DOCKER_LATEST_TAG" ]]; then docker push $ALL_IN_ONE_IMAGE_LATEST; fi
# only:
# - develop@funkwhale/funkwhale
# - master@funkwhale/funkwhale
# - tags@funkwhale/funkwhale
# tags:
# - docker-build
build_api:
# Simply publish a zip containing api/ directory
......@@ -319,12 +353,13 @@ build_api:
- api
script:
- rm -rf api/tests
- (if [ "$CI_COMMIT_REF_NAME" == "develop" ] || [ "$CI_COMMIT_REF_NAME" == "master" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
- (if [ "$CI_COMMIT_REF_NAME" == "develop" ] || [ "$CI_COMMIT_REF_NAME" == "stable" ] || [ "$CI_COMMIT_REF_NAME" == "master" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
- chmod -R 750 api
- echo Done!
only:
- tags@funkwhale/funkwhale
- master@funkwhale/funkwhale
- stable@funkwhale/funkwhale
- develop@funkwhale/funkwhale
check_api_dependencies:
......
Please avoid merging the base branch into your feature branch. We are working with rebases and those merged tend to cause trouble.
For further questions, join us at Matrix: https://matrix.to/#/#funkwhale-dev:matrix.org
If your contribution is fixing an issue by a small change, please consider a merge into `stable` by using it as target branch.
Related issue: #XXX <!-- it's okay to have no issue for small changes -->
This Merge Request includes:
......
......@@ -10,7 +10,17 @@ This changelog is viewable on the web at https://docs.funkwhale.audio/changelog.
.. towncrier
1.1.3 (unreleased)
1.1.4 (2021-08-02)
------------------
Upgrade instructions are available at
https://docs.funkwhale.audio/admin/upgrading.html
- Pinned version of asgiref to avoid trouble with latest release. For further information, see #1516
1.1.3 (2021-08-02)
------------------
Upgrade instructions are available at
https://docs.funkwhale.audio/admin/upgrading.html
......
......@@ -100,7 +100,7 @@ Visit https://dev.funkwhale.audio/funkwhale/funkwhale and clone the repository u
A note about branches
^^^^^^^^^^^^^^^^^^^^^
Next release development occurs on the "develop" branch, and releases are made on the "master" branch. Therefore, when submitting Merge Requests, ensure you are merging on the develop branch.
Next release development occurs on the "develop" branch, and releases are made on the "stable" branch. Therefore, when submitting Merge Requests, ensure you are merging on the develop branch.
Working with docker
......@@ -750,7 +750,7 @@ To make a new 3.4 release::
export PREVIOUS_RELEASE=3.3 # replace with the previous release number
# ensure you have an up-to-date repo
git checkout develop # use master if you're doing a hotfix release
git checkout develop # use stable if you're doing a hotfix release
git pull
# compile changelog
......@@ -759,7 +759,8 @@ To make a new 3.4 release::
# polish changelog
# - update the date
# - look for typos
# - add list of contributors via `python3 scripts/get-contributions-stats.py develop $PREVIOUS_RELEASE`
# - add list of contributors via `python3 scripts/get-contributions-stats.py $NEXT_RELEASE`
git log $PREVIOUS_RELEASE.. --format="%aN" --reverse | sort | uniq # Get all commit authors since last release
nano CHANGELOG
# Set the `__version__` variable to $NEXT_RELEASE
......@@ -775,11 +776,11 @@ To make a new 3.4 release::
# publish
git push --tags && git push
# if you're doing a hotfix release from master
git checkout develop && git merge master && git push
# if you're doing a hotfix release from stable
git checkout develop && git merge stable && git push
# if you're doing a non-hotfix release, and a real release (not a real release) from develop
git checkout master && git merge develop && git push
git checkout stable && git merge develop && git push
Then, visit https://dev.funkwhale.audio/funkwhale/funkwhale/-/tags, copy-paste the changelog on the corresponding
tag, and announce the good news ;)
......@@ -2,25 +2,24 @@ FROM alpine:3.13 as builder
RUN \
echo 'installing dependencies' && \
apk add --no-cache \
git \
musl-dev \
apk add --no-cache \
git \
musl-dev \
gcc \
postgresql-dev \
python3-dev \
py3-psycopg2 \
libldap \
libffi-dev \
make \
zlib-dev \
jpeg-dev \
python3-dev \
py3-psycopg2 \
libldap \
libffi-dev \
make \
zlib-dev \
jpeg-dev \
openldap-dev \
openssl-dev \
cargo \
libxml2-dev \
libxslt-dev \
&& \
\
ln -s /usr/bin/python3 /usr/bin/python
# create virtual env for next stage
......@@ -53,21 +52,21 @@ ENV PATH="/venv/bin:$PATH"
RUN apk add --no-cache \
libmagic \
bash \
gettext \
python3 \
bash \
gettext \
python3 \
jpeg-dev \
ffmpeg \
libpq \
ffmpeg \
libpq \
libxml2 \
libxslt \
&& \
\
ln -s /usr/bin/python3 /usr/bin/python
COPY . /app
WORKDIR /app
RUN find . -type d -exec chmod 755 {} \+
ENTRYPOINT ["./compose/django/entrypoint.sh"]
CMD ["./compose/django/server.sh"]
COPY . /app
WORKDIR /app
......@@ -19,8 +19,7 @@ if [ -z "$CELERY_BROKER_URL" ]; then
fi
# we copy the frontend files, if any so we can serve them from the outside
if [ -d "frontend" ]; then
mkdir -p /frontend
if [ -d "frontend" ] && [ -d "/frontend" ]; then
cp -r frontend/* /frontend/
export FUNKWHALE_SPA_HTML_ROOT=/frontend/index.html
fi
......
......@@ -88,6 +88,7 @@ sys.path.append(FUNKWHALE_PLUGINS_PATH)
CORE_PLUGINS = [
"funkwhale_api.contrib.scrobbler",
"funkwhale_api.contrib.listenbrainz",
"funkwhale_api.contrib.maloja",
]
LOAD_CORE_PLUGINS = env.bool("FUNKWHALE_LOAD_CORE_PLUGINS", default=True)
......
# -*- coding: utf-8 -*-
__version__ = "1.1.3"
__version__ = "1.1.4"
__version_info__ = tuple(
[
int(num) if num.isdigit() else num
......
......@@ -636,6 +636,7 @@ class RssFeedItemSerializer(serializers.Serializer):
links = serializers.ListField()
tags = serializers.ListField(required=False)
summary_detail = serializers.DictField(required=False)
content = serializers.ListField(required=False)
published_parsed = DummyField(required=False)
image = serializers.DictField(required=False)
......@@ -648,6 +649,16 @@ class RssFeedItemSerializer(serializers.Serializer):
"text": content,
}
def validate_content(self, v):
# TODO: Are there RSS feeds that use more than one content item?
content = v[0].get("value")
if not content:
return
return {
"content_type": v[0].get("type", "text/plain"),
"text": content,
}
def validate_image(self, v):
url = v.get("href")
if url:
......@@ -782,7 +793,16 @@ class RssFeedItemSerializer(serializers.Serializer):
if tags:
tags_models.set_tags(track, *tags)
summary = validated_data.get("summary_detail")
# "content" refers to the <content:encoded> node in the RSS feed,
# whereas "summary_detail" refers to the <description> node.
# <description> is intended to be used as a short summary and is often
# encoded merely as plain text, whereas <content:encoded> contains
# the full episode description and is generally encoded as HTML.
#
# For details, see https://www.rssboard.org/rss-profile#element-channel-item-description
summary = validated_data.get("content")
if not summary:
summary = validated_data.get("summary_detail")
if summary:
common_utils.attach_content(track, "description", summary)
......
......@@ -297,6 +297,7 @@ SAFE_TAGS = [
"acronym",
"b",
"blockquote",
"br",
"code",
"em",
"i",
......
......@@ -5,6 +5,7 @@ PLUGIN = plugins.get_plugin_config(
name="listenbrainz",
label="ListenBrainz",
description="A plugin that allows you to submit your listens to ListenBrainz.",
homepage="https://docs.funkwhale.audio/users/builtinplugins.html#listenbrainz-plugin", # noqa
version="0.1",
user=True,
conf=[
......
import json
from config import plugins
from .funkwhale_startup import PLUGIN
class MalojaException(Exception):
pass
@plugins.register_hook(plugins.LISTENING_CREATED, PLUGIN)
def submit_listen(listening, conf, **kwargs):
server_url = conf["server_url"]
api_key = conf["api_key"]
if not server_url or not api_key:
return
logger = PLUGIN["logger"]
logger.info("Submitting listening to Majola at %s", server_url)
payload = get_payload(listening, api_key)
logger.debug("Majola payload: %r", payload)
url = server_url.rstrip("/") + "/apis/mlj_1/newscrobble"
session = plugins.get_session()
response = session.post(url, payload)
response.raise_for_status()
details = json.loads(response.text)
if details["status"] == "success":
logger.info("Majola listening submitted successfully")
else:
raise MalojaException(response.text)
def get_payload(listening, api_key):
track = listening.track
payload = {
"key": api_key,
"artists": track.artist.name,
"title": track.title,
"time": int(listening.creation_date.timestamp()),
}
if track.album:
payload["album"] = track.album.title
return payload
from config import plugins
PLUGIN = plugins.get_plugin_config(
name="maloja",
label="Maloja",
description="A plugin that allows you to submit your listens to your Maloja server.",
homepage="https://docs.funkwhale.audio/users/builtinplugins.html#maloja-plugin",
version="0.1",
user=True,
conf=[
{"name": "server_url", "type": "text", "label": "Maloja server URL"},
{"name": "api_key", "type": "text", "label": "Your Maloja API key"},
],
)
......@@ -7,7 +7,7 @@ PLUGIN = plugins.get_plugin_config(
"A plugin that enables scrobbling to ListenBrainz and Last.fm. "
"It must be configured on the server if you use Last.fm."
),
homepage="https://dev.funkwhale.audio/funkwhale/funkwhale/-/blob/develop/api/funkwhale_api/contrib/scrobbler/README.rst", # noqa
homepage="https://docs.funkwhale.audio/users/builtinplugins.html#scrobbler-plugin", # noqa
version="0.1",
user=True,
conf=[
......
......@@ -357,6 +357,8 @@ class Metadata(Mapping):
self._file = kind(filething)
if self._file is None:
raise ValueError("Cannot parse metadata from {}".format(filething))
if len(self._file) == 0:
raise ValueError("No tags found in {}".format(filething))
self.fallback = self.load_fallback(filething, self._file)
ft = self.get_file_type(self._file)
try:
......
......@@ -75,7 +75,12 @@ def dict_to_xml_tree(root_tag, d, parent=None):
root.append(dict_to_xml_tree(key, value, parent=root))
elif isinstance(value, list):
for obj in value:
root.append(dict_to_xml_tree(key, obj, parent=root))
if isinstance(obj, dict):
el = dict_to_xml_tree(key, obj, parent=root)
else:
el = ET.Element(key)
el.text = str(obj)
root.append(el)
else:
if key == "value":
root.text = str(value)
......
......@@ -49,6 +49,7 @@ def get_artist_data(artist_values):
"id": artist_values["id"],
"name": artist_values["name"],
"albumCount": artist_values["_albums_count"],
"coverArt": "ar-{}".format(artist_values["id"]),
}
......@@ -82,6 +83,8 @@ class GetArtistSerializer(serializers.Serializer):
"albumCount": len(albums),
"album": [],
}
if artist.attachment_cover_id:
payload["coverArt"] = "ar-{}".format(artist.id)
for album in albums:
album_data = {
"id": album.id,
......
......@@ -14,6 +14,7 @@ from rest_framework import permissions as rest_permissions
from rest_framework import renderers, response, viewsets
from rest_framework.decorators import action