Commit 1ef252e2 authored by Georg Krause's avatar Georg Krause
Browse files

Implement several pipeline improvements

* Build docker image to run tests in
* Collect backend coverage reports
* Update dependencies
parent 58447196
...@@ -13,8 +13,8 @@ variables: ...@@ -13,8 +13,8 @@ variables:
stages: stages:
- review - review
- lint - lint
- test
- build - build
- test
- deploy - deploy
- deps - deps
...@@ -123,7 +123,7 @@ test_api: ...@@ -123,7 +123,7 @@ test_api:
- postgres:11 - postgres:11
- redis:5 - redis:5
stage: test stage: test
image: funkwhale/funkwhale:develop image: $CI_REGISTRY/$IMAGE
cache: cache:
key: "$CI_PROJECT_ID__pip_cache" key: "$CI_PROJECT_ID__pip_cache"
paths: paths:
...@@ -136,20 +136,21 @@ test_api: ...@@ -136,20 +136,21 @@ test_api:
only: only:
- branches - branches
before_script: before_script:
- apk add make git gcc python3-dev musl-dev - apk add make git gcc python3-dev musl-dev python3-dev openssl-dev cargo
- apk add postgresql-dev py3-psycopg2 libldap libffi-dev make zlib-dev jpeg-dev openldap-dev - apk add postgresql-dev py3-psycopg2 libldap libffi-dev make zlib-dev jpeg-dev openldap-dev
- cd api - cd api
- pip3 install -r requirements/base.txt - pip3 install -r requirements/base.txt
- pip3 install -r requirements/local.txt - pip3 install -r requirements/local.txt
- pip3 install -r requirements/test.txt - pip3 install -r requirements/test.txt
script: script:
- pytest --cov=funkwhale_api --junitxml=report.xml tests/ - pytest --cov-report xml --cov=funkwhale_api --junitxml=report.xml tests/
tags: tags:
- docker - docker
artifacts: artifacts:
when: always when: always
reports: reports:
junit: api/report.xml junit: api/report.xml
cobertura: api/coverage.xml
test_front: test_front:
interruptible: true interruptible: true
...@@ -198,10 +199,21 @@ build_front: ...@@ -198,10 +199,21 @@ build_front:
- tags@funkwhale/funkwhale - tags@funkwhale/funkwhale
- master@funkwhale/funkwhale - master@funkwhale/funkwhale
- develop@funkwhale/funkwhale - develop@funkwhale/funkwhale
tags: tags:
- docker - docker
build_backend:
stage: build
image: bash
before_script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
script:
- cd api
- docker build -t $CI_REGISTRY/$IMAGE .
- docker push $CI_REGISTRY/$IMAGE
tags:
- docker-build
pages: pages:
stage: test stage: test
image: python:3.6 image: python:3.6
...@@ -296,7 +308,7 @@ build_api: ...@@ -296,7 +308,7 @@ build_api:
check_api_dependencies: check_api_dependencies:
interruptible: true interruptible: true
stage: deps stage: deps
image: funkwhale/funkwhale:develop image: $CI_REGISTRY/$IMAGE
cache: cache:
key: "$CI_PROJECT_ID__pip_cache" key: "$CI_PROJECT_ID__pip_cache"
paths: paths:
......
FROM alpine:3.10 as builder FROM alpine:3.13 as builder
RUN \ RUN \
echo 'installing dependencies' && \ echo 'installing dependencies' && \
...@@ -15,6 +15,8 @@ RUN \ ...@@ -15,6 +15,8 @@ RUN \
zlib-dev \ zlib-dev \
jpeg-dev \ jpeg-dev \
openldap-dev \ openldap-dev \
openssl-dev \
cargo \
&& \ && \
\ \
ln -s /usr/bin/python3 /usr/bin/python ln -s /usr/bin/python3 /usr/bin/python
...@@ -41,7 +43,7 @@ RUN \ ...@@ -41,7 +43,7 @@ RUN \
if [ "$install_dev_deps" = "1" ] ; then echo "Installing dev dependencies" && pip3 install --no-cache-dir -r /requirements/local.txt -r /requirements/test.txt ; else echo "Skipping dev deps installation" ; fi if [ "$install_dev_deps" = "1" ] ; then echo "Installing dev dependencies" && pip3 install --no-cache-dir -r /requirements/local.txt -r /requirements/test.txt ; else echo "Skipping dev deps installation" ; fi
FROM alpine:3.10 as build-image FROM alpine:3.13 as build-image
COPY --from=builder /venv /venv COPY --from=builder /venv /venv
# emulate activation by prefixing PATH # emulate activation by prefixing PATH
......
...@@ -142,7 +142,6 @@ class Domain(models.Model): ...@@ -142,7 +142,6 @@ class Domain(models.Model):
from funkwhale_api.music import models as music_models from funkwhale_api.music import models as music_models
data = Domain.objects.filter(pk=self.pk).aggregate( data = Domain.objects.filter(pk=self.pk).aggregate(
actors=models.Count("actors", distinct=True),
outbox_activities=models.Count("actors__outbox_activities", distinct=True), outbox_activities=models.Count("actors__outbox_activities", distinct=True),
libraries=models.Count("actors__libraries", distinct=True), libraries=models.Count("actors__libraries", distinct=True),
channels=models.Count("actors__owned_channels", distinct=True), channels=models.Count("actors__owned_channels", distinct=True),
...@@ -152,6 +151,7 @@ class Domain(models.Model): ...@@ -152,6 +151,7 @@ class Domain(models.Model):
emitted_library_follows=models.Count( emitted_library_follows=models.Count(
"actors__library_follows", distinct=True "actors__library_follows", distinct=True
), ),
actors=models.Count("actors", distinct=True),
) )
data["artists"] = music_models.Artist.objects.filter( data["artists"] = music_models.Artist.objects.filter(
from_activity__actor__domain_id=self.pk from_activity__actor__domain_id=self.pk
...@@ -283,12 +283,12 @@ class Actor(models.Model): ...@@ -283,12 +283,12 @@ class Actor(models.Model):
data = Actor.objects.filter(pk=self.pk).aggregate( data = Actor.objects.filter(pk=self.pk).aggregate(
outbox_activities=models.Count("outbox_activities", distinct=True), outbox_activities=models.Count("outbox_activities", distinct=True),
libraries=models.Count("libraries", distinct=True),
channels=models.Count("owned_channels", distinct=True), channels=models.Count("owned_channels", distinct=True),
received_library_follows=models.Count( received_library_follows=models.Count(
"libraries__received_follows", distinct=True "libraries__received_follows", distinct=True
), ),
emitted_library_follows=models.Count("library_follows", distinct=True), emitted_library_follows=models.Count("library_follows", distinct=True),
libraries=models.Count("libraries", distinct=True),
) )
data["artists"] = music_models.Artist.objects.filter( data["artists"] = music_models.Artist.objects.filter(
from_activity__actor=self.pk from_activity__actor=self.pk
......
...@@ -461,7 +461,9 @@ class PreserveSomeDataCollector(Collector): ...@@ -461,7 +461,9 @@ class PreserveSomeDataCollector(Collector):
def related_objects(self, related, *args, **kwargs): def related_objects(self, related, *args, **kwargs):
qs = super().related_objects(related, *args, **kwargs) qs = super().related_objects(related, *args, **kwargs)
if related.name == "outbox_activities": # We can only exclude the actions if these fields are available, most likely its a
# model.Activity than
if hasattr(related, "type") and hasattr(related, "creation_date"):
# exclude the delete activity can be broadcasted properly # exclude the delete activity can be broadcasted properly
qs = qs.exclude(type="Delete", creation_date__gte=self.creation_date) qs = qs.exclude(type="Delete", creation_date__gte=self.creation_date)
......
django~=3.0.8 django~=3.2.3
setuptools>=49 setuptools~=56.2.0
# Configuration # Configuration
django-environ~=0.4.0 django-environ~=0.4.0
...@@ -11,7 +11,7 @@ django-allauth~=0.42.0 ...@@ -11,7 +11,7 @@ django-allauth~=0.42.0
psycopg2-binary~=2.8.0 psycopg2-binary~=2.8.0
# Time zones support # Time zones support
pytz==2020.1 pytz==2021.1
# Redis support # Redis support
django-redis~=4.12.0 django-redis~=4.12.0
...@@ -29,7 +29,7 @@ persisting-theory~=0.2.0 ...@@ -29,7 +29,7 @@ persisting-theory~=0.2.0
django-versatileimagefield~=2.0.0 django-versatileimagefield~=2.0.0
django-filter~=2.4.0 django-filter~=2.4.0
django-rest-auth~=0.9.0 django-rest-auth~=0.9.0
ipython~=7.22.0 ipython~=7.23.1
mutagen~=1.45.0 mutagen~=1.45.0
pymemoize~=1.0.0 pymemoize~=1.0.0
...@@ -41,13 +41,13 @@ channels_redis~=3.2.0 ...@@ -41,13 +41,13 @@ channels_redis~=3.2.0
uvicorn[standard]~=0.13.4 uvicorn[standard]~=0.13.4
gunicorn~=20.1.0 gunicorn~=20.1.0
cryptography~=2.9.0 cryptography~=3.4.7
# requests-http-signature==0.0.3 # requests-http-signature==0.0.3
# clone until the branch is merged and released upstream # clone until the branch is merged and released upstream
git+https://github.com/agateblue/requests-http-signature.git@signature-header-support git+https://github.com/agateblue/requests-http-signature.git@signature-header-support
django-cleanup~=5.2.0 django-cleanup~=5.2.0
requests~=2.25.1 requests~=2.25.1
pyOpenSSL~=19.1.0 pyOpenSSL~=20.0.1
# for LDAP authentication # for LDAP authentication
python-ldap~=3.3.0 python-ldap~=3.3.0
...@@ -61,14 +61,14 @@ django-oauth-toolkit~=1.3.0 ...@@ -61,14 +61,14 @@ django-oauth-toolkit~=1.3.0
django-storages~=1.11.1 django-storages~=1.11.1
boto3~=1.17.59 boto3~=1.17.59
unicode-slugify~=0.1.0 unicode-slugify~=0.1.0
django-cacheops~=5.1.0 django-cacheops~=6.0.0
service_identity~=21.1.0
click~=7.1.0 click~=7.1.0
service_identity~=18.1.0
markdown~=3.3.4 markdown~=3.3.4
bleach~=3.3.0 bleach~=3.3.0
feedparser~=6.0.0 feedparser~=6.0.0
watchdog~=2.0.3 watchdog~=2.1.2
## Pin third party dependency to avoid issue with latest version ## Pin third party dependency to avoid issue with latest version
twisted==20.3.0 twisted==20.3.0
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
coverage~=5.5.0 coverage~=5.5.0
django_coverage_plugin~=1.8.0 django_coverage_plugin~=1.8.0
factory_boy~=2.11.0 factory_boy~=2.12.0
# django-debug-toolbar that works with Django 1.5+ # django-debug-toolbar that works with Django 1.5+
django-debug-toolbar~=3.2.1 django-debug-toolbar~=3.2.1
......
...@@ -2,11 +2,11 @@ ...@@ -2,11 +2,11 @@
flake8~=3.9.1 flake8~=3.9.1
pytest~=6.2.3 pytest~=6.2.3
pytest-cov~=2.11.1 pytest-cov~=2.12.0
pytest-django~=4.2.0 pytest-django~=4.3.0
pytest-env~=0.6.0 pytest-env~=0.6.0
pytest-mock~=3.6.0 pytest-mock~=3.6.0
pytest-randomly~=3.7.0 pytest-randomly~=3.8.0
pytest-sugar~=0.9.0 pytest-sugar~=0.9.0
requests-mock~=1.9.0 requests-mock~=1.9.0
faker~=8.1.1 faker~=8.2.0
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment