diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d0c609165735e56d993442c1a6795431da5e3afc..f238e440fdd3e2e882c40ebe117b5a6de37a24f7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,22 +1,24 @@ +# You can override the included template(s) by including variable overrides +# See https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings +# Note that environment variables can be set in several places +# See https://docs.gitlab.com/ee/ci/variables/#priority-of-environment-variables variables: IMAGE_NAME: funkwhale/funkwhale - IMAGE: $IMAGE_NAME:$CI_COMMIT_REF_NAME - IMAGE_LATEST: $IMAGE_NAME:latest + IMAGE: "$IMAGE_NAME:$CI_COMMIT_REF_NAME" + IMAGE_LATEST: "$IMAGE_NAME:latest" ALL_IN_ONE_IMAGE_NAME: funkwhale/all-in-one - ALL_IN_ONE_IMAGE: $ALL_IN_ONE_IMAGE_NAME:$CI_COMMIT_REF_NAME - ALL_IN_ONE_IMAGE_LATEST: $ALL_IN_ONE_IMAGE_NAME:latest + ALL_IN_ONE_IMAGE: "$ALL_IN_ONE_IMAGE_NAME:$CI_COMMIT_REF_NAME" + ALL_IN_ONE_IMAGE_LATEST: "$ALL_IN_ONE_IMAGE_NAME:latest" PIP_CACHE_DIR: "$CI_PROJECT_DIR/pip-cache" - PYTHONDONTWRITEBYTECODE: "true" + PYTHONDONTWRITEBYTECODE: 'true' REVIEW_DOMAIN: preview.funkwhale.audio REVIEW_INSTANCE_URL: https://demo.funkwhale.audio - stages: - - review - - lint - - test - - build - - deploy - +- review +- lint +- test +- build +- deploy review_front: interruptible: true stage: review @@ -24,39 +26,36 @@ review_front: when: manual allow_failure: true variables: - BASE_URL: /-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/front-review/ - VUE_APP_ROUTER_BASE_URL: /-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/front-review/ - VUE_APP_INSTANCE_URL: $REVIEW_INSTANCE_URL + BASE_URL: "/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/front-review/" + VUE_APP_ROUTER_BASE_URL: "/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/front-review/" + VUE_APP_INSTANCE_URL: "$REVIEW_INSTANCE_URL" before_script: - - curl -L -o /usr/local/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 - - chmod +x /usr/local/bin/jq - - rm -rf front-review - - mkdir front-review - - cd front + - curl -L -o /usr/local/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 + - chmod +x /usr/local/bin/jq + - rm -rf front-review + - mkdir front-review + - cd front script: - - yarn install - - yarn run i18n-compile - # this is to ensure we don't have any errors in the output, - # cf https://dev.funkwhale.audio/funkwhale/funkwhale/issues/169 - - yarn run build | tee /dev/stderr | (! grep -i 'ERROR in') - - cp -r dist/* ../front-review + - yarn install + - yarn run i18n-compile + - yarn run build | tee /dev/stderr | (! grep -i 'ERROR in') + - cp -r dist/* ../front-review artifacts: expire_in: 2 weeks paths: - - front-review + - front-review cache: - key: "funkwhale__front_dependencies" + key: funkwhale__front_dependencies paths: - - front/node_modules - - front/yarn.lock + - front/node_modules + - front/yarn.lock only: - - branches + - branches tags: - - docker + - docker environment: name: review/front/$CI_COMMIT_REF_NAME url: http://$CI_PROJECT_NAMESPACE.pages.funkwhale.audio/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/front-review/index.html - review_docs: interruptible: true stage: review @@ -66,30 +65,29 @@ review_docs: variables: BUILD_PATH: "../docs-review" before_script: - - rm -rf docs-review - - mkdir docs-review - - cd docs - - apt-get update - - apt-get install -y graphviz - - pip install sphinx sphinx_rtd_theme django-environ django + - rm -rf docs-review + - mkdir docs-review + - cd docs + - apt-get update + - apt-get install -y graphviz + - pip install sphinx sphinx_rtd_theme django-environ django script: - - ./build_docs.sh + - "./build_docs.sh" cache: key: "$CI_PROJECT_ID__sphinx" paths: - - "$PIP_CACHE_DIR" + - "$PIP_CACHE_DIR" artifacts: expire_in: 2 weeks paths: - - docs-review + - docs-review only: - - branches + - branches tags: - - docker + - docker environment: name: review/docs/$CI_COMMIT_REF_NAME url: http://$CI_PROJECT_NAMESPACE.pages.funkwhale.audio/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/docs-review/index.html - black: interruptible: true image: python:3.6 @@ -97,10 +95,9 @@ black: variables: GIT_STRATEGY: fetch before_script: - - pip install black==19.10b0 + - pip install black==19.10b0 script: - - black --check --diff api/ - + - black --check --diff api/ flake8: interruptible: true image: python:3.6 @@ -108,137 +105,133 @@ flake8: variables: GIT_STRATEGY: fetch before_script: - - pip install 'flake8<3.7' + - pip install 'flake8<3.7' script: - - flake8 -v api + - flake8 -v api cache: key: "$CI_PROJECT_ID__flake8_pip_cache" paths: - - "$PIP_CACHE_DIR" - + - "$PIP_CACHE_DIR" test_api: interruptible: true services: - - postgres:11 - - redis:5 + - postgres:11 + - redis:5 stage: test image: funkwhale/funkwhale:develop cache: key: "$CI_PROJECT_ID__pip_cache" paths: - - "$PIP_CACHE_DIR" + - "$PIP_CACHE_DIR" variables: - DATABASE_URL: "postgresql://postgres@postgres/postgres" - FUNKWHALE_URL: "https://funkwhale.ci" + DATABASE_URL: postgresql://postgres@postgres/postgres + FUNKWHALE_URL: https://funkwhale.ci DJANGO_SETTINGS_MODULE: config.settings.local POSTGRES_HOST_AUTH_METHOD: trust only: - - branches + - branches before_script: - - apk add make git gcc python3-dev musl-dev - - apk add postgresql-dev py3-psycopg2 libldap libffi-dev make zlib-dev jpeg-dev openldap-dev - - cd api - - pip3 install -r requirements/base.txt - - pip3 install -r requirements/local.txt - - pip3 install -r requirements/test.txt + - apk add make git gcc python3-dev musl-dev + - apk add postgresql-dev py3-psycopg2 libldap libffi-dev make zlib-dev jpeg-dev + openldap-dev + - cd api + - pip3 install -r requirements/base.txt + - pip3 install -r requirements/local.txt + - pip3 install -r requirements/test.txt script: - - pytest --cov=funkwhale_api tests/ + - pytest --cov=funkwhale_api tests/ tags: - - docker - + - docker test_front: interruptible: true stage: test image: node:12-buster before_script: - - cd front + - cd front only: - - branches + - branches script: - - yarn install --check-files - - yarn test:unit + - yarn install --check-files + - yarn test:unit cache: - key: "funkwhale__front_dependencies" + key: funkwhale__front_dependencies paths: - - front/node_modules - - front/yarn.lock + - front/node_modules + - front/yarn.lock artifacts: - name: "front_${CI_COMMIT_REF_NAME}" + name: front_${CI_COMMIT_REF_NAME} paths: - - front/dist/ + - front/dist/ tags: - - docker - + - docker build_front: stage: build image: node:12-buster before_script: - - curl -L -o /usr/local/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 - - chmod +x /usr/local/bin/jq - - cd front + - curl -L -o /usr/local/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 + - chmod +x /usr/local/bin/jq + - cd front script: - - yarn install - - yarn run i18n-compile - # this is to ensure we don't have any errors in the output, - # cf https://dev.funkwhale.audio/funkwhale/funkwhale/issues/169 - - yarn build | tee /dev/stderr | (! grep -i 'ERROR in') - - chmod -R 755 dist + - yarn install + - yarn run i18n-compile + - yarn build | tee /dev/stderr | (! grep -i 'ERROR in') + - chmod -R 755 dist artifacts: - name: "front_${CI_COMMIT_REF_NAME}" + name: front_${CI_COMMIT_REF_NAME} paths: - - front/dist/ + - front/dist/ only: - - tags@funkwhale/funkwhale - - master@funkwhale/funkwhale - - develop@funkwhale/funkwhale - + - tags@funkwhale/funkwhale + - master@funkwhale/funkwhale + - develop@funkwhale/funkwhale tags: - - docker - + - docker pages: stage: test image: python:3.6 variables: BUILD_PATH: "../public" before_script: - - cd docs - - apt-get update - - apt-get install -y graphviz - - pip install sphinx sphinx_rtd_theme django-environ django + - cd docs + - apt-get update + - apt-get install -y graphviz + - pip install sphinx sphinx_rtd_theme django-environ django script: - - ./build_docs.sh + - "./build_docs.sh" cache: key: "$CI_PROJECT_ID__sphinx" paths: - - "$PIP_CACHE_DIR" + - "$PIP_CACHE_DIR" artifacts: paths: - - public + - public only: - - master@funkwhale/funkwhale + - master@funkwhale/funkwhale tags: - - docker - + - docker docker_release: stage: deploy image: bash before_script: - - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD - - cp -r front/dist api/frontend - - (if [ "$CI_COMMIT_REF_NAME" == "develop" ] || [ "$CI_COMMIT_REF_NAME" == "master" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi); + - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD + - cp -r front/dist api/frontend + - (if [ "$CI_COMMIT_REF_NAME" == "develop" ] || [ "$CI_COMMIT_REF_NAME" == "master" + ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); + fi); script: - - if [[ ! -z "$CI_COMMIT_TAG" ]]; then (./docs/get-releases-json.py | scripts/is-docker-latest.py $CI_COMMIT_TAG -) && export DOCKER_LATEST_TAG="-t $IMAGE_LATEST" || export DOCKER_LATEST_TAG=; fi - - cd api - - docker build -t $IMAGE $DOCKER_LATEST_TAG . - - docker push $IMAGE - - if [[ ! -z "$DOCKER_LATEST_TAG" ]]; then docker push $IMAGE_LATEST; fi + - if [[ ! -z "$CI_COMMIT_TAG" ]]; then (./docs/get-releases-json.py | scripts/is-docker-latest.py + $CI_COMMIT_TAG -) && export DOCKER_LATEST_TAG="-t $IMAGE_LATEST" || export DOCKER_LATEST_TAG=; + fi + - cd api + - docker build -t $IMAGE $DOCKER_LATEST_TAG . + - docker push $IMAGE + - if [[ ! -z "$DOCKER_LATEST_TAG" ]]; then docker push $IMAGE_LATEST; fi only: - - develop@funkwhale/funkwhale - - master@funkwhale/funkwhale - - tags@funkwhale/funkwhale + - develop@funkwhale/funkwhale + - master@funkwhale/funkwhale + - tags@funkwhale/funkwhale tags: - - docker-build - + - docker-build docker_all_in_one_release: stage: deploy image: bash @@ -247,41 +240,50 @@ docker_all_in_one_release: ALL_IN_ONE_ARTIFACT_URL: https://github.com/thetarkus/docker-funkwhale/archive/$ALL_IN_ONE_REF.zip BUILD_PATH: all_in_one before_script: - - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD - - (if [ "$CI_COMMIT_REF_NAME" == "develop" ] || [ "$CI_COMMIT_REF_NAME" == "master" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi); + - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD + - (if [ "$CI_COMMIT_REF_NAME" == "develop" ] || [ "$CI_COMMIT_REF_NAME" == "master" + ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); + fi); script: - - if [[ ! -z "$CI_COMMIT_TAG" ]]; then (./docs/get-releases-json.py | scripts/is-docker-latest.py $CI_COMMIT_TAG -) && export DOCKER_LATEST_TAG="-t $ALL_IN_ONE_IMAGE_LATEST" || export DOCKER_LATEST_TAG=; fi - - wget $ALL_IN_ONE_ARTIFACT_URL -O all_in_one.zip - - unzip -o all_in_one.zip -d tmpdir - - mv tmpdir/docker-funkwhale-$ALL_IN_ONE_REF $BUILD_PATH && rmdir tmpdir - - cp -r api $BUILD_PATH/src/api - - cp -r front $BUILD_PATH/src/front - - cd $BUILD_PATH - - ./scripts/download-nginx-template.sh src/ $CI_COMMIT_REF_NAME - - docker build -t $ALL_IN_ONE_IMAGE $DOCKER_LATEST_TAG . - - docker push $ALL_IN_ONE_IMAGE - - if [[ ! -z "$DOCKER_LATEST_TAG" ]]; then docker push $ALL_IN_ONE_IMAGE_LATEST; fi + - if [[ ! -z "$CI_COMMIT_TAG" ]]; then (./docs/get-releases-json.py | scripts/is-docker-latest.py + $CI_COMMIT_TAG -) && export DOCKER_LATEST_TAG="-t $ALL_IN_ONE_IMAGE_LATEST" || + export DOCKER_LATEST_TAG=; fi + - wget $ALL_IN_ONE_ARTIFACT_URL -O all_in_one.zip + - unzip -o all_in_one.zip -d tmpdir + - mv tmpdir/docker-funkwhale-$ALL_IN_ONE_REF $BUILD_PATH && rmdir tmpdir + - cp -r api $BUILD_PATH/src/api + - cp -r front $BUILD_PATH/src/front + - cd $BUILD_PATH + - "./scripts/download-nginx-template.sh src/ $CI_COMMIT_REF_NAME" + - docker build -t $ALL_IN_ONE_IMAGE $DOCKER_LATEST_TAG . + - docker push $ALL_IN_ONE_IMAGE + - if [[ ! -z "$DOCKER_LATEST_TAG" ]]; then docker push $ALL_IN_ONE_IMAGE_LATEST; + fi only: - - develop@funkwhale/funkwhale - - master@funkwhale/funkwhale - - tags@funkwhale/funkwhale + - develop@funkwhale/funkwhale + - master@funkwhale/funkwhale + - tags@funkwhale/funkwhale tags: - - docker-build - + - docker-build build_api: - # Simply publish a zip containing api/ directory stage: deploy image: bash artifacts: - name: "api_${CI_COMMIT_REF_NAME}" + name: api_${CI_COMMIT_REF_NAME} paths: - - api + - api script: - - rm -rf api/tests - - (if [ "$CI_COMMIT_REF_NAME" == "develop" ] || [ "$CI_COMMIT_REF_NAME" == "master" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi); - - chmod -R 750 api - - echo Done! + - rm -rf api/tests + - (if [ "$CI_COMMIT_REF_NAME" == "develop" ] || [ "$CI_COMMIT_REF_NAME" == "master" + ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); + fi); + - chmod -R 750 api + - echo Done! only: - - tags@funkwhale/funkwhale - - master@funkwhale/funkwhale - - develop@funkwhale/funkwhale + - tags@funkwhale/funkwhale + - master@funkwhale/funkwhale + - develop@funkwhale/funkwhale +sast: + stage: test +include: +- template: Security/SAST.gitlab-ci.yml