From 51b0a571c0d66b2c3a390b3dc2f1b5457740d144 Mon Sep 17 00:00:00 2001
From: Eliot Berriot <contact@eliotberriot.com>
Date: Fri, 20 Dec 2019 15:36:18 +0100
Subject: [PATCH] Fix #997: Added support for CELERYD_CONCURRENCY env var to
control the number of worker processes
---
changes/changelog.d/997.enhancement | 1 +
deploy/docker-compose.yml | 2 +-
deploy/env.prod.sample | 8 +++++++-
deploy/funkwhale-worker.service | 11 ++---------
dev.yml | 2 +-
5 files changed, 12 insertions(+), 12 deletions(-)
create mode 100644 changes/changelog.d/997.enhancement
diff --git a/changes/changelog.d/997.enhancement b/changes/changelog.d/997.enhancement
new file mode 100644
index 0000000000..2fdb7d78b7
--- /dev/null
+++ b/changes/changelog.d/997.enhancement
@@ -0,0 +1 @@
+Added support for CELERYD_CONCURRENCY env var to control the number of worker processes (#997)
diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml
index bcf24422f9..f31115b5f1 100644
--- a/deploy/docker-compose.yml
+++ b/deploy/docker-compose.yml
@@ -36,7 +36,7 @@ services:
# of CPUs. You can adjust this, by explicitly setting the --concurrency
# flag:
# celery -A funkwhale_api.taskapp worker -l INFO --concurrency=4
- command: celery -A funkwhale_api.taskapp worker -l INFO
+ command: celery -A funkwhale_api.taskapp worker -l INFO --concurrency=${CELERYD_CONCURRENCY-0}
environment:
- C_FORCE_ROOT=true
volumes:
diff --git a/deploy/env.prod.sample b/deploy/env.prod.sample
index 672306a1ac..fc88505e49 100644
--- a/deploy/env.prod.sample
+++ b/deploy/env.prod.sample
@@ -86,6 +86,12 @@ REVERSE_PROXY_TYPE=nginx
# For the Celery/asynchronous tasks part:
# CELERY_BROKER_URL=redis+socket:///run/redis/redis.sock?virtual_host=0
+# Number of worker processes to execute. Defaults to 0, in which case it uses your number of CPUs
+# Celery workers handle background tasks (such file imports or federation
+# messaging). The more processes a worker gets, the more tasks
+# can be processed in parallel. However, more processes also means
+# a bigger memory footprint.
+# CELERYD_CONCURRENCY=0
# Where media files (such as album covers or audio tracks) should be stored
# on your system?
@@ -175,4 +181,4 @@ AWS_STORAGE_BUCKET_NAME=
# If you are using Amazon S3, use this setting to configure how long generated URLs should stay
# valid. The default value is 3600 (60 minutes). The maximum accepted value is 604800 (7 days)
-# AWS_QUERYSTRING_EXPIRE=
\ No newline at end of file
+# AWS_QUERYSTRING_EXPIRE=
diff --git a/deploy/funkwhale-worker.service b/deploy/funkwhale-worker.service
index 4df60b5e98..fbc2420819 100644
--- a/deploy/funkwhale-worker.service
+++ b/deploy/funkwhale-worker.service
@@ -8,15 +8,8 @@ User=funkwhale
# adapt this depending on the path of your funkwhale installation
WorkingDirectory=/srv/funkwhale/api
EnvironmentFile=/srv/funkwhale/config/.env
-# Celery workers handle background tasks (such file imports or federation
-# messaging). The more processes a worker gets, the more tasks
-# can be processed in parallel. However, more processes also means
-# a bigger memory footprint.
-# By default, a worker will span a number of process equal to your number
-# of CPUs. You can adjust this, by explicitly setting the --concurrency
-# flag:
-# celery -A funkwhale_api.taskapp worker -l INFO --concurrency=4
-ExecStart=/srv/funkwhale/virtualenv/bin/celery -A funkwhale_api.taskapp worker -l INFO
+ExecStart=/srv/funkwhale/virtualenv/bin/celery -A funkwhale_api.taskapp worker -l INFO --concurrency=${CELERYD_CONCURRENCY-0}
+
[Install]
WantedBy=multi-user.target
diff --git a/dev.yml b/dev.yml
index c332f97c46..630ab68eca 100644
--- a/dev.yml
+++ b/dev.yml
@@ -86,7 +86,7 @@ services:
- postgres
# - minio
- redis
- command: celery -A funkwhale_api.taskapp worker -l debug -B
+ command: celery -A funkwhale_api.taskapp worker -l debug -B --concurrency=${CELERYD_CONCURRENCY-0}
environment:
- "FUNKWHALE_HOSTNAME=${FUNKWHALE_HOSTNAME-localhost}"
- "FUNKWHALE_HOSTNAME_SUFFIX=funkwhale.test"
--
GitLab