diff --git a/changes/changelog.d/997.enhancement b/changes/changelog.d/997.enhancement new file mode 100644 index 0000000000000000000000000000000000000000..2fdb7d78b7e5347ef9f36cc714a57d429fd3844e --- /dev/null +++ b/changes/changelog.d/997.enhancement @@ -0,0 +1 @@ +Added support for CELERYD_CONCURRENCY env var to control the number of worker processes (#997) diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index bcf24422f9591414c18e5e51f833ed4d1e9bfb73..f31115b5f1171c3185e6a730cf5fc64affb0927e 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -36,7 +36,7 @@ services: # of CPUs. You can adjust this, by explicitly setting the --concurrency # flag: # celery -A funkwhale_api.taskapp worker -l INFO --concurrency=4 - command: celery -A funkwhale_api.taskapp worker -l INFO + command: celery -A funkwhale_api.taskapp worker -l INFO --concurrency=${CELERYD_CONCURRENCY-0} environment: - C_FORCE_ROOT=true volumes: diff --git a/deploy/env.prod.sample b/deploy/env.prod.sample index 672306a1ac9e975a49abb84331d98b9a1037a85b..fc88505e4951cf44745366f0466a6f26cf1112ac 100644 --- a/deploy/env.prod.sample +++ b/deploy/env.prod.sample @@ -86,6 +86,12 @@ REVERSE_PROXY_TYPE=nginx # For the Celery/asynchronous tasks part: # CELERY_BROKER_URL=redis+socket:///run/redis/redis.sock?virtual_host=0 +# Number of worker processes to execute. Defaults to 0, in which case it uses your number of CPUs +# Celery workers handle background tasks (such file imports or federation +# messaging). The more processes a worker gets, the more tasks +# can be processed in parallel. However, more processes also means +# a bigger memory footprint. +# CELERYD_CONCURRENCY=0 # Where media files (such as album covers or audio tracks) should be stored # on your system? @@ -175,4 +181,4 @@ AWS_STORAGE_BUCKET_NAME= # If you are using Amazon S3, use this setting to configure how long generated URLs should stay # valid. The default value is 3600 (60 minutes). The maximum accepted value is 604800 (7 days) -# AWS_QUERYSTRING_EXPIRE= \ No newline at end of file +# AWS_QUERYSTRING_EXPIRE= diff --git a/deploy/funkwhale-worker.service b/deploy/funkwhale-worker.service index 4df60b5e98ea5b997f882984624a3c79ebe6f01b..fbc242081912ca1977ec4258fa2822279da69ee5 100644 --- a/deploy/funkwhale-worker.service +++ b/deploy/funkwhale-worker.service @@ -8,15 +8,8 @@ User=funkwhale # adapt this depending on the path of your funkwhale installation WorkingDirectory=/srv/funkwhale/api EnvironmentFile=/srv/funkwhale/config/.env -# Celery workers handle background tasks (such file imports or federation -# messaging). The more processes a worker gets, the more tasks -# can be processed in parallel. However, more processes also means -# a bigger memory footprint. -# By default, a worker will span a number of process equal to your number -# of CPUs. You can adjust this, by explicitly setting the --concurrency -# flag: -# celery -A funkwhale_api.taskapp worker -l INFO --concurrency=4 -ExecStart=/srv/funkwhale/virtualenv/bin/celery -A funkwhale_api.taskapp worker -l INFO +ExecStart=/srv/funkwhale/virtualenv/bin/celery -A funkwhale_api.taskapp worker -l INFO --concurrency=${CELERYD_CONCURRENCY-0} + [Install] WantedBy=multi-user.target diff --git a/dev.yml b/dev.yml index c332f97c46f9213f23c8f0d6f760f93fb4ddb460..630ab68eca783998b21d3b212351bb38f76c5a8f 100644 --- a/dev.yml +++ b/dev.yml @@ -86,7 +86,7 @@ services: - postgres # - minio - redis - command: celery -A funkwhale_api.taskapp worker -l debug -B + command: celery -A funkwhale_api.taskapp worker -l debug -B --concurrency=${CELERYD_CONCURRENCY-0} environment: - "FUNKWHALE_HOSTNAME=${FUNKWHALE_HOSTNAME-localhost}" - "FUNKWHALE_HOSTNAME_SUFFIX=funkwhale.test"