Commit 101ae278 authored by Eliot Berriot's avatar Eliot Berriot 💬

Fix #565: store media files in S3 bucket

parent 31d99049
Pipeline #4016 passed with stages
in 3 minutes and 35 seconds
......@@ -306,6 +306,28 @@ STATIC_ROOT = env("STATIC_ROOT", default=str(ROOT_DIR("staticfiles")))
STATIC_URL = env("STATIC_URL", default="/staticfiles/")
DEFAULT_FILE_STORAGE = "funkwhale_api.common.storage.ASCIIFileSystemStorage"
AWS_DEFAULT_ACL = None
AWS_QUERYSTRING_AUTH = False
# MINIO_ACCESS_KEY_ID = env("MINIO_ACCESS_KEY_ID", default=None)
# if MINIO_ACCESS_KEY_ID:
# AWS_ACCESS_KEY_ID = MINIO_ACCESS_KEY_ID
# AWS_SECRET_ACCESS_KEY = env("MINIO_SECRET_KEY")
# AWS_STORAGE_BUCKET_NAME = env("MINIO_STORAGE_BUCKET_NAME")
# AWS_S3_ENDPOINT_URL = env("MINIO_URL")
# AWS_LOCATION = env("MINIO_BUCKET_DIRECTORY", default="")
# DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
AWS_ACCESS_KEY_ID = env("AWS_ACCESS_KEY_ID", default=None)
if AWS_ACCESS_KEY_ID:
AWS_ACCESS_KEY_ID = AWS_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY = env("AWS_SECRET_ACCESS_KEY")
AWS_STORAGE_BUCKET_NAME = env("AWS_STORAGE_BUCKET_NAME")
AWS_S3_ENDPOINT_URL = env("AWS_S3_ENDPOINT_URL", default=None)
AWS_LOCATION = env("AWS_LOCATION", default="")
DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
# See: https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#std:setting-STATICFILES_DIRS
STATICFILES_DIRS = (str(APPS_DIR.path("static")),)
......
......@@ -838,7 +838,7 @@ class AlbumSerializer(MusicEntitySerializer):
d["cover"] = {
"type": "Link",
"href": utils.full_url(instance.cover.url),
"mediaType": mimetypes.guess_type(instance.cover.path)[0]
"mediaType": mimetypes.guess_type(instance.cover_path)[0]
or "image/jpeg",
}
if self.context.get("include_ap_context", self.parent is None):
......
......@@ -346,6 +346,16 @@ class Album(APIModelMixin):
def __str__(self):
return self.title
@property
def cover_path(self):
if not self.cover:
return None
try:
return self.cover.path
except NotImplementedError:
# external storage
return self.cover.name
@property
def tags(self):
t = []
......
......@@ -240,6 +240,9 @@ def get_file_path(audio_file):
"MUSIC_DIRECTORY_PATH to serve in-place imported files"
)
path = "/music" + audio_file.replace(prefix, "", 1)
if path.startswith("http://") or path.startswith("https://"):
raise
return (settings.PROTECT_FILES_PATH + "/media/" + path).encode("utf-8")
return (settings.PROTECT_FILES_PATH + path).encode("utf-8")
if t == "apache2":
try:
......
......@@ -69,3 +69,5 @@ aiohttp==3.5.4
autobahn>=19.3.2
django-oauth-toolkit==1.2
django-storages==1.7.1
boto3
Support S3-compatible storages for media files (#565)
......@@ -79,6 +79,16 @@ or invalid, and additional debug information to share in your support requests.
This information is available in all pages that list uploads, when clicking on the button next to the upload status.
Support for S3-compatible storages to store media files
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Storing all media files on the Funkwhale server itself may not be possible or desirable
in all scenarios. You can now configure Funkwhale to store those files in a S3
bucket instead.
Check-out `https://docs.funkwhale.audio/admin/external-storages.html`_ if you want to use
this feature.
Prune library command
^^^^^^^^^^^^^^^^^^^^^
......
......@@ -57,13 +57,20 @@ server {
alias ${MEDIA_ROOT}/;
}
# this is an internal location that is used to serve
# audio files once correct permission / authentication
# has been checked on API side
location /_protected/media {
# this is an internal location that is used to serve
# audio files once correct permission / authentication
# has been checked on API side
internal;
alias ${MEDIA_ROOT};
}
# Comment the previous location and uncomment this one if you're storing
# media files in a S3 bucket
# location ~ /_protected/media/(.+) {
# internal;
# proxy_pass $1;
# }
location /_protected/music {
# this is an internal location that is used to serve
......
......@@ -136,3 +136,19 @@ FUNKWHALE_FRONTEND_PATH=/srv/funkwhale/front/dist
# Nginx related configuration
NGINX_MAX_BODY_SIZE=100M
## External storages configuration
# Funkwhale can store uploaded files on Amazon S3 and S3-compatible storages (such as Minio)
# Uncomment and fill the variables below
AWS_ACCESS_KEY_ID=
AWS_SECRET_ACCESS_KEY=
AWS_STORAGE_BUCKET_NAME=
# An optional bucket subdirectory were you want to store the files. This is especially useful
# if you plan to use share the bucket with other services
# AWS_LOCATION=
# If you use a S3-compatible storage such as minio, set the following variable
# the full URL to the storage server. Example:
# AWS_S3_ENDPOINT_URL=https://minio.mydomain.com
# AWS_S3_ENDPOINT_URL=
......@@ -109,8 +109,22 @@ server {
# audio files once correct permission / authentication
# has been checked on API side
internal;
}
# this is an internal location that is used to serve
# audio files once correct permission / authentication
# has been checked on API side
location /_protected/media {
internal;
alias ${MEDIA_ROOT};
}
# Comment the previous location and uncomment this one if you're storing
# media files in a S3 bucket
# location ~ /_protected/media/(.+) {
# internal;
# proxy_pass $1;
# }
location /_protected/music {
# this is an internal location that is used to serve
......
......@@ -63,6 +63,7 @@ services:
depends_on:
- postgres
# - minio
- redis
networks:
- internal
......@@ -76,6 +77,7 @@ services:
build: *backend
depends_on:
- postgres
# - minio
- redis
command: celery -A funkwhale_api.taskapp worker -l debug -B
environment:
......@@ -146,6 +148,23 @@ services:
volumes:
- "./docs/swagger.yml:/usr/share/nginx/html/swagger.yml"
# minio:
# image: minio/minio
# command: server /data
# volumes:
# - "./data/${COMPOSE_PROJECT_NAME-node1}/minio:/data"
# environment:
# - "MINIO_ACCESS_KEY=${AWS_ACCESS_KEY_ID-access_key}"
# - "MINIO_SECRET_KEY=${AWS_SECRET_ACCESS_KEY-secret_key}"
# - "MINIO_HTTP_TRACE: /dev/stdout"
# ports:
# - "9000:9000"
# networks:
# - federation
# - internal
networks:
? internal
federation:
......
......@@ -93,13 +93,21 @@ http {
alias /protected/media/;
}
# this is an internal location that is used to serve
# audio files once correct permission / authentication
# has been checked on API side
location /_protected/media {
# this is an internal location that is used to serve
# audio files once correct permission / authentication
# has been checked on API side
internal;
alias /protected/media;
}
# Comment the previous location and uncomment this one if you're storing
# media files in a S3 bucket
# location ~ /_protected/media/(.+) {
# internal;
# resolver 127.0.0.11;
# proxy_pass $1;
# }
location /_protected/music {
# this is an internal location that is used to serve
......
Using external storages to store Funkwhale content
==================================================
By default, Funkwhale will store user-uploaded and related media such as audio files,
transcoded files, avatars and album covers on a server directory.
However, for bigger instances or more complex deployment scenarios, you may want
to use distributed or external storages.
S3 and S3-compatible servers
----------------------------
.. note::
This feature was released in Funkwhale 0.19 and is still considered experimental.
Please let us know if you see anything unusual while using it.
Funkwhale supports storing media files Amazon S3 and compatible implementations such as Minio or Wasabi.
In this scenario, the content itself is stored in the S3 bucket. Non-sensitive media such as
album covers or user avatars are served directly from the bucket. However, audio files
are still served by the reverse proxy, to enforce proper authentication.
To enable S3 on Funkwhale, add the following environment variables::
AWS_ACCESS_KEY_ID=
AWS_SECRET_ACCESS_KEY=
AWS_STORAGE_BUCKET_NAME=
# An optional bucket subdirectory were you want to store the files. This is especially useful
# if you plan to use share the bucket with other services
# AWS_LOCATION=
# If you use a S3-compatible storage such as minio, set the following variable
# the full URL to the storage server. Example:
# AWS_S3_ENDPOINT_URL=https://minio.mydomain.com
# AWS_S3_ENDPOINT_URL=
Then, edit your nginx configuration. On docker setups, the file is located at ``/srv/funkwhale/nginx/funkwhale.template``,
and at ``/etc/nginx/sites-available/funkwhale.template`` on non-docker setups.
Replace the ``location /_protected/media`` block with the following::
location ~ /_protected/media/(.+) {
internal;
proxy_pass $1;
}
Then restart Funkwhale and nginx.
From now on, media files will be stored on the S3 bucket you configured. If you already
had media files before configuring the S3 bucket, you also have to move those on the bucket
by hand (which is outside the scope of this guide).
.. note::
At the moment, we do not support S3 when using Apache as a reverse proxy.
Securing your S3 bucket
***********************
It's important to ensure your the root of your bucket doesn't list its content,
which is the default on many S3 servers. Otherwise, anyone could find out the true
URLs of your audio files and bypass authentication.
To avoid that, you can set the following policy on your bucket::
{
"Version": "2012-10-17",
"Statement": [
{
"Action": [
"s3:GetObject"
],
"Effect": "Allow",
"Principal": {
"AWS": [
"*"
]
},
"Resource": [
"arn:aws:s3:::<yourbucketname>/*"
],
"Sid": "Public"
}
]
}
If you are using ``awscli``, you can store this policy in a ``/tmp/policy`` file, and
apply it using the following command::
aws s3api put-bucket-policy --bucket <yourbucketname> --policy file:///tmp/policy
......@@ -14,6 +14,7 @@ Setup Guides
../installation/index
configuration
importing-music
external-storages
Administration
--------------
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment