Merge branch 'release/0.17'

.env.dev

@@ -11,3 +11,4 @@ VUE_PORT=8080
 MUSIC_DIRECTORY_PATH=/music
 BROWSABLE_API_ENABLED=True
 FORWARDED_PROTO=http
+LDAP_ENABLED=False

.gitlab-ci.yml

@@ -148,6 +148,8 @@ test_api:
     - branches
   before_script:
     - cd api
+    - apt-get update
+    - grep "^[^#;]" requirements.apt | grep -Fv "python3-dev" | xargs apt-get install -y --no-install-recommends
     - pip install -r requirements/base.txt
     - pip install -r requirements/local.txt
     - pip install -r requirements/test.txt

.gitlab/merge_request_templates/Merge request.md

+Related issue: #XXX <!-- it's okay to have no issue for small changes -->
+
+This Merge Request includes:
+
+- [ ] Tests
+- [ ] A changelog fragment (cf https://docs.funkwhale.audio/contributing.html#changelog-management)

CHANGELOG

@@ -10,6 +10,200 @@ This changelog is viewable on the web at https://docs.funkwhale.audio/changelog.
 
 .. towncrier
 
+0.17 (2018-10-07)
+-----------------
+
+Per user libraries
+^^^^^^^^^^^^^^^^^^
+
+This release contains a big change in music management. This has a lot of impact
+on how Funkwhale behaves, and you should have a look at
+https://docs.funkwhale.audio/upgrading/0.17.html for information
+about what changed and how to migrate.
+
+
+Features:
+
+- Per user libraries (#463, also fixes #160 and #147)
+- Authentication using a LDAP directory (#194)
+
+
+Enhancements:
+
+- Add configuration option to set Musicbrainz hostname
+- Add sign up link in the sidebar (#408)
+- Added a library widget to display libraries associated with a track, album
+  and artist (#551)
+- Ensure from_activity field is not required in django's admin (#546)
+- Move setting link from profile page to the sidebar (#406)
+- Simplified and less error-prone nginx setup (#358)
+
+Bugfixes:
+
+- Do not restart current song when rordering queue, deleting tracks from queue
+  or adding tracks to queue (#464)
+- Fix broken icons in playlist editor (#515)
+- Fixed a few untranslated strings (#559)
+- Fixed splitted album when importing from federation (#346)
+- Fixed toggle mute in volume bar does not restore previous volume level (#514)
+- Fixed wrong env file URL and display bugs in deployment documentation (#520)
+- Fixed wrong title in PlayButton (#435)
+- Remove transparency on artist page button (#517)
+- Set sane width default for ui cards and center play button (#530)
+- Updated wrong icon and copy in play button dropdown (#436)
+
+
+Documentation:
+
+- Fixed wrong URLs for docker / nginx files in documentation (#537)
+
+
+Other:
+
+- Added a merge request template and more documentation about the changelog
+
+
+Using a LDAP directory to authenticate to your Funkwhale instance
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Funkwhale now support LDAP as an authentication source: you can configure
+your instance to delegate login to a LDAP directory, which is especially
+useful when you have an existing directory and don't want to manage users
+manually.
+
+You can use this authentication backend side by side with the classic one.
+
+Have a look at https://docs.funkwhale.audio/installation/ldap.html
+for detailed instructions on how to set this up.
+
+
+Simplified nginx setup [Docker: Manual action required]
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+We've received a lot of user feedback regarding our installation process,
+and it seems the proxy part is the one which is the most confusing and difficult.
+Unfortunately, this is also the one where errors and mistakes can completely break
+the application.
+
+To make things easier for everyone, we now offer a simplified deployment
+process for the reverse proxy part. This will make upgrade of the proxy configuration
+significantly easier on docker deployments.
+
+On non-docker instances, you have nothing to do.
+
+If you have a dockerized instance, here is the upgrade path.
+
+First, tweak your .env file::
+
+    # remove the FUNKWHALE_URL variable
+    # and add the next variables
+    FUNKWHALE_HOSTNAME=yourdomain.funkwhale
+    FUNKWHALE_PROTOCOL=https
+
+    # add the following variable, matching the path your app is deployed
+    # leaving the default should work fine if you deployed using the same
+    # paths as the documentation
+    FUNKWHALE_FRONTEND_PATH=/srv/funkwhale/front/dist
+
+Then, add the following block at the end of your docker-compose.yml file::
+
+    # existing services
+    api:
+        ...
+    celeryworker:
+        ...
+
+    # new service
+    nginx:
+      image: nginx
+      env_file:
+        - .env
+      environment:
+        # Override those variables in your .env file if needed
+        - "NGINX_MAX_BODY_SIZE=${NGINX_MAX_BODY_SIZE-30M}"
+      volumes:
+        - "./nginx/funkwhale.template:/etc/nginx/conf.d/funkwhale.template:ro"
+        - "./nginx/funkwhale_proxy.conf:/etc/nginx/funkwhale_proxy.conf:ro"
+        - "${MUSIC_DIRECTORY_SERVE_PATH-/srv/funkwhale/data/music}:${MUSIC_DIRECTORY_SERVE_PATH-/srv/funkwhale/data/music}:ro"
+        - "${MEDIA_ROOT}:${MEDIA_ROOT}:ro"
+        - "${STATIC_ROOT}:${STATIC_ROOT}:ro"
+        - "${FUNKWHALE_FRONTEND_PATH}:/frontend:ro"
+      ports:
+        # override those variables in your .env file if needed
+        - "${FUNKWHALE_API_IP}:${FUNKWHALE_API_PORT}:80"
+      command: >
+          sh -c "envsubst \"`env | awk -F = '{printf \" $$%s\", $$1}'`\"
+          < /etc/nginx/conf.d/funkwhale.template
+          > /etc/nginx/conf.d/default.conf
+          && cat /etc/nginx/conf.d/default.conf
+          && nginx -g 'daemon off;'"
+      links:
+        - api
+
+By doing that, you'll enable a dockerized nginx that will automatically be
+configured to serve your Funkwhale instance.
+
+Download the required configuration files for the nginx container:
+
+.. parsed-literal::
+
+    cd /srv/funkwhale
+    mkdir nginx
+    curl -L -o nginx/funkwhale.template "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/deploy/docker.nginx.template"
+    curl -L -o nginx/funkwhale_proxy.conf "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/deploy/funkwhale_proxy.conf"
+
+Update the funkwhale.conf configuration of your server's reverse-proxy::
+
+    # the file should match something like that, upgrade all variables
+    # between ${} to match the ones in your .env file,
+    # and your SSL configuration if you're not using let's encrypt
+    # The important thing is that you only have a single location block
+    # that proxies everything to your dockerized nginx.
+
+    sudo nano /etc/nginx/sites-enabled/funkwhale.conf
+    upstream fw {
+        # depending on your setup, you may want to update this
+        server ${FUNKWHALE_API_IP}:${FUNKWHALE_API_PORT};
+    }
+    map $http_upgrade $connection_upgrade {
+        default upgrade;
+        ''      close;
+    }
+
+    server {
+        listen 80;
+        listen [::]:80;
+        server_name ${FUNKWHALE_HOSTNAME};
+        location / { return 301 https://$host$request_uri; }
+    }
+    server {
+        listen      443 ssl;
+        listen [::]:443 ssl;
+        server_name ${FUNKWHALE_HOSTNAME};
+
+        # TLS
+        ssl_protocols TLSv1.2;
+        ssl_ciphers HIGH:!MEDIUM:!LOW:!aNULL:!NULL:!SHA;
+        ssl_prefer_server_ciphers on;
+        ssl_session_cache shared:SSL:10m;
+        ssl_certificate     /etc/letsencrypt/live/${FUNKWHALE_HOSTNAME}/fullchain.pem;
+        ssl_certificate_key /etc/letsencrypt/live/${FUNKWHALE_HOSTNAME}/privkey.pem;
+
+        # HSTS
+        add_header Strict-Transport-Security "max-age=31536000";
+
+        location / {
+            include /etc/nginx/funkwhale_proxy.conf;
+            proxy_pass   http://fw/;
+        }
+    }
+
+Check that your configuration is valid then reload:
+
+    sudo nginx -t
+    sudo systemctl reload nginx
+
+
 0.16.3 (2018-08-21)
 -------------------
 

CONTRIBUTING.rst

@@ -249,6 +249,7 @@ Then, in separate terminals, you can setup as many different instances as you
 need::
 
     export COMPOSE_PROJECT_NAME=node2
+    export VUE_PORT=1234  # this has to be unique for each instance
     docker-compose -f dev.yml run --rm api python manage.py migrate
     docker-compose -f dev.yml run --rm api python manage.py createsuperuser
     docker-compose -f dev.yml up nginx api front nginx api celeryworker
@@ -285,6 +286,62 @@ Typical workflow for a contribution
 8. Create your merge request
 9. Take a step back and enjoy, we're really grateful you did all of this and took the time to contribute!
 
+Changelog management
+--------------------
+
+To ensure we have extensive and well-structured changelog, any significant
+work such as closing an issue must include a changelog fragment. Small changes
+may include a changelog fragment as well but this is not mandatory. If you're not
+sure about what to do, do not panic, open your merge request normally and we'll
+figure everything during the review ;)
+
+Changelog fragments are text files that can contain one or multiple lines
+that describe the changes occuring in a bunch of commits. Those files reside
+in ``changes/changelog.d``.
+
+Content
+^^^^^^^
+
+A typical fragment looks like that:
+
+    Fixed broken audio player on Chrome 42 for ogg files (#567)
+
+If the work fixes one or more issues, the issue number should be included at the
+end of the fragment (``(#567)`` is the issue number in the previous example.
+
+If your work is not related to a specific issue, use the merge request
+identifier instead, like this:
+
+    Fixed a typo in landing page copy (!342)
+
+Naming
+^^^^^^
+
+Fragment files should respect the following naming pattern: ``changes/changelog.d/<name>.<category>``.
+Name can be anything describing your work, or simply the identifier of the issue number you are fixing.
+Category can be one of:
+
+- ``feature``: for new features
+- ``enhancement``: for enhancements on existing features
+- ``bugfix``: for bugfixes
+- ``doc``: for documentation
+- ``i18n``: for internationalization-related work
+- ``misc``: for anything else
+
+Shortcuts
+^^^^^^^^^
+
+Here is a shortcut you can use/adapt to easily create new fragments from command-line:
+
+.. code-block:: bash
+
+    issue="42"
+    content="Fixed an overflowing issue on small resolutions (#$issue)"
+    category="bugfix"
+    echo $content > changes/changelog.d/$issue.$category
+
+You can of course create fragments by hand in your text editor, or from Gitlab's
+interface as well.
 
 Internationalization
 --------------------

api/config/api_urls.py

@@ -14,12 +14,11 @@ router.register(r"settings", GlobalPreferencesViewSet, base_name="settings")
 router.register(r"activity", activity_views.ActivityViewSet, "activity")
 router.register(r"tags", views.TagViewSet, "tags")
 router.register(r"tracks", views.TrackViewSet, "tracks")
-router.register(r"trackfiles", views.TrackFileViewSet, "trackfiles")
+router.register(r"uploads", views.UploadViewSet, "uploads")
+router.register(r"libraries", views.LibraryViewSet, "libraries")
+router.register(r"listen", views.ListenViewSet, "listen")
 router.register(r"artists", views.ArtistViewSet, "artists")
 router.register(r"albums", views.AlbumViewSet, "albums")
-router.register(r"import-batches", views.ImportBatchViewSet, "import-batches")
-router.register(r"import-jobs", views.ImportJobViewSet, "import-jobs")
-router.register(r"submit", views.SubmitViewSet, "submit")
 router.register(r"playlists", playlists_views.PlaylistViewSet, "playlists")
 router.register(
     r"playlist-tracks", playlists_views.PlaylistTrackViewSet, "playlist-tracks"
@@ -66,10 +65,6 @@ v1_patterns += [
         r"^users/",
         include(("funkwhale_api.users.api_urls", "users"), namespace="users"),
     ),
-    url(
-        r"^requests/",
-        include(("funkwhale_api.requests.api_urls", "requests"), namespace="requests"),
-    ),
     url(r"^token/$", jwt_views.obtain_jwt_token, name="token"),
     url(r"^token/refresh/$", jwt_views.refresh_jwt_token, name="token_refresh"),
 ]

api/config/routing.py

@@ -8,9 +8,7 @@ application = ProtocolTypeRouter(
     {
         # Empty for now (http->django views is added by default)
         "websocket": TokenAuthMiddleware(
-            URLRouter(
-                [url("^api/v1/instance/activity$", consumers.InstanceActivityConsumer)]
-            )
+            URLRouter([url("^api/v1/activity$", consumers.InstanceActivityConsumer)])
         )
     }
 )

api/config/settings/common.py

@@ -125,8 +125,6 @@ LOCAL_APPS = (
     "funkwhale_api.radios",
     "funkwhale_api.history",
     "funkwhale_api.playlists",
-    "funkwhale_api.providers.audiofile",
-    "funkwhale_api.providers.youtube",
     "funkwhale_api.providers.acoustid",
     "funkwhale_api.subsonic",
 )
@@ -280,7 +278,7 @@ MEDIA_ROOT = env("MEDIA_ROOT", default=str(APPS_DIR("media")))
 
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#media-url
 MEDIA_URL = env("MEDIA_URL", default="/media/")
-
+FILE_UPLOAD_PERMISSIONS = 0o644
 # URL Configuration
 # ------------------------------------------------------------------------------
 ROOT_URLCONF = "config.urls"
@@ -310,6 +308,71 @@ AUTH_USER_MODEL = "users.User"
 LOGIN_REDIRECT_URL = "users:redirect"
 LOGIN_URL = "account_login"
 
+# LDAP AUTHENTICATION CONFIGURATION
+# ------------------------------------------------------------------------------
+AUTH_LDAP_ENABLED = env.bool("LDAP_ENABLED", default=False)
+if AUTH_LDAP_ENABLED:
+
+    # Import the LDAP modules here; this way, we don't need the dependency unless someone
+    # actually enables the LDAP support
+    import ldap
+    from django_auth_ldap.config import LDAPSearch, LDAPSearchUnion, GroupOfNamesType
+
+    # Add LDAP to the authentication backends
+    AUTHENTICATION_BACKENDS += ("django_auth_ldap.backend.LDAPBackend",)
+
+    # Basic configuration
+    AUTH_LDAP_SERVER_URI = env("LDAP_SERVER_URI")
+    AUTH_LDAP_BIND_DN = env("LDAP_BIND_DN", default="")
+    AUTH_LDAP_BIND_PASSWORD = env("LDAP_BIND_PASSWORD", default="")
+    AUTH_LDAP_SEARCH_FILTER = env("LDAP_SEARCH_FILTER", default="(uid={0})").format(
+        "%(user)s"
+    )
+    AUTH_LDAP_START_TLS = env.bool("LDAP_START_TLS", default=False)
+
+    DEFAULT_USER_ATTR_MAP = [
+        "first_name:givenName",
+        "last_name:sn",
+        "username:cn",
+        "email:mail",
+    ]
+    LDAP_USER_ATTR_MAP = env.list("LDAP_USER_ATTR_MAP", default=DEFAULT_USER_ATTR_MAP)
+    AUTH_LDAP_USER_ATTR_MAP = {}
+    for m in LDAP_USER_ATTR_MAP:
+        funkwhale_field, ldap_field = m.split(":")
+        AUTH_LDAP_USER_ATTR_MAP[funkwhale_field.strip()] = ldap_field.strip()
+
+    # Determine root DN supporting multiple root DNs
+    AUTH_LDAP_ROOT_DN = env("LDAP_ROOT_DN")
+    AUTH_LDAP_ROOT_DN_LIST = []
+    for ROOT_DN in AUTH_LDAP_ROOT_DN.split():
+        AUTH_LDAP_ROOT_DN_LIST.append(
+            LDAPSearch(ROOT_DN, ldap.SCOPE_SUBTREE, AUTH_LDAP_SEARCH_FILTER)
+        )
+    # Search for the user in all the root DNs
+    AUTH_LDAP_USER_SEARCH = LDAPSearchUnion(*AUTH_LDAP_ROOT_DN_LIST)
+
+    # Search for group types
+    LDAP_GROUP_DN = env("LDAP_GROUP_DN", default="")
+    if LDAP_GROUP_DN:
+        AUTH_LDAP_GROUP_DN = LDAP_GROUP_DN
+        # Get filter
+        AUTH_LDAP_GROUP_FILTER = env("LDAP_GROUP_FILER", default="")
+        # Search for the group in the specified DN
+        AUTH_LDAP_GROUP_SEARCH = LDAPSearch(
+            AUTH_LDAP_GROUP_DN, ldap.SCOPE_SUBTREE, AUTH_LDAP_GROUP_FILTER
+        )
+        AUTH_LDAP_GROUP_TYPE = GroupOfNamesType()
+
+        # Configure basic group support
+        LDAP_REQUIRE_GROUP = env("LDAP_REQUIRE_GROUP", default="")
+        if LDAP_REQUIRE_GROUP:
+            AUTH_LDAP_REQUIRE_GROUP = LDAP_REQUIRE_GROUP
+        LDAP_DENY_GROUP = env("LDAP_DENY_GROUP", default="")
+        if LDAP_DENY_GROUP:
+            AUTH_LDAP_DENY_GROUP = LDAP_DENY_GROUP
+
+
 # SLUGLIFIER
 AUTOSLUG_SLUGIFY_FUNCTION = "slugify.slugify"
 
@@ -381,7 +444,7 @@ REST_FRAMEWORK = {
     "DEFAULT_AUTHENTICATION_CLASSES": (
         "funkwhale_api.common.authentication.JSONWebTokenAuthenticationQS",
         "funkwhale_api.common.authentication.BearerTokenHeaderAuth",
-        "rest_framework_jwt.authentication.JSONWebTokenAuthentication",
+        "funkwhale_api.common.authentication.JSONWebTokenAuthentication",
         "rest_framework.authentication.SessionAuthentication",
         "rest_framework.authentication.BasicAuthentication",
     ),
@@ -422,6 +485,11 @@ PROTECT_FILES_PATH = env("PROTECT_FILES_PATH", default="/_protected")
 # musicbrainz results. (value is in seconds)
 MUSICBRAINZ_CACHE_DURATION = env.int("MUSICBRAINZ_CACHE_DURATION", default=300)
 
+# Use this setting to change the musicbrainz hostname, for instance to
+# use a mirror. The hostname can also contain a port number (so, e.g.,
+# "localhost:5000" is a valid name to set).
+MUSICBRAINZ_HOSTNAME = env("MUSICBRAINZ_HOSTNAME", default="musicbrainz.org")
+
 # Custom Admin URL, use {% url 'admin:index' %}
 ADMIN_URL = env("DJANGO_ADMIN_URL", default="^api/admin/")
 CSRF_USE_SESSIONS = True
@@ -445,8 +513,14 @@ ACCOUNT_USERNAME_BLACKLIST = [
     "me",
     "ghost",
     "_",
+    "-",
     "hello",
     "contact",
+    "inbox",
+    "outbox",
+    "shared-inbox",
+    "shared_inbox",
+    "actor",
 ] + env.list("ACCOUNT_USERNAME_BLACKLIST", default=[])
 
 EXTERNAL_REQUESTS_VERIFY_SSL = env.bool("EXTERNAL_REQUESTS_VERIFY_SSL", default=True)

api/config/settings/local.py

@@ -67,6 +67,7 @@ LOGGING = {
             "propagate": True,
             "level": "DEBUG",
         },
+        "django_auth_ldap": {"handlers": ["console"], "level": "DEBUG"},
         "": {"level": "DEBUG", "handlers": ["console"]},
     },
 }

api/config/urls.py

@@ -2,11 +2,13 @@
 from __future__ import unicode_literals
 
 from django.conf import settings
-from django.conf.urls import include, url
+from django.conf.urls import url
+from django.urls import include, path
 from django.conf.urls.static import static
-from django.contrib import admin
+from funkwhale_api.common import admin
 from django.views import defaults as default_views
 
+
 urlpatterns = [
     # Django Admin, use {% url 'admin:index' %}
     url(settings.ADMIN_URL, admin.site.urls),
@@ -36,4 +38,6 @@ if settings.DEBUG:
     if "debug_toolbar" in settings.INSTALLED_APPS:
         import debug_toolbar
 
-        urlpatterns += [url(r"^__debug__/", include(debug_toolbar.urls))]
+        urlpatterns = [
+            path("api/__debug__/", include(debug_toolbar.urls))
+        ] + urlpatterns

api/funkwhale_api/__init__.py

 # -*- coding: utf-8 -*-
-__version__ = "0.16.3"
+__version__ = "0.17"
 __version_info__ = tuple(
     [
         int(num) if num.isdigit() else num

api/funkwhale_api/common/admin.py

+from django.contrib.admin import register as initial_register, site, ModelAdmin  # noqa
+from django.db.models.fields.related import RelatedField
+
+
+def register(model):
+    """
+    To make the admin more performant, we ensure all the the relations
+    are listed under raw_id_fields
+    """
+
+    def decorator(modeladmin):
+        raw_id_fields = []
+        for field in model._meta.fields:
+            if isinstance(field, RelatedField):
+                raw_id_fields.append(field.name)
+        setattr(modeladmin, "raw_id_fields", raw_id_fields)
+        return initial_register(model)(modeladmin)
+
+    return decorator

api/funkwhale_api/common/authentication.py

@@ -56,3 +56,20 @@ class BearerTokenHeaderAuth(authentication.BaseJSONWebTokenAuthentication):
 
     def authenticate_header(self, request):
         return '{0} realm="{1}"'.format("Bearer", self.www_authenticate_realm)
+
+    def authenticate(self, request):
+        auth = super().authenticate(request)
+        if auth:
+            if not auth[0].actor:
+                auth[0].create_actor()
+        return auth
+
+
+class JSONWebTokenAuthentication(authentication.JSONWebTokenAuthentication):
+    def authenticate(self, request):
+        auth = super().authenticate(request)
+
+        if auth:
+            if not auth[0].actor:
+                auth[0].create_actor()
+        return auth

api/funkwhale_api/common/channels.py

+import json
+import logging
+
 from asgiref.sync import async_to_sync
 from channels.layers import get_channel_layer
+from django.core.serializers.json import DjangoJSONEncoder
 
+logger = logging.getLogger(__file__)
 channel_layer = get_channel_layer()
-group_send = async_to_sync(channel_layer.group_send)
 group_add = async_to_sync(channel_layer.group_add)
+
+
+def group_send(group, event):
+    # we serialize the payload ourselves and deserialize it to ensure it
+    # works with msgpack. This is dirty, but we'll find a better solution
+    # later
+    s = json.dumps(event, cls=DjangoJSONEncoder)
+    event = json.loads(s)
+    logger.debug(
+        "[channels] Dispatching %s to group %s: %s",
+        event["type"],
+        group,
+        {"type": event["data"]["type"]},
+    )
+    async_to_sync(channel_layer.group_send)(group, event)

api/funkwhale_api/common/consumers.py

@@ -16,3 +16,5 @@ class JsonAuthConsumer(JsonWebsocketConsumer):
         super().accept()
         for group in self.groups:
             channels.group_add(group, self.channel_name)
+        for group in self.scope["user"].get_channels_groups():
+            channels.group_add(group, self.channel_name)

api/funkwhale_api/common/permissions.py

@@ -9,7 +9,9 @@ from funkwhale_api.common import preferences
 class ConditionalAuthentication(BasePermission):
     def has_permission(self, request, view):
         if preferences.get("common__api_authentication_required"):
-            return request.user and request.user.is_authenticated
+            return (request.user and request.user.is_authenticated) or (
+                hasattr(request, "actor") and request.actor
+            )
         return True
 
 

api/funkwhale_api/common/scripts/__init__.py

 from . import create_actors
 from . import create_image_variations
 from . import django_permissions_to_user_permissions
+from . import migrate_to_user_libraries
 from . import test
 
 
@@ -8,5 +9,6 @@ __all__ = [
     "create_actors",
     "create_image_variations",
     "django_permissions_to_user_permissions",
+    "migrate_to_user_libraries",
     "test",
 ]

api/funkwhale_api/common/scripts/migrate_to_user_libraries.py

+"""
+Mirate instance files to a library #463. For each user that imported music on an
+instance, we will create a "default" library with related files and an instance-level
+visibility (unless instance has common__api_authentication_required set to False,
+in which case the libraries will be public).
+
+Files without any import job will be bounded to a "default" library on the first
+superuser account found. This should now happen though.
+
+This command will also generate federation ids for existing resources.
+
+"""
+
+from django.conf import settings
+from django.db.models import functions, CharField, F, Value
+
+from funkwhale_api.music import models
+from funkwhale_api.users.models import User
+from funkwhale_api.federation import models as federation_models
+from funkwhale_api.common import preferences
+
+
+def create_libraries(open_api, stdout):
+    local_actors = federation_models.Actor.objects.exclude(user=None).only("pk", "user")
+    privacy_level = "everyone" if open_api else "instance"
+    stdout.write(
+        "* Creating {} libraries with {} visibility".format(
+            len(local_actors), privacy_level
+        )
+    )
+    libraries_by_user = {}
+
+    for a in local_actors:
+        library, created = models.Library.objects.get_or_create(
+            name="default", actor=a, defaults={"privacy_level": privacy_level}
+        )
+        libraries_by_user[library.actor.user.pk] = library.pk
+        if created:
+            stdout.write(
+                "  * Created library {} for user {}".format(library.pk, a.user.pk)
+            )
+        else:
+            stdout.write(
+                "  * Found existing library {} for user {}".format(
+                    library.pk, a.user.pk
+                )
+            )
+
+    return libraries_by_user
+
+
+def update_uploads(libraries_by_user, stdout):
+    stdout.write("* Updating uploads with proper libraries...")
+    for user_id, library_id in libraries_by_user.items():
+        jobs = models.ImportJob.objects.filter(
+            upload__library=None, batch__submitted_by=user_id
+        )
+        candidates = models.Upload.objects.filter(
+            pk__in=jobs.values_list("upload", flat=True)
+        )
+        total = candidates.update(library=library_id, import_status="finished")
+        if total:
+            stdout.write(
+                "  * Assigned {} uploads to user {}'s library".format(total, user_id)
+            )
+        else:
+            stdout.write(
+                "  * No uploads to assign to user {}'s library".format(user_id)
+            )
+
+
+def update_orphan_uploads(open_api, stdout):
+    privacy_level = "everyone" if open_api else "instance"
+    first_superuser = (
+        User.objects.filter(is_superuser=True)
+        .exclude(actor=None)
+        .order_by("pk")
+        .first()
+    )
+    if not first_superuser:
+        stdout.write("* No superuser found, skipping update orphan uploads")
+        return
+    library, _ = models.Library.objects.get_or_create(
+        name="default",
+        actor=first_superuser.actor,
+        defaults={"privacy_level": privacy_level},
+    )
+    candidates = (
+        models.Upload.objects.filter(library=None, jobs__isnull=True)
+        .exclude(audio_file=None)
+        .exclude(audio_file="")
+    )
+
+    total = candidates.update(library=library, import_status="finished")
+    if total:
+        stdout.write(
+            "* Assigned {} orphaned uploads to superuser {}".format(
+                total, first_superuser.pk
+            )
+        )
+    else:
+        stdout.write("* No orphaned uploads found")
+
+
+def set_fid(queryset, path, stdout):
+    model = queryset.model._meta.label
+    qs = queryset.filter(fid=None)
+    base_url = "{}{}".format(settings.FUNKWHALE_URL, path)
+    stdout.write(
+        "* Assigning federation ids to {} entries (path: {})".format(model, base_url)
+    )
+    new_fid = functions.Concat(Value(base_url), F("uuid"), output_field=CharField())
+    total = qs.update(fid=new_fid)
+
+    stdout.write("  * {} entries updated".format(total))
+
+
+def update_shared_inbox_url(stdout):
+    stdout.write("* Update shared inbox url for local actors...")
+    candidates = federation_models.Actor.objects.local()
+    url = federation_models.get_shared_inbox_url()
+    candidates.update(shared_inbox_url=url)
+
+
+def generate_actor_urls(part, stdout):
+    field = "{}_url".format(part)
+    stdout.write("* Update {} for local actors...".format(field))
+
+    queryset = federation_models.Actor.objects.local().filter(**{field: None})
+    base_url = "{}/federation/actors/".format(settings.FUNKWHALE_URL)
+
+    new_field = functions.Concat(
+        Value(base_url),
+        F("preferred_username"),
+        Value("/{}".format(part)),
+        output_field=CharField(),
+    )
+
+    queryset.update(**{field: new_field})
+
+
+def main(command, **kwargs):
+    open_api = not preferences.get("common__api_authentication_required")
+    libraries_by_user = create_libraries(open_api, command.stdout)
+    update_uploads(libraries_by_user, command.stdout)
+    update_orphan_uploads(open_api, command.stdout)
+
+    set_fid_params = [
+        (
+            models.Upload.objects.exclude(library__actor__user=None),
+            "/federation/music/uploads/",
+        ),
+        (models.Artist.objects.all(), "/federation/music/artists/"),
+        (models.Album.objects.all(), "/federation/music/albums/"),
+        (models.Track.objects.all(), "/federation/music/tracks/"),
+    ]
+    for qs, path in set_fid_params:
+        set_fid(qs, path, command.stdout)
+
+    update_shared_inbox_url(command.stdout)
+
+    for part in ["followers", "following"]:
+        generate_actor_urls(part, command.stdout)

api/funkwhale_api/common/serializers.py

+import collections
+
 from rest_framework import serializers
 
+from django.core.exceptions import ObjectDoesNotExist
+from django.utils.encoding import smart_text
+from django.utils.translation import ugettext_lazy as _
+
+
+class RelatedField(serializers.RelatedField):
+    default_error_messages = {
+        "does_not_exist": _("Object with {related_field_name}={value} does not exist."),
+        "invalid": _("Invalid value."),
+    }
+
+    def __init__(self, related_field_name, serializer, **kwargs):
+        self.related_field_name = related_field_name
+        self.serializer = serializer
+        self.filters = kwargs.pop("filters", None)
+        kwargs["queryset"] = kwargs.pop(
+            "queryset", self.serializer.Meta.model.objects.all()
+        )
+        super().__init__(**kwargs)
+
+    def get_filters(self, data):
+        filters = {self.related_field_name: data}
+        if self.filters:
+            filters.update(self.filters(self.context))
+        return filters
+
+    def to_internal_value(self, data):
+        try:
+            queryset = self.get_queryset()
+            filters = self.get_filters(data)
+            return queryset.get(**filters)
+        except ObjectDoesNotExist:
+            self.fail(
+                "does_not_exist",
+                related_field_name=self.related_field_name,
+                value=smart_text(data),
+            )
+        except (TypeError, ValueError):
+            self.fail("invalid")
+
+    def to_representation(self, obj):
+        return self.serializer.to_representation(obj)
+
+    def get_choices(self, cutoff=None):
+        queryset = self.get_queryset()
+        if queryset is None:
+            # Ensure that field.choices returns something sensible
+            # even when accessed with a read-only field.
+            return {}
+
+        if cutoff is not None:
+            queryset = queryset[:cutoff]
+
+        return collections.OrderedDict(
+            [
+                (
+                    self.to_representation(item)[self.related_field_name],
+                    self.display_value(item),
+                )
+                for item in queryset
+            ]
+        )
+
 
 class Action(object):
     def __init__(self, name, allow_all=False, qs_filter=None):
@@ -21,6 +86,7 @@ class ActionSerializer(serializers.Serializer):
     objects = serializers.JSONField(required=True)
     filters = serializers.DictField(required=False)
     actions = None
+    pk_field = "pk"
 
     def __init__(self, *args, **kwargs):
         self.actions_by_name = {a.name: a for a in self.actions}
@@ -51,7 +117,9 @@ class ActionSerializer(serializers.Serializer):
         if value == "all":
             return self.queryset.all().order_by("id")
         if type(value) in [list, tuple]:
-            return self.queryset.filter(pk__in=value).order_by("id")
+            return self.queryset.filter(
+                **{"{}__in".format(self.pk_field): value}
+            ).order_by("id")
 
         raise serializers.ValidationError(
             "{} is not a valid value for objects. You must provide either a "

api/funkwhale_api/common/utils.py

@@ -64,3 +64,46 @@ class ChunkedPath(object):
             new_filename = "".join(chunks[3:]) + ".{}".format(ext)
             parts = chunks[:3] + [new_filename]
         return os.path.join(self.root, *parts)
+
+
+def chunk_queryset(source_qs, chunk_size):
+    """
+    From https://github.com/peopledoc/django-chunkator/blob/master/chunkator/__init__.py
+    """
+    pk = None
+    # In django 1.9, _fields is always present and `None` if 'values()' is used
+    # In Django 1.8 and below, _fields will only be present if using `values()`
+    has_fields = hasattr(source_qs, "_fields") and source_qs._fields
+    if has_fields:
+        if "pk" not in source_qs._fields:
+            raise ValueError("The values() call must include the `pk` field")
+
+    field = source_qs.model._meta.pk
+    # set the correct field name:
+    # for ForeignKeys, we want to use `model_id` field, and not `model`,
+    # to bypass default ordering on related model
+    order_by_field = field.attname
+
+    source_qs = source_qs.order_by(order_by_field)
+    queryset = source_qs
+    while True:
+        if pk:
+            queryset = source_qs.filter(pk__gt=pk)
+        page = queryset[:chunk_size]
+        page = list(page)
+        nb_items = len(page)
+
+        if nb_items == 0:
+            return
+
+        last_item = page[-1]
+        # source_qs._fields exists *and* is not none when using "values()"
+        if has_fields:
+            pk = last_item["pk"]
+        else:
+            pk = last_item.pk
+
+        yield page
+
+        if nb_items < chunk_size:
+            return