Commit b6e376ed authored by Eliot Berriot's avatar Eliot Berriot
Browse files

0.17 release documentation initial draft and migration script

parent 98591b2a
"""
Mirate instance files to a library #463. For each user that imported music on an
instance, we will create a "default" library with related files and an instance-level
visibility.
visibility (unless instance has common__api_authentication_required set to False,
in which case the libraries will be public).
Files without any import job will be bounded to a "default" library on the first
superuser account found. This should now happen though.
XXX TODO:
This command will also generate federation ids for existing resources.
- add followers url on actor
- shared inbox url on actor
- compute hash from files
"""
from django.conf import settings
from django.db.models import functions, CharField, F, Value
from funkwhale_api.music import models
from funkwhale_api.users.models import User
from funkwhale_api.federation import models as federation_models
from funkwhale_api.common import preferences
def main(command, **kwargs):
importer_ids = set(
models.ImportBatch.objects.values_list("submitted_by", flat=True)
def create_libraries(open_api, stdout):
local_actors = federation_models.Actor.objects.exclude(user=None).only("pk", "user")
privacy_level = "everyone" if open_api else "instance"
stdout.write(
"* Creating {} libraries with {} visibility".format(
len(local_actors), privacy_level
)
)
importers = User.objects.filter(pk__in=importer_ids).order_by("id").select_related()
command.stdout.write(
"* {} users imported music on this instance".format(len(importers))
libraries_by_user = {}
for a in local_actors:
library, created = models.Library.objects.get_or_create(
name="default", actor=a, defaults={"privacy_level": privacy_level}
)
libraries_by_user[library.actor.user.pk] = library.pk
if created:
stdout.write(
" * Created library {} for user {}".format(library.pk, a.user.pk)
)
else:
stdout.write(
" * Found existing library {} for user {}".format(
library.pk, a.user.pk
)
)
return libraries_by_user
def update_uploads(libraries_by_user, stdout):
stdout.write("* Updating uploads with proper libraries...")
for user_id, library_id in libraries_by_user.items():
jobs = models.ImportJob.objects.filter(
upload__library=None, batch__submitted_by=user_id
)
candidates = models.Upload.objects.filter(
pk__in=jobs.values_list("upload", flat=True)
)
total = candidates.update(library=library_id, import_status="finished")
if total:
stdout.write(
" * Assigned {} uploads to user {}'s library".format(total, user_id)
)
else:
stdout.write(
" * No uploads to assign to user {}'s library".format(user_id)
)
def update_orphan_uploads(open_api, stdout):
privacy_level = "everyone" if open_api else "instance"
first_superuser = User.objects.filter(is_superuser=True).order_by("pk").first()
library, _ = models.Library.objects.get_or_create(
name="default",
actor=first_superuser.actor,
defaults={"privacy_level": privacy_level},
)
files = models.Upload.objects.filter(
library__isnull=True, jobs__isnull=False
).distinct()
command.stdout.write(
"* Reassigning {} files to importers libraries...".format(files.count())
candidates = (
models.Upload.objects.filter(library=None, jobs__isnull=True)
.exclude(audio_file=None)
.exclude(audio_file="")
)
for user in importers:
command.stdout.write(
" * Setting up @{}'s 'default' library".format(user.username)
)
library = user.actor.libraries.get_or_create(actor=user.actor, name="default")[
0
]
user_files = files.filter(jobs__batch__submitted_by=user)
total = user_files.count()
command.stdout.write(
" * Reassigning {} files to the user library...".format(total)
total = candidates.update(library=library, import_status="finished")
if total:
stdout.write(
"* Assigned {} orphaned uploads to superuser {}".format(
total, first_superuser.pk
)
)
user_files.update(library=library)
else:
stdout.write("* No orphaned uploads found")
files = models.Upload.objects.filter(
library__isnull=True, jobs__isnull=True
).distinct()
command.stdout.write(
"* Handling {} files with no import jobs...".format(files.count())
def set_fid(queryset, path, stdout):
model = queryset.model._meta.label
qs = queryset.filter(fid=None)
base_url = "{}{}".format(settings.FUNKWHALE_URL, path)
stdout.write(
"* Assigning federation ids to {} entries (path: {})".format(model, base_url)
)
new_fid = functions.Concat(Value(base_url), F("uuid"), output_field=CharField())
total = qs.update(fid=new_fid)
stdout.write(" * {} entries updated".format(total))
def update_shared_inbox_url(stdout):
stdout.write("* Update shared inbox url for local actors...")
candidates = federation_models.Actor.objects.local().filter(shared_inbox_url=None)
url = federation_models.get_shared_inbox_url()
candidates.update(shared_inbox_url=url)
user = User.objects.order_by("id").filter(is_superuser=True).first()
command.stdout.write(" * Setting up @{}'s 'default' library".format(user.username))
library = user.actor.libraries.get_or_create(actor=user.actor, name="default")[0]
total = files.count()
command.stdout.write(
" * Reassigning {} files to the user library...".format(total)
def generate_actor_urls(part, stdout):
field = "{}_url".format(part)
stdout.write("* Update {} for local actors...".format(field))
queryset = federation_models.Actor.objects.local().filter(**{field: None})
base_url = "{}/federation/actors/".format(settings.FUNKWHALE_URL)
new_field = functions.Concat(
Value(base_url),
F("preferred_username"),
Value("/{}".format(part)),
output_field=CharField(),
)
files.update(library=library)
command.stdout.write(" * Done!")
queryset.update(**{field: new_field})
def main(command, **kwargs):
open_api = not preferences.get("common__api_authentication_required")
libraries_by_user = create_libraries(open_api, command.stdout)
update_uploads(libraries_by_user, command.stdout)
update_orphan_uploads(open_api, command.stdout)
set_fid_params = [
(
models.Upload.objects.exclude(library__actor__user=None),
"/federation/music/uploads/",
),
(models.Artist.objects.all(), "/federation/music/artists/"),
(models.Album.objects.all(), "/federation/music/albums/"),
(models.Track.objects.all(), "/federation/music/tracks/"),
]
for qs, path in set_fid_params:
set_fid(qs, path, command.stdout)
update_shared_inbox_url(command.stdout)
for part in ["followers", "following"]:
generate_actor_urls(part, command.stdout)
......@@ -9,6 +9,7 @@ from django.core.exceptions import ObjectDoesNotExist
from django.core.serializers.json import DjangoJSONEncoder
from django.db import models
from django.utils import timezone
from django.urls import reverse
from funkwhale_api.common import session
from funkwhale_api.common import utils as common_utils
......@@ -29,6 +30,10 @@ def empty_dict():
return {}
def get_shared_inbox_url():
return federation_utils.full_url(reverse("federation:shared-inbox"))
class FederationMixin(models.Model):
# federation id/url
fid = models.URLField(unique=True, max_length=500, db_index=True)
......
......@@ -4,6 +4,7 @@ import factory
from funkwhale_api.factories import ManyToManyFromList, registry
from funkwhale_api.federation import factories as federation_factories
from funkwhale_api.users import factories as users_factories
SAMPLES_PATH = os.path.join(
......@@ -100,3 +101,24 @@ class TagFactory(factory.django.DjangoModelFactory):
class Meta:
model = "taggit.Tag"
# XXX To remove
class ImportBatchFactory(factory.django.DjangoModelFactory):
submitted_by = factory.SubFactory(users_factories.UserFactory)
class Meta:
model = "music.ImportBatch"
@registry.register
class ImportJobFactory(factory.django.DjangoModelFactory):
batch = factory.SubFactory(ImportBatchFactory)
source = factory.Faker("url")
mbid = factory.Faker("uuid4")
replace_if_duplicate = False
class Meta:
model = "music.ImportJob"
......@@ -557,8 +557,8 @@ class UploadQuerySet(models.QuerySet):
libraries = Library.objects.viewable_by(actor)
if include:
return self.filter(library__in=libraries)
return self.exclude(library__in=libraries)
return self.filter(library__in=libraries, import_status="finished")
return self.exclude(library__in=libraries, import_status="finished")
def local(self, include=True):
return self.exclude(library__actor__user__isnull=include)
......@@ -899,7 +899,7 @@ class Library(federation_models.FederationMixin):
)
def save(self, **kwargs):
if not self.pk and not self.fid and self.actor.is_local:
if not self.pk and not self.fid and self.actor.get_user():
self.fid = self.get_federation_id()
self.followers_url = self.fid + "/followers"
......
......@@ -248,10 +248,9 @@ class Invitation(models.Model):
return super().save(**kwargs)
def create_actor(user):
def get_actor_data(user):
username = federation_utils.slugify_username(user.username)
private, public = keys.get_key_pair()
args = {
return {
"preferred_username": username,
"domain": settings.FEDERATION_HOSTNAME,
"type": "Person",
......@@ -260,9 +259,7 @@ def create_actor(user):
"fid": federation_utils.full_url(
reverse("federation:actors-detail", kwargs={"preferred_username": username})
),
"shared_inbox_url": federation_utils.full_url(
reverse("federation:shared-inbox")
),
"shared_inbox_url": federation_models.get_shared_inbox_url(),
"inbox_url": federation_utils.full_url(
reverse("federation:actors-inbox", kwargs={"preferred_username": username})
),
......@@ -280,6 +277,11 @@ def create_actor(user):
)
),
}
def create_actor(user):
args = get_actor_data(user)
private, public = keys.get_key_pair()
args["private_key"] = private.decode("utf-8")
args["public_key"] = public.decode("utf-8")
......
......@@ -2,6 +2,8 @@ import pytest
from funkwhale_api.common import scripts
from funkwhale_api.common.management.commands import script
from funkwhale_api.federation import models as federation_models
from funkwhale_api.music import models as music_models
@pytest.fixture
......@@ -44,29 +46,216 @@ def test_django_permissions_to_user_permissions(factories, command):
assert user2.permission_federation is True
@pytest.mark.skip("Refactoring in progress")
def test_migrate_to_user_libraries(factories, command):
user1 = factories["users.User"](is_superuser=False, with_actor=True)
user2 = factories["users.User"](is_superuser=True, with_actor=True)
factories["users.User"](is_superuser=True)
no_import_files = factories["music.Upload"].create_batch(size=5, library=None)
import_jobs = factories["music.ImportJob"].create_batch(
batch__submitted_by=user1, size=5, finished=True
@pytest.mark.parametrize(
"open_api,expected_visibility", [(True, "everyone"), (False, "instance")]
)
def test_migrate_to_user_libraries_create_libraries(
factories, open_api, expected_visibility, stdout
):
user1 = factories["users.User"](with_actor=True)
user2 = factories["users.User"](with_actor=True)
result = scripts.migrate_to_user_libraries.create_libraries(open_api, stdout)
user1_library = user1.actor.libraries.get(
name="default", privacy_level=expected_visibility
)
# we delete libraries that are created automatically
for j in import_jobs:
j.upload.library = None
j.upload.save()
scripts.migrate_to_user_libraries.main(command)
user2_library = user2.actor.libraries.get(
name="default", privacy_level=expected_visibility
)
assert result == {user1.pk: user1_library.pk, user2.pk: user2_library.pk}
def test_migrate_to_user_libraries_update_uploads(factories, stdout):
user1 = factories["users.User"](with_actor=True)
user2 = factories["users.User"](with_actor=True)
library1 = factories["music.Library"](actor=user1.actor)
library2 = factories["music.Library"](actor=user2.actor)
upload1 = factories["music.Upload"]()
upload2 = factories["music.Upload"]()
# we delete libraries
upload1.library = None
upload2.library = None
upload1.save()
upload2.save()
factories["music.ImportJob"](batch__submitted_by=user1, upload=upload1)
factories["music.ImportJob"](batch__submitted_by=user2, upload=upload2)
libraries_by_user = {user1.pk: library1.pk, user2.pk: library2.pk}
scripts.migrate_to_user_libraries.update_uploads(libraries_by_user, stdout)
upload1.refresh_from_db()
upload2.refresh_from_db()
assert upload1.library == library1
assert upload1.import_status == "finished"
assert upload2.library == library2
assert upload2.import_status == "finished"
@pytest.mark.parametrize(
"open_api,expected_visibility", [(True, "everyone"), (False, "instance")]
)
def test_migrate_to_user_libraries_without_jobs(
factories, open_api, expected_visibility, stdout
):
superuser = factories["users.User"](is_superuser=True, with_actor=True)
upload1 = factories["music.Upload"]()
upload2 = factories["music.Upload"]()
upload3 = factories["music.Upload"](audio_file=None)
# we delete libraries
upload1.library = None
upload2.library = None
upload3.library = None
upload1.save()
upload2.save()
upload3.save()
factories["music.ImportJob"](upload=upload2)
scripts.migrate_to_user_libraries.update_orphan_uploads(open_api, stdout)
upload1.refresh_from_db()
upload2.refresh_from_db()
upload3.refresh_from_db()
superuser_library = superuser.actor.libraries.get(
name="default", privacy_level=expected_visibility
)
assert upload1.library == superuser_library
assert upload1.import_status == "finished"
# left untouched because they don't match filters
assert upload2.library is None
assert upload3.library is None
# tracks with import jobs are bound to the importer's library
library = user1.actor.libraries.get(name="default")
assert list(library.uploads.order_by("id").values_list("id", flat=True)) == sorted(
[ij.upload.pk for ij in import_jobs]
@pytest.mark.parametrize(
"model,args,path",
[
("music.Upload", {"library__actor__local": True}, "/federation/music/uploads/"),
("music.Artist", {}, "/federation/music/artists/"),
("music.Album", {}, "/federation/music/albums/"),
("music.Track", {}, "/federation/music/tracks/"),
],
)
def test_migrate_to_user_libraries_generate_fids(
factories, args, model, path, settings, stdout
):
template = "{}{}{}"
objects = factories[model].create_batch(5, fid=None, **args)
klass = factories[model]._meta.model
# we leave a fid on the first one, and set the others to None
existing_fid = objects[0].fid
base_path = existing_fid.replace(str(objects[0].uuid), "")
klass.objects.filter(pk__in=[o.pk for o in objects[1:]]).update(fid=None)
scripts.migrate_to_user_libraries.set_fid(klass.objects.all(), path, stdout)
for i, o in enumerate(objects):
o.refresh_from_db()
if i == 0:
assert o.fid == existing_fid
else:
assert o.fid == template.format(settings.FUNKWHALE_URL, path, o.uuid)
# we also ensure the path we insert match the one that is generated
# by the app on objects creation, as a safe guard for typos
assert base_path == o.fid.replace(str(o.uuid), "")
def test_migrate_to_user_libraries_update_actors_shared_inbox_url(factories, stdout):
local = factories["federation.Actor"](local=True, shared_inbox_url=None)
remote = factories["federation.Actor"](local=False, shared_inbox_url=None)
expected = federation_models.get_shared_inbox_url()
scripts.migrate_to_user_libraries.update_shared_inbox_url(stdout)
local.refresh_from_db()
remote.refresh_from_db()
assert local.shared_inbox_url == expected
assert remote.shared_inbox_url is None
@pytest.mark.parametrize("part", ["following", "followers"])
def test_migrate_to_user_libraries_generate_actor_urls(
factories, part, settings, stdout
):
field = "{}_url".format(part)
ok = factories["users.User"]().create_actor()
local = factories["federation.Actor"](local=True, **{field: None})
remote = factories["federation.Actor"](local=False, **{field: None})
assert getattr(local, field) is None
expected = "{}/federation/actors/{}/{}".format(
settings.FUNKWHALE_URL, local.preferred_username, part
)
ok_url = getattr(ok, field)
scripts.migrate_to_user_libraries.generate_actor_urls(part, stdout)
# tracks without import jobs are bound to first superuser
library = user2.actor.libraries.get(name="default")
assert list(library.uploads.order_by("id").values_list("id", flat=True)) == sorted(
[upload.pk for upload in no_import_files]
ok.refresh_from_db()
local.refresh_from_db()
remote.refresh_from_db()
# unchanged
assert getattr(ok, field) == ok_url
assert getattr(remote, field) is None
assert getattr(local, field) == expected
assert expected.replace(local.preferred_username, "") == ok_url.replace(
ok.preferred_username, ""
)
def test_migrate_to_users_libraries_command(
preferences, mocker, db, command, queryset_equal_queries
):
preferences["common__api_authentication_required"] = False
open_api = not preferences["common__api_authentication_required"]
create_libraries = mocker.patch.object(
scripts.migrate_to_user_libraries,
"create_libraries",
return_value={"hello": "world"},
)
update_uploads = mocker.patch.object(
scripts.migrate_to_user_libraries, "update_uploads"
)
update_orphan_uploads = mocker.patch.object(
scripts.migrate_to_user_libraries, "update_orphan_uploads"
)
set_fid = mocker.patch.object(scripts.migrate_to_user_libraries, "set_fid")
update_shared_inbox_url = mocker.patch.object(
scripts.migrate_to_user_libraries, "update_shared_inbox_url"
)
generate_actor_urls = mocker.patch.object(
scripts.migrate_to_user_libraries, "generate_actor_urls"
)
scripts.migrate_to_user_libraries.main(command)
create_libraries.assert_called_once_with(open_api, command.stdout)
update_uploads.assert_called_once_with({"hello": "world"}, command.stdout)
update_orphan_uploads.assert_called_once_with(open_api, command.stdout)
set_fid_params = [
(
music_models.Upload.objects.exclude(library__actor__user=None),
"/federation/music/uploads/",
),
(music_models.Artist.objects.all(), "/federation/music/artists/"),
(music_models.Album.objects.all(), "/federation/music/albums/"),
(music_models.Track.objects.all(), "/federation/music/tracks/"),
]
for qs, path in set_fid_params:
set_fid.assert_any_call(qs, path, command.stdout)
update_shared_inbox_url.assert_called_once_with(command.stdout)
# generate_actor_urls(part, stdout):
for part in ["followers", "following"]:
generate_actor_urls.assert_any_call(part, command.stdout)
......@@ -377,3 +377,8 @@ def temp_signal(mocker):
signal.disconnect(stub)
return connect
@pytest.fixture()
def stdout():
yield io.StringIO()
......@@ -41,7 +41,7 @@ def test_upload_url_is_accessible_to_authenticated_users(
):
actor = logged_in_api_client.user.create_actor()
preferences["common__api_authentication_required"] = True
upload = factories["music.Upload"](library__actor=actor)
upload = factories["music.Upload"](library__actor=actor, import_status="finished")
assert upload.audio_file is not None
url = upload.track.listen_url
response = logged_in_api_client.get(url)
......
......@@ -208,11 +208,25 @@ def test_library(factories):
assert library.uuid is not None
@pytest.mark.parametrize(
"status,expected", [("pending", False), ("errored", False), ("finished", True)]
)
def test_playable_by_correct_status(status, expected, factories):
upload = factories["music.Upload"](
library__privacy_level="everyone", import_status=status
)
queryset = upload.library.uploads.playable_by(None)
match = upload in list(queryset)
assert match is expected