diff --git a/api/funkwhale_api/federation/actors.py b/api/funkwhale_api/federation/actors.py index 27a418c7dad9ec7f233905b659554df26eb51b6b..380bb23c01e38e0651e3a3fc7ee0244025adcf34 100644 --- a/api/funkwhale_api/federation/actors.py +++ b/api/funkwhale_api/federation/actors.py @@ -12,6 +12,9 @@ from rest_framework.exceptions import PermissionDenied from dynamic_preferences.registries import global_preferences_registry from funkwhale_api.common import session +from funkwhale_api.common import utils as funkwhale_utils +from funkwhale_api.music import models as music_models +from funkwhale_api.music import tasks as music_tasks from . import activity from . import keys @@ -243,7 +246,7 @@ class LibraryActor(SystemActor): data=i, context={'library': remote_library}) for i in items ] - + now = timezone.now() valid_serializers = [] for s in item_serializers: if s.is_valid(): @@ -252,8 +255,30 @@ class LibraryActor(SystemActor): logger.debug( 'Skipping invalid item %s, %s', s.initial_data, s.errors) + lts = [] for s in valid_serializers: - s.save() + lts.append(s.save()) + + if remote_library.autoimport: + batch = music_models.ImportBatch.objects.create( + source='federation', + ) + for lt in lts: + if lt.creation_date < now: + # track was already in the library, we do not trigger + # an import + continue + job = music_models.ImportJob.objects.create( + batch=batch, + library_track=lt, + mbid=lt.mbid, + source=lt.url, + ) + funkwhale_utils.on_commit( + music_tasks.import_job_run.delay, + import_job_id=job.pk, + use_acoustid=False, + ) class TestActor(SystemActor): diff --git a/api/funkwhale_api/federation/filters.py b/api/funkwhale_api/federation/filters.py index c911f1a891966adf0ba942670ad1a74fbaca4da7..7a388ff1298fbab740342c5de1885157ea433a94 100644 --- a/api/funkwhale_api/federation/filters.py +++ b/api/funkwhale_api/federation/filters.py @@ -24,6 +24,7 @@ class LibraryFilter(django_filters.FilterSet): class LibraryTrackFilter(django_filters.FilterSet): library = django_filters.CharFilter('library__uuid') + imported = django_filters.CharFilter(method='filter_imported') q = fields.SearchFilter(search_fields=[ 'artist_name', 'title', @@ -31,6 +32,13 @@ class LibraryTrackFilter(django_filters.FilterSet): 'library__actor__domain', ]) + def filter_imported(self, queryset, field_name, value): + if value.lower() in ['true', '1', 'yes']: + queryset = queryset.filter(local_track_file__isnull=False) + elif value.lower() in ['false', '0', 'no']: + queryset = queryset.filter(local_track_file__isnull=True) + return queryset + class Meta: model = models.LibraryTrack fields = { diff --git a/api/funkwhale_api/federation/models.py b/api/funkwhale_api/federation/models.py index 066c5847b0f7c115788162ca3039169e499e5198..d91a00c8b50f5c103fc818f9dea47f6a55cbf9cf 100644 --- a/api/funkwhale_api/federation/models.py +++ b/api/funkwhale_api/federation/models.py @@ -97,6 +97,11 @@ class Actor(models.Model): if self.is_system: return actors.SYSTEM_ACTORS[self.preferred_username] + def get_approved_followers(self): + follows = self.received_follows.filter(approved=True) + return self.followers.filter( + pk__in=follows.values_list('actor', flat=True)) + class Follow(models.Model): ap_type = 'Follow' diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py index 4964106d8f3d5be3f1029b0811aec2beffcabc7f..b56dd3f44b6bd7ca6e07b9e2740fada8f5d2e65b 100644 --- a/api/funkwhale_api/federation/serializers.py +++ b/api/funkwhale_api/federation/serializers.py @@ -493,7 +493,7 @@ class ActorWebfingerSerializer(serializers.Serializer): class ActivitySerializer(serializers.Serializer): actor = serializers.URLField() - id = serializers.URLField() + id = serializers.URLField(required=False) type = serializers.ChoiceField( choices=[(c, c) for c in activity.ACTIVITY_TYPES]) object = serializers.JSONField() @@ -525,6 +525,14 @@ class ActivitySerializer(serializers.Serializer): ) return value + def to_representation(self, conf): + d = {} + d.update(conf) + + if self.context.get('include_ap_context', True): + d['@context'] = AP_CONTEXT + return d + class ObjectSerializer(serializers.Serializer): id = serializers.URLField() diff --git a/api/funkwhale_api/music/factories.py b/api/funkwhale_api/music/factories.py index 2bf1960caf73f0d6f84ad2323e3580b3e8a0cfa1..ea7ff64dfa0cb154c871089e75c70cf9c19a6e1a 100644 --- a/api/funkwhale_api/music/factories.py +++ b/api/funkwhale_api/music/factories.py @@ -81,6 +81,9 @@ class ImportBatchFactory(factory.django.DjangoModelFactory): submitted_by=None, source='federation', ) + finished = factory.Trait( + status='finished', + ) @registry.register @@ -98,6 +101,10 @@ class ImportJobFactory(factory.django.DjangoModelFactory): library_track=factory.SubFactory(LibraryTrackFactory), batch=factory.SubFactory(ImportBatchFactory, federation=True), ) + finished = factory.Trait( + status='finished', + track_file=factory.SubFactory(TrackFileFactory), + ) @registry.register(name='music.FileImportJob') diff --git a/api/funkwhale_api/music/models.py b/api/funkwhale_api/music/models.py index beec551a544b9a5ea431dcfa130e92f735d7bcfd..4ec3ff4274efca8c6ad5f2e370ce90eda6b10141 100644 --- a/api/funkwhale_api/music/models.py +++ b/api/funkwhale_api/music/models.py @@ -505,8 +505,17 @@ class ImportBatch(models.Model): return str(self.pk) def update_status(self): + old_status = self.status self.status = utils.compute_status(self.jobs.all()) self.save(update_fields=['status']) + if self.status != old_status and self.status == 'finished': + from . import tasks + tasks.import_batch_notify_followers.delay(import_batch_id=self.pk) + + def get_federation_url(self): + return federation_utils.full_url( + '/federation/music/import/batch/{}'.format(self.uuid) + ) class ImportJob(models.Model): diff --git a/api/funkwhale_api/music/tasks.py b/api/funkwhale_api/music/tasks.py index 012b72cd28bf0721fa92a036f5f6f4b13305b51a..bc5ab94f0ae7a56470f42e3c705c40ff11d86053 100644 --- a/api/funkwhale_api/music/tasks.py +++ b/api/funkwhale_api/music/tasks.py @@ -2,6 +2,10 @@ from django.core.files.base import ContentFile from dynamic_preferences.registries import global_preferences_registry +from funkwhale_api.federation import activity +from funkwhale_api.federation import actors +from funkwhale_api.federation import models as federation_models +from funkwhale_api.federation import serializers as federation_serializers from funkwhale_api.taskapp import celery from funkwhale_api.providers.acoustid import get_acoustid_client from funkwhale_api.providers.audiofile.tasks import import_track_data_from_path @@ -128,6 +132,7 @@ def _do_import(import_job, replace, use_acoustid=True): # it's imported on the track, we don't need it anymore import_job.audio_file.delete() import_job.save() + return track.pk @@ -162,3 +167,44 @@ def fetch_content(lyrics): cleaned_content = lyrics_utils.clean_content(content) lyrics.content = cleaned_content lyrics.save(update_fields=['content']) + + +@celery.app.task(name='music.import_batch_notify_followers') +@celery.require_instance( + models.ImportBatch.objects.filter(status='finished'), 'import_batch') +def import_batch_notify_followers(import_batch): + if not settings.FEDERATION_ENABLED: + return + + if import_batch.source == 'federation': + return + + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + followers = library_actor.get_approved_followers() + jobs = import_batch.jobs.filter( + status='finished', + library_track__isnull=True, + track_file__isnull=False, + ).select_related( + 'track_file__track__artist', + 'track_file__track__album__artist', + ) + track_files = [job.track_file for job in jobs] + collection = federation_serializers.CollectionSerializer({ + 'actor': library_actor, + 'id': import_batch.get_federation_url(), + 'items': track_files, + 'item_serializer': federation_serializers.AudioSerializer + }).data + for f in followers: + create = federation_serializers.ActivitySerializer( + { + 'type': 'Create', + 'id': collection['id'], + 'object': collection, + 'actor': library_actor.url, + 'to': [f.url], + } + ).data + + activity.deliver(create, on_behalf_of=library_actor, to=[f.url]) diff --git a/api/tests/conftest.py b/api/tests/conftest.py index 4f1ee896227d8b10ca4ff822ee53d36f2b81787f..64dc394e7eef40663bba19b26bd6896f5a5088b1 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -69,6 +69,11 @@ def tmpdir(): shutil.rmtree(d) +@pytest.fixture +def tmpfile(): + yield tempfile.NamedTemporaryFile() + + @pytest.fixture def logged_in_client(db, factories, client): user = factories['users.User']() diff --git a/api/tests/federation/test_actors.py b/api/tests/federation/test_actors.py index fe70cc6e5cc5d8f97386e61c39f52147463972b6..7281147a1b143a9d48adce68bf124b8a57009933 100644 --- a/api/tests/federation/test_actors.py +++ b/api/tests/federation/test_actors.py @@ -12,6 +12,8 @@ from funkwhale_api.federation import actors from funkwhale_api.federation import models from funkwhale_api.federation import serializers from funkwhale_api.federation import utils +from funkwhale_api.music import models as music_models +from funkwhale_api.music import tasks as music_tasks def test_actor_fetching(r_mock): @@ -465,3 +467,62 @@ def test_library_actor_handle_create_audio(mocker, factories): assert lt.artist_name == a['metadata']['artist']['name'] assert lt.album_title == a['metadata']['release']['title'] assert lt.published_date == arrow.get(a['published']) + + +def test_library_actor_handle_create_audio_autoimport(mocker, factories): + mocked_import = mocker.patch( + 'funkwhale_api.common.utils.on_commit') + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + remote_library = factories['federation.Library']( + federation_enabled=True, + autoimport=True, + ) + + data = { + 'actor': remote_library.actor.url, + 'type': 'Create', + 'id': 'http://test.federation/audio/create', + 'object': { + 'id': 'https://batch.import', + 'type': 'Collection', + 'totalItems': 2, + 'items': factories['federation.Audio'].create_batch(size=2) + }, + } + + library_actor.system_conf.post_inbox(data, actor=remote_library.actor) + + lts = list(remote_library.tracks.order_by('id')) + + assert len(lts) == 2 + + for i, a in enumerate(data['object']['items']): + lt = lts[i] + assert lt.pk is not None + assert lt.url == a['id'] + assert lt.library == remote_library + assert lt.audio_url == a['url']['href'] + assert lt.audio_mimetype == a['url']['mediaType'] + assert lt.metadata == a['metadata'] + assert lt.title == a['metadata']['recording']['title'] + assert lt.artist_name == a['metadata']['artist']['name'] + assert lt.album_title == a['metadata']['release']['title'] + assert lt.published_date == arrow.get(a['published']) + + batch = music_models.ImportBatch.objects.latest('id') + + assert batch.jobs.count() == len(lts) + assert batch.source == 'federation' + assert batch.submitted_by is None + + for i, job in enumerate(batch.jobs.order_by('id')): + lt = lts[i] + assert job.library_track == lt + assert job.mbid == lt.mbid + assert job.source == lt.url + + mocked_import.assert_any_call( + music_tasks.import_job_run.delay, + import_job_id=job.pk, + use_acoustid=False, + ) diff --git a/api/tests/music/test_import.py b/api/tests/music/test_import.py index a15f027bac802992d16ee8dcd0277f8349e1f40d..2f22ed69ad8db3ffb6ec098d606b194fbdb8ab3e 100644 --- a/api/tests/music/test_import.py +++ b/api/tests/music/test_import.py @@ -3,6 +3,8 @@ import pytest from django.urls import reverse +from funkwhale_api.federation import actors +from funkwhale_api.federation import serializers as federation_serializers from funkwhale_api.music import tasks @@ -144,3 +146,88 @@ def test_import_job_from_federation_musicbrainz_artist(factories, mocker): artist_from_api.assert_called_once_with( mbid=lt.metadata['artist']['musicbrainz_id']) + + +def test_import_job_run_triggers_notifies_followers( + factories, mocker, tmpfile): + mocker.patch( + 'funkwhale_api.downloader.download', + return_value={'audio_file_path': tmpfile.name}) + mocked_notify = mocker.patch( + 'funkwhale_api.music.tasks.import_batch_notify_followers.delay') + batch = factories['music.ImportBatch']() + job = factories['music.ImportJob']( + finished=True, batch=batch) + track = factories['music.Track'](mbid=job.mbid) + + batch.update_status() + batch.refresh_from_db() + + assert batch.status == 'finished' + + mocked_notify.assert_called_once_with(import_batch_id=batch.pk) + + +def test_import_batch_notifies_followers_skip_on_disabled_federation( + settings, factories, mocker): + mocked_deliver = mocker.patch('funkwhale_api.federation.activity.deliver') + batch = factories['music.ImportBatch'](finished=True) + settings.FEDERATION_ENABLED = False + tasks.import_batch_notify_followers(import_batch_id=batch.pk) + + mocked_deliver.assert_not_called() + + +def test_import_batch_notifies_followers_skip_on_federation_import( + factories, mocker): + mocked_deliver = mocker.patch('funkwhale_api.federation.activity.deliver') + batch = factories['music.ImportBatch'](finished=True, federation=True) + tasks.import_batch_notify_followers(import_batch_id=batch.pk) + + mocked_deliver.assert_not_called() + + +def test_import_batch_notifies_followers( + factories, mocker): + library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance() + + f1 = factories['federation.Follow'](approved=True, target=library_actor) + f2 = factories['federation.Follow'](approved=False, target=library_actor) + f3 = factories['federation.Follow']() + + mocked_deliver = mocker.patch('funkwhale_api.federation.activity.deliver') + batch = factories['music.ImportBatch']() + job1 = factories['music.ImportJob']( + finished=True, batch=batch) + job2 = factories['music.ImportJob']( + finished=True, federation=True, batch=batch) + job3 = factories['music.ImportJob']( + status='pending', batch=batch) + + batch.status = 'finished' + batch.save() + tasks.import_batch_notify_followers(import_batch_id=batch.pk) + + # only f1 match the requirements to be notified + # and only job1 is a non federated track with finished import + expected = { + '@context': federation_serializers.AP_CONTEXT, + 'actor': library_actor.url, + 'type': 'Create', + 'id': batch.get_federation_url(), + 'to': [f1.actor.url], + 'object': federation_serializers.CollectionSerializer( + { + 'id': batch.get_federation_url(), + 'items': [job1.track_file], + 'actor': library_actor, + 'item_serializer': federation_serializers.AudioSerializer + } + ).data + } + + mocked_deliver.assert_called_once_with( + expected, + on_behalf_of=library_actor, + to=[f1.actor.url] + ) diff --git a/dev.yml b/dev.yml index 9488d4a6f31b62a9b31bbdb7e519192dcb3c1884..2df7b44e60100b4812db940f45fb8baf14da2544 100644 --- a/dev.yml +++ b/dev.yml @@ -13,6 +13,7 @@ services: - "${WEBPACK_DEVSERVER_PORT_BINDING-8080:}${WEBPACK_DEVSERVER_PORT-8080}" volumes: - './front:/app' + - '/app/node_modules' - './po:/po' networks: - federation diff --git a/front/.dockerignore b/front/.dockerignore new file mode 100644 index 0000000000000000000000000000000000000000..3c3629e647f5ddf82548912e337bea9826b434af --- /dev/null +++ b/front/.dockerignore @@ -0,0 +1 @@ +node_modules diff --git a/front/Dockerfile b/front/Dockerfile index 60b03c9b8e3c9ff54cf7dee5f7e4e96273a749b6..3d4c65e6418514a3db595bce3896655edaac82a7 100644 --- a/front/Dockerfile +++ b/front/Dockerfile @@ -4,7 +4,7 @@ EXPOSE 8080 WORKDIR /app/ ADD package.json . RUN yarn install -VOLUME ["/app/node_modules"] + COPY . . CMD ["npm", "run", "dev"] diff --git a/front/src/components/federation/LibraryCard.vue b/front/src/components/federation/LibraryCard.vue index f91b003ebae5140059f5657563a1437248513623..757561fb352be6300ae376780239f6e5fbbda612 100644 --- a/front/src/components/federation/LibraryCard.vue +++ b/front/src/components/federation/LibraryCard.vue @@ -33,7 +33,7 @@ :disabled="isLoading" :class="['ui', 'basic', {loading: isLoading}, 'green', 'button']"> <i18next v-if="manuallyApprovesFollowers" path="Send a follow request"/> - <i18next v-else path="Follow"> + <i18next v-else path="Follow"/> </div> <router-link v-else diff --git a/front/src/components/federation/LibraryTrackTable.vue b/front/src/components/federation/LibraryTrackTable.vue index 814f94f25cc01198caa6d4d73b94c3a412176b13..925ef3889668d5fbae39cf434fa8db390d0f4d06 100644 --- a/front/src/components/federation/LibraryTrackTable.vue +++ b/front/src/components/federation/LibraryTrackTable.vue @@ -1,7 +1,20 @@ <template> <div> <div class="ui inline form"> - <input type="text" v-model="search" placeholder="Search by title, artist, domain..." /> + <div class="fields"> + <div class="ui field"> + <label>{{ $t('Search') }}</label> + <input type="text" v-model="search" placeholder="Search by title, artist, domain..." /> + </div> + <div class="ui field"> + <label>{{ $t('Import status') }}</label> + <select class="ui dropdown" v-model="importedFilter"> + <option :value="null">{{ $t('Any') }}</option> + <option :value="true">{{ $t('Imported') }}</option> + <option :value="false">{{ $t('Not imported') }}</option> + </select> + </div> + </div> </div> <table v-if="result" class="ui compact very basic single line unstackable table"> <thead> @@ -65,22 +78,18 @@ </th> <th v-if="result && result.results.length > 0"> - <i18next path="Showing results {%0%}-{%1%} on {%2%}"> - {{ ((page-1) * paginateBy) + 1 }} - {{ ((page-1) * paginateBy) + result.results.length }} - {{ result.count }} - </i18next> + {{ $t('Showing results {%start%}-{%end%} on {%total%}', {start: ((page-1) * paginateBy) + 1 , end: ((page-1) * paginateBy) + result.results.length, total: result.count})}} <th> <button @click="launchImport" :disabled="checked.length === 0 || isImporting" :class="['ui', 'green', {loading: isImporting}, 'button']"> - <i18next path="Import {%count%} tracks" :count="checked.length"/> + {{ $t('Import {%count%} tracks', {'count': checked.length}) }} </button> <router-link v-if="importBatch" :to="{name: 'library.import.batches.detail', params: {id: importBatch.id }}"> - <i18next path="Import #{%id%} launched" :id="importBatch.id"/> + <i18next path="Import #{%id%} launched" :id="importBatch.id"/> </router-link> </th> <th></th> @@ -116,7 +125,8 @@ export default { search: '', checked: {}, isImporting: false, - importBatch: null + importBatch: null, + importedFilter: null } }, created () { @@ -129,6 +139,9 @@ export default { 'page_size': this.paginateBy, 'q': this.search }, this.filters) + if (this.importedFilter !== null) { + params.imported = this.importedFilter + } let self = this self.isLoading = true self.checked = [] @@ -185,6 +198,9 @@ export default { }, page () { this.fetchData() + }, + importedFilter () { + this.fetchData() } } } diff --git a/front/src/views/federation/LibraryDetail.vue b/front/src/views/federation/LibraryDetail.vue index c64ca2cf23353005655a1d874717d16d403c471d..20250e333d866941e7605667b1a3541b7095e716 100644 --- a/front/src/views/federation/LibraryDetail.vue +++ b/front/src/views/federation/LibraryDetail.vue @@ -18,7 +18,10 @@ <table class="ui collapsing very basic table"> <tbody> <tr> - <td>Follow status</td> + <td > + Follow status + <span :data-tooltip="$t('This indicate if the remote library granted you access')"><i class="question circle icon"></i></span> + </td> <td> <template v-if="object.follow.approved === null"> <i class="loading icon"></i> Pending approval @@ -34,7 +37,10 @@ </td> </tr> <tr> - <td>Federation</td> + <td> + Federation + <span :data-tooltip="$t('Use this flag to enable/disable federation with this library')"><i class="question circle icon"></i></span> + </td> <td> <div class="ui toggle checkbox"> <input @@ -46,9 +52,11 @@ <td> </td> </tr> - <!-- Disabled until properly implemented on the backend <tr> - <td>Auto importing</td> + <td> + Auto importing + <span :data-tooltip="$t('When enabled, auto importing will automatically import new tracks published in this library')"><i class="question circle icon"></i></span> + </td> <td> <div class="ui toggle checkbox"> <input @@ -59,6 +67,7 @@ </td> <td></td> </tr> + <!-- Disabled until properly implemented on the backend <tr> <td>File mirroring</td> <td>