Merge branch 'activity-pub-import' into 'develop'

Activity pub import

See merge request funkwhale/funkwhale!132

api/funkwhale_api/federation/actors.py

@@ -12,6 +12,9 @@ from rest_framework.exceptions import PermissionDenied
 from dynamic_preferences.registries import global_preferences_registry
 
 from funkwhale_api.common import session
+from funkwhale_api.common import utils as funkwhale_utils
+from funkwhale_api.music import models as music_models
+from funkwhale_api.music import tasks as music_tasks
 
 from . import activity
 from . import keys
@@ -243,7 +246,7 @@ class LibraryActor(SystemActor):
                 data=i, context={'library': remote_library})
             for i in items
         ]
-
+        now = timezone.now()
         valid_serializers = []
         for s in item_serializers:
             if s.is_valid():
@@ -252,8 +255,30 @@ class LibraryActor(SystemActor):
                 logger.debug(
                     'Skipping invalid item %s, %s', s.initial_data, s.errors)
 
+        lts = []
         for s in valid_serializers:
-            s.save()
+            lts.append(s.save())
+
+        if remote_library.autoimport:
+            batch = music_models.ImportBatch.objects.create(
+                source='federation',
+            )
+            for lt in lts:
+                if lt.creation_date < now:
+                    # track was already in the library, we do not trigger
+                    # an import
+                    continue
+                job = music_models.ImportJob.objects.create(
+                    batch=batch,
+                    library_track=lt,
+                    mbid=lt.mbid,
+                    source=lt.url,
+                )
+                funkwhale_utils.on_commit(
+                    music_tasks.import_job_run.delay,
+                    import_job_id=job.pk,
+                    use_acoustid=False,
+                )
 
 
 class TestActor(SystemActor):

api/funkwhale_api/federation/filters.py

@@ -24,6 +24,7 @@ class LibraryFilter(django_filters.FilterSet):
 
 class LibraryTrackFilter(django_filters.FilterSet):
     library = django_filters.CharFilter('library__uuid')
+    imported = django_filters.CharFilter(method='filter_imported')
     q = fields.SearchFilter(search_fields=[
         'artist_name',
         'title',
@@ -31,6 +32,13 @@ class LibraryTrackFilter(django_filters.FilterSet):
         'library__actor__domain',
     ])
 
+    def filter_imported(self, queryset, field_name, value):
+        if value.lower() in ['true', '1', 'yes']:
+            queryset = queryset.filter(local_track_file__isnull=False)
+        elif value.lower() in ['false', '0', 'no']:
+            queryset = queryset.filter(local_track_file__isnull=True)
+        return queryset
+
     class Meta:
         model = models.LibraryTrack
         fields = {

api/funkwhale_api/federation/models.py

@@ -97,6 +97,11 @@ class Actor(models.Model):
         if self.is_system:
             return actors.SYSTEM_ACTORS[self.preferred_username]
 
+    def get_approved_followers(self):
+        follows = self.received_follows.filter(approved=True)
+        return self.followers.filter(
+            pk__in=follows.values_list('actor', flat=True))
+
 
 class Follow(models.Model):
     ap_type = 'Follow'

api/funkwhale_api/federation/serializers.py

@@ -493,7 +493,7 @@ class ActorWebfingerSerializer(serializers.Serializer):
 
 class ActivitySerializer(serializers.Serializer):
     actor = serializers.URLField()
-    id = serializers.URLField()
+    id = serializers.URLField(required=False)
     type = serializers.ChoiceField(
         choices=[(c, c) for c in activity.ACTIVITY_TYPES])
     object = serializers.JSONField()
@@ -525,6 +525,14 @@ class ActivitySerializer(serializers.Serializer):
             )
         return value
 
+    def to_representation(self, conf):
+        d = {}
+        d.update(conf)
+
+        if self.context.get('include_ap_context', True):
+            d['@context'] = AP_CONTEXT
+        return d
+
 
 class ObjectSerializer(serializers.Serializer):
     id = serializers.URLField()

api/funkwhale_api/music/factories.py

@@ -81,6 +81,9 @@ class ImportBatchFactory(factory.django.DjangoModelFactory):
             submitted_by=None,
             source='federation',
         )
+        finished = factory.Trait(
+            status='finished',
+        )
 
 
 @registry.register
@@ -98,6 +101,10 @@ class ImportJobFactory(factory.django.DjangoModelFactory):
             library_track=factory.SubFactory(LibraryTrackFactory),
             batch=factory.SubFactory(ImportBatchFactory, federation=True),
         )
+        finished = factory.Trait(
+            status='finished',
+            track_file=factory.SubFactory(TrackFileFactory),
+        )
 
 
 @registry.register(name='music.FileImportJob')

api/funkwhale_api/music/models.py

@@ -505,8 +505,17 @@ class ImportBatch(models.Model):
         return str(self.pk)
 
     def update_status(self):
+        old_status = self.status
         self.status = utils.compute_status(self.jobs.all())
         self.save(update_fields=['status'])
+        if self.status != old_status and self.status == 'finished':
+            from . import tasks
+            tasks.import_batch_notify_followers.delay(import_batch_id=self.pk)
+
+    def get_federation_url(self):
+        return federation_utils.full_url(
+            '/federation/music/import/batch/{}'.format(self.uuid)
+        )
 
 
 class ImportJob(models.Model):

api/funkwhale_api/music/tasks.py

@@ -2,6 +2,10 @@ from django.core.files.base import ContentFile
 
 from dynamic_preferences.registries import global_preferences_registry
 
+from funkwhale_api.federation import activity
+from funkwhale_api.federation import actors
+from funkwhale_api.federation import models as federation_models
+from funkwhale_api.federation import serializers as federation_serializers
 from funkwhale_api.taskapp import celery
 from funkwhale_api.providers.acoustid import get_acoustid_client
 from funkwhale_api.providers.audiofile.tasks import import_track_data_from_path
@@ -128,6 +132,7 @@ def _do_import(import_job, replace, use_acoustid=True):
         # it's imported on the track, we don't need it anymore
         import_job.audio_file.delete()
     import_job.save()
+
     return track.pk
 
 
@@ -162,3 +167,44 @@ def fetch_content(lyrics):
     cleaned_content = lyrics_utils.clean_content(content)
     lyrics.content = cleaned_content
     lyrics.save(update_fields=['content'])
+
+
+@celery.app.task(name='music.import_batch_notify_followers')
+@celery.require_instance(
+    models.ImportBatch.objects.filter(status='finished'), 'import_batch')
+def import_batch_notify_followers(import_batch):
+    if not settings.FEDERATION_ENABLED:
+        return
+
+    if import_batch.source == 'federation':
+        return
+
+    library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
+    followers = library_actor.get_approved_followers()
+    jobs = import_batch.jobs.filter(
+        status='finished',
+        library_track__isnull=True,
+        track_file__isnull=False,
+    ).select_related(
+        'track_file__track__artist',
+        'track_file__track__album__artist',
+    )
+    track_files = [job.track_file for job in jobs]
+    collection = federation_serializers.CollectionSerializer({
+        'actor': library_actor,
+        'id': import_batch.get_federation_url(),
+        'items': track_files,
+        'item_serializer': federation_serializers.AudioSerializer
+    }).data
+    for f in followers:
+        create = federation_serializers.ActivitySerializer(
+            {
+                'type': 'Create',
+                'id': collection['id'],
+                'object': collection,
+                'actor': library_actor.url,
+                'to': [f.url],
+            }
+        ).data
+
+        activity.deliver(create, on_behalf_of=library_actor, to=[f.url])

api/tests/conftest.py

@@ -69,6 +69,11 @@ def tmpdir():
     shutil.rmtree(d)
 
 
+@pytest.fixture
+def tmpfile():
+    yield tempfile.NamedTemporaryFile()
+
+
 @pytest.fixture
 def logged_in_client(db, factories, client):
     user = factories['users.User']()

api/tests/federation/test_actors.py

@@ -12,6 +12,8 @@ from funkwhale_api.federation import actors
 from funkwhale_api.federation import models
 from funkwhale_api.federation import serializers
 from funkwhale_api.federation import utils
+from funkwhale_api.music import models as music_models
+from funkwhale_api.music import tasks as music_tasks
 
 
 def test_actor_fetching(r_mock):
@@ -465,3 +467,62 @@ def test_library_actor_handle_create_audio(mocker, factories):
         assert lt.artist_name == a['metadata']['artist']['name']
         assert lt.album_title == a['metadata']['release']['title']
         assert lt.published_date == arrow.get(a['published'])
+
+
+def test_library_actor_handle_create_audio_autoimport(mocker, factories):
+    mocked_import = mocker.patch(
+        'funkwhale_api.common.utils.on_commit')
+    library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
+    remote_library = factories['federation.Library'](
+        federation_enabled=True,
+        autoimport=True,
+    )
+
+    data = {
+        'actor': remote_library.actor.url,
+        'type': 'Create',
+        'id': 'http://test.federation/audio/create',
+        'object': {
+            'id': 'https://batch.import',
+            'type': 'Collection',
+            'totalItems': 2,
+            'items': factories['federation.Audio'].create_batch(size=2)
+        },
+    }
+
+    library_actor.system_conf.post_inbox(data, actor=remote_library.actor)
+
+    lts = list(remote_library.tracks.order_by('id'))
+
+    assert len(lts) == 2
+
+    for i, a in enumerate(data['object']['items']):
+        lt = lts[i]
+        assert lt.pk is not None
+        assert lt.url == a['id']
+        assert lt.library == remote_library
+        assert lt.audio_url == a['url']['href']
+        assert lt.audio_mimetype == a['url']['mediaType']
+        assert lt.metadata == a['metadata']
+        assert lt.title == a['metadata']['recording']['title']
+        assert lt.artist_name == a['metadata']['artist']['name']
+        assert lt.album_title == a['metadata']['release']['title']
+        assert lt.published_date == arrow.get(a['published'])
+
+    batch = music_models.ImportBatch.objects.latest('id')
+
+    assert batch.jobs.count() == len(lts)
+    assert batch.source == 'federation'
+    assert batch.submitted_by is None
+
+    for i, job in enumerate(batch.jobs.order_by('id')):
+        lt = lts[i]
+        assert job.library_track == lt
+        assert job.mbid == lt.mbid
+        assert job.source == lt.url
+
+        mocked_import.assert_any_call(
+            music_tasks.import_job_run.delay,
+            import_job_id=job.pk,
+            use_acoustid=False,
+        )

api/tests/music/test_import.py

@@ -3,6 +3,8 @@ import pytest
 
 from django.urls import reverse
 
+from funkwhale_api.federation import actors
+from funkwhale_api.federation import serializers as federation_serializers
 from funkwhale_api.music import tasks
 
 
@@ -144,3 +146,88 @@ def test_import_job_from_federation_musicbrainz_artist(factories, mocker):
 
     artist_from_api.assert_called_once_with(
         mbid=lt.metadata['artist']['musicbrainz_id'])
+
+
+def test_import_job_run_triggers_notifies_followers(
+        factories, mocker, tmpfile):
+    mocker.patch(
+        'funkwhale_api.downloader.download',
+        return_value={'audio_file_path': tmpfile.name})
+    mocked_notify = mocker.patch(
+        'funkwhale_api.music.tasks.import_batch_notify_followers.delay')
+    batch = factories['music.ImportBatch']()
+    job = factories['music.ImportJob'](
+        finished=True, batch=batch)
+    track = factories['music.Track'](mbid=job.mbid)
+
+    batch.update_status()
+    batch.refresh_from_db()
+
+    assert batch.status == 'finished'
+
+    mocked_notify.assert_called_once_with(import_batch_id=batch.pk)
+
+
+def test_import_batch_notifies_followers_skip_on_disabled_federation(
+        settings, factories, mocker):
+    mocked_deliver = mocker.patch('funkwhale_api.federation.activity.deliver')
+    batch = factories['music.ImportBatch'](finished=True)
+    settings.FEDERATION_ENABLED = False
+    tasks.import_batch_notify_followers(import_batch_id=batch.pk)
+
+    mocked_deliver.assert_not_called()
+
+
+def test_import_batch_notifies_followers_skip_on_federation_import(
+        factories, mocker):
+    mocked_deliver = mocker.patch('funkwhale_api.federation.activity.deliver')
+    batch = factories['music.ImportBatch'](finished=True, federation=True)
+    tasks.import_batch_notify_followers(import_batch_id=batch.pk)
+
+    mocked_deliver.assert_not_called()
+
+
+def test_import_batch_notifies_followers(
+        factories, mocker):
+    library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
+
+    f1 = factories['federation.Follow'](approved=True, target=library_actor)
+    f2 = factories['federation.Follow'](approved=False, target=library_actor)
+    f3 = factories['federation.Follow']()
+
+    mocked_deliver = mocker.patch('funkwhale_api.federation.activity.deliver')
+    batch = factories['music.ImportBatch']()
+    job1 = factories['music.ImportJob'](
+        finished=True, batch=batch)
+    job2 = factories['music.ImportJob'](
+        finished=True, federation=True, batch=batch)
+    job3 = factories['music.ImportJob'](
+        status='pending', batch=batch)
+
+    batch.status = 'finished'
+    batch.save()
+    tasks.import_batch_notify_followers(import_batch_id=batch.pk)
+
+    # only f1 match the requirements to be notified
+    # and only job1 is a non federated track with finished import
+    expected = {
+        '@context': federation_serializers.AP_CONTEXT,
+        'actor': library_actor.url,
+        'type': 'Create',
+        'id': batch.get_federation_url(),
+        'to': [f1.actor.url],
+        'object': federation_serializers.CollectionSerializer(
+            {
+                'id': batch.get_federation_url(),
+                'items': [job1.track_file],
+                'actor': library_actor,
+                'item_serializer': federation_serializers.AudioSerializer
+            }
+        ).data
+    }
+
+    mocked_deliver.assert_called_once_with(
+        expected,
+        on_behalf_of=library_actor,
+        to=[f1.actor.url]
+    )

dev.yml

@@ -13,6 +13,7 @@ services:
       - "${WEBPACK_DEVSERVER_PORT_BINDING-8080:}${WEBPACK_DEVSERVER_PORT-8080}"
     volumes:
       - './front:/app'
+      - '/app/node_modules'
       - './po:/po'
     networks:
       - federation

front/.dockerignore

+node_modules

front/Dockerfile

@@ -4,7 +4,7 @@ EXPOSE 8080
 WORKDIR /app/
 ADD package.json .
 RUN yarn install
-VOLUME ["/app/node_modules"]
+
 COPY . .
 
 CMD ["npm", "run", "dev"]

front/src/components/federation/LibraryCard.vue

@@ -33,7 +33,7 @@
         :disabled="isLoading"
         :class="['ui', 'basic', {loading: isLoading}, 'green', 'button']">
         <i18next v-if="manuallyApprovesFollowers" path="Send a follow request"/>
-        <i18next v-else path="Follow">
+        <i18next v-else path="Follow"/>
       </div>
       <router-link
         v-else

front/src/components/federation/LibraryTrackTable.vue

 <template>
   <div>
     <div class="ui inline form">
-      <input type="text" v-model="search" placeholder="Search by title, artist, domain..." />
+      <div class="fields">
+        <div class="ui field">
+          <label>{{ $t('Search') }}</label>
+          <input type="text" v-model="search" placeholder="Search by title, artist, domain..." />
+        </div>
+        <div class="ui field">
+          <label>{{ $t('Import status') }}</label>
+          <select class="ui dropdown" v-model="importedFilter">
+            <option :value="null">{{ $t('Any') }}</option>
+            <option :value="true">{{ $t('Imported') }}</option>
+            <option :value="false">{{ $t('Not imported') }}</option>
+          </select>
+        </div>
+      </div>
     </div>
     <table v-if="result" class="ui compact very basic single line unstackable table">
       <thead>
@@ -65,22 +78,18 @@
 
           </th>
           <th v-if="result && result.results.length > 0">
-            <i18next path="Showing results {%0%}-{%1%} on {%2%}">
-              {{ ((page-1) * paginateBy) + 1 }}
-              {{ ((page-1) * paginateBy) + result.results.length }}
-              {{ result.count }}
-            </i18next>
+            {{ $t('Showing results {%start%}-{%end%} on {%total%}', {start: ((page-1) * paginateBy) + 1 , end: ((page-1) * paginateBy) + result.results.length, total: result.count})}}
           <th>
             <button
               @click="launchImport"
               :disabled="checked.length === 0 || isImporting"
               :class="['ui', 'green', {loading: isImporting}, 'button']">
-              <i18next path="Import {%count%} tracks" :count="checked.length"/>
+              {{ $t('Import {%count%} tracks', {'count': checked.length}) }}
             </button>
             <router-link
               v-if="importBatch"
               :to="{name: 'library.import.batches.detail', params: {id: importBatch.id }}">
-              <i18next path="Import #{%id%} launched" :id="importBatch.id"/>              
+              <i18next path="Import #{%id%} launched" :id="importBatch.id"/>
             </router-link>
           </th>
           <th></th>
@@ -116,7 +125,8 @@ export default {
       search: '',
       checked: {},
       isImporting: false,
-      importBatch: null
+      importBatch: null,
+      importedFilter: null
     }
   },
   created () {
@@ -129,6 +139,9 @@ export default {
         'page_size': this.paginateBy,
         'q': this.search
       }, this.filters)
+      if (this.importedFilter !== null) {
+        params.imported = this.importedFilter
+      }
       let self = this
       self.isLoading = true
       self.checked = []
@@ -185,6 +198,9 @@ export default {
     },
     page () {
       this.fetchData()
+    },
+    importedFilter () {
+      this.fetchData()
     }
   }
 }

front/src/views/federation/LibraryDetail.vue

@@ -18,7 +18,10 @@
           <table class="ui collapsing very basic table">
             <tbody>
               <tr>
-                <td>Follow status</td>
+                <td >
+                  Follow status
+                  <span :data-tooltip="$t('This indicate if the remote library granted you access')"><i class="question circle icon"></i></span>
+                </td>
                 <td>
                   <template v-if="object.follow.approved === null">
                     <i class="loading icon"></i> Pending approval
@@ -34,7 +37,10 @@
                 </td>
               </tr>
               <tr>
-                <td>Federation</td>
+                <td>
+                  Federation
+                  <span :data-tooltip="$t('Use this flag to enable/disable federation with this library')"><i class="question circle icon"></i></span>
+                </td>
                 <td>
                   <div class="ui toggle checkbox">
                     <input
@@ -46,9 +52,11 @@
                 <td>
                 </td>
               </tr>
-              <!-- Disabled until properly implemented on the backend
               <tr>
-                <td>Auto importing</td>
+                <td>
+                  Auto importing
+                  <span :data-tooltip="$t('When enabled, auto importing will automatically import new tracks published in this library')"><i class="question circle icon"></i></span>
+                </td>
                 <td>
                   <div class="ui toggle checkbox">
                     <input
@@ -59,6 +67,7 @@
                 </td>
                 <td></td>
               </tr>
+              <!-- Disabled until properly implemented on the backend
               <tr>
                 <td>File mirroring</td>
                 <td>