Commit 6fd77a0a authored by Eliot Berriot's avatar Eliot Berriot
Browse files

Merge branch 'activity-pub-import' into 'develop'

Activity pub import

See merge request funkwhale/funkwhale!132
parents 9544a582 5fee42f8
......@@ -12,6 +12,9 @@ from rest_framework.exceptions import PermissionDenied
from dynamic_preferences.registries import global_preferences_registry
from funkwhale_api.common import session
from funkwhale_api.common import utils as funkwhale_utils
from funkwhale_api.music import models as music_models
from funkwhale_api.music import tasks as music_tasks
from . import activity
from . import keys
......@@ -243,7 +246,7 @@ class LibraryActor(SystemActor):
data=i, context={'library': remote_library})
for i in items
]
now = timezone.now()
valid_serializers = []
for s in item_serializers:
if s.is_valid():
......@@ -252,8 +255,30 @@ class LibraryActor(SystemActor):
logger.debug(
'Skipping invalid item %s, %s', s.initial_data, s.errors)
lts = []
for s in valid_serializers:
s.save()
lts.append(s.save())
if remote_library.autoimport:
batch = music_models.ImportBatch.objects.create(
source='federation',
)
for lt in lts:
if lt.creation_date < now:
# track was already in the library, we do not trigger
# an import
continue
job = music_models.ImportJob.objects.create(
batch=batch,
library_track=lt,
mbid=lt.mbid,
source=lt.url,
)
funkwhale_utils.on_commit(
music_tasks.import_job_run.delay,
import_job_id=job.pk,
use_acoustid=False,
)
class TestActor(SystemActor):
......
......@@ -24,6 +24,7 @@ class LibraryFilter(django_filters.FilterSet):
class LibraryTrackFilter(django_filters.FilterSet):
library = django_filters.CharFilter('library__uuid')
imported = django_filters.CharFilter(method='filter_imported')
q = fields.SearchFilter(search_fields=[
'artist_name',
'title',
......@@ -31,6 +32,13 @@ class LibraryTrackFilter(django_filters.FilterSet):
'library__actor__domain',
])
def filter_imported(self, queryset, field_name, value):
if value.lower() in ['true', '1', 'yes']:
queryset = queryset.filter(local_track_file__isnull=False)
elif value.lower() in ['false', '0', 'no']:
queryset = queryset.filter(local_track_file__isnull=True)
return queryset
class Meta:
model = models.LibraryTrack
fields = {
......
......@@ -97,6 +97,11 @@ class Actor(models.Model):
if self.is_system:
return actors.SYSTEM_ACTORS[self.preferred_username]
def get_approved_followers(self):
follows = self.received_follows.filter(approved=True)
return self.followers.filter(
pk__in=follows.values_list('actor', flat=True))
class Follow(models.Model):
ap_type = 'Follow'
......
......@@ -493,7 +493,7 @@ class ActorWebfingerSerializer(serializers.Serializer):
class ActivitySerializer(serializers.Serializer):
actor = serializers.URLField()
id = serializers.URLField()
id = serializers.URLField(required=False)
type = serializers.ChoiceField(
choices=[(c, c) for c in activity.ACTIVITY_TYPES])
object = serializers.JSONField()
......@@ -525,6 +525,14 @@ class ActivitySerializer(serializers.Serializer):
)
return value
def to_representation(self, conf):
d = {}
d.update(conf)
if self.context.get('include_ap_context', True):
d['@context'] = AP_CONTEXT
return d
class ObjectSerializer(serializers.Serializer):
id = serializers.URLField()
......
......@@ -81,6 +81,9 @@ class ImportBatchFactory(factory.django.DjangoModelFactory):
submitted_by=None,
source='federation',
)
finished = factory.Trait(
status='finished',
)
@registry.register
......@@ -98,6 +101,10 @@ class ImportJobFactory(factory.django.DjangoModelFactory):
library_track=factory.SubFactory(LibraryTrackFactory),
batch=factory.SubFactory(ImportBatchFactory, federation=True),
)
finished = factory.Trait(
status='finished',
track_file=factory.SubFactory(TrackFileFactory),
)
@registry.register(name='music.FileImportJob')
......
......@@ -505,8 +505,17 @@ class ImportBatch(models.Model):
return str(self.pk)
def update_status(self):
old_status = self.status
self.status = utils.compute_status(self.jobs.all())
self.save(update_fields=['status'])
if self.status != old_status and self.status == 'finished':
from . import tasks
tasks.import_batch_notify_followers.delay(import_batch_id=self.pk)
def get_federation_url(self):
return federation_utils.full_url(
'/federation/music/import/batch/{}'.format(self.uuid)
)
class ImportJob(models.Model):
......
......@@ -2,6 +2,10 @@ from django.core.files.base import ContentFile
from dynamic_preferences.registries import global_preferences_registry
from funkwhale_api.federation import activity
from funkwhale_api.federation import actors
from funkwhale_api.federation import models as federation_models
from funkwhale_api.federation import serializers as federation_serializers
from funkwhale_api.taskapp import celery
from funkwhale_api.providers.acoustid import get_acoustid_client
from funkwhale_api.providers.audiofile.tasks import import_track_data_from_path
......@@ -128,6 +132,7 @@ def _do_import(import_job, replace, use_acoustid=True):
# it's imported on the track, we don't need it anymore
import_job.audio_file.delete()
import_job.save()
return track.pk
......@@ -162,3 +167,44 @@ def fetch_content(lyrics):
cleaned_content = lyrics_utils.clean_content(content)
lyrics.content = cleaned_content
lyrics.save(update_fields=['content'])
@celery.app.task(name='music.import_batch_notify_followers')
@celery.require_instance(
models.ImportBatch.objects.filter(status='finished'), 'import_batch')
def import_batch_notify_followers(import_batch):
if not settings.FEDERATION_ENABLED:
return
if import_batch.source == 'federation':
return
library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
followers = library_actor.get_approved_followers()
jobs = import_batch.jobs.filter(
status='finished',
library_track__isnull=True,
track_file__isnull=False,
).select_related(
'track_file__track__artist',
'track_file__track__album__artist',
)
track_files = [job.track_file for job in jobs]
collection = federation_serializers.CollectionSerializer({
'actor': library_actor,
'id': import_batch.get_federation_url(),
'items': track_files,
'item_serializer': federation_serializers.AudioSerializer
}).data
for f in followers:
create = federation_serializers.ActivitySerializer(
{
'type': 'Create',
'id': collection['id'],
'object': collection,
'actor': library_actor.url,
'to': [f.url],
}
).data
activity.deliver(create, on_behalf_of=library_actor, to=[f.url])
......@@ -69,6 +69,11 @@ def tmpdir():
shutil.rmtree(d)
@pytest.fixture
def tmpfile():
yield tempfile.NamedTemporaryFile()
@pytest.fixture
def logged_in_client(db, factories, client):
user = factories['users.User']()
......
......@@ -12,6 +12,8 @@ from funkwhale_api.federation import actors
from funkwhale_api.federation import models
from funkwhale_api.federation import serializers
from funkwhale_api.federation import utils
from funkwhale_api.music import models as music_models
from funkwhale_api.music import tasks as music_tasks
def test_actor_fetching(r_mock):
......@@ -465,3 +467,62 @@ def test_library_actor_handle_create_audio(mocker, factories):
assert lt.artist_name == a['metadata']['artist']['name']
assert lt.album_title == a['metadata']['release']['title']
assert lt.published_date == arrow.get(a['published'])
def test_library_actor_handle_create_audio_autoimport(mocker, factories):
mocked_import = mocker.patch(
'funkwhale_api.common.utils.on_commit')
library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
remote_library = factories['federation.Library'](
federation_enabled=True,
autoimport=True,
)
data = {
'actor': remote_library.actor.url,
'type': 'Create',
'id': 'http://test.federation/audio/create',
'object': {
'id': 'https://batch.import',
'type': 'Collection',
'totalItems': 2,
'items': factories['federation.Audio'].create_batch(size=2)
},
}
library_actor.system_conf.post_inbox(data, actor=remote_library.actor)
lts = list(remote_library.tracks.order_by('id'))
assert len(lts) == 2
for i, a in enumerate(data['object']['items']):
lt = lts[i]
assert lt.pk is not None
assert lt.url == a['id']
assert lt.library == remote_library
assert lt.audio_url == a['url']['href']
assert lt.audio_mimetype == a['url']['mediaType']
assert lt.metadata == a['metadata']
assert lt.title == a['metadata']['recording']['title']
assert lt.artist_name == a['metadata']['artist']['name']
assert lt.album_title == a['metadata']['release']['title']
assert lt.published_date == arrow.get(a['published'])
batch = music_models.ImportBatch.objects.latest('id')
assert batch.jobs.count() == len(lts)
assert batch.source == 'federation'
assert batch.submitted_by is None
for i, job in enumerate(batch.jobs.order_by('id')):
lt = lts[i]
assert job.library_track == lt
assert job.mbid == lt.mbid
assert job.source == lt.url
mocked_import.assert_any_call(
music_tasks.import_job_run.delay,
import_job_id=job.pk,
use_acoustid=False,
)
......@@ -3,6 +3,8 @@ import pytest
from django.urls import reverse
from funkwhale_api.federation import actors
from funkwhale_api.federation import serializers as federation_serializers
from funkwhale_api.music import tasks
......@@ -144,3 +146,88 @@ def test_import_job_from_federation_musicbrainz_artist(factories, mocker):
artist_from_api.assert_called_once_with(
mbid=lt.metadata['artist']['musicbrainz_id'])
def test_import_job_run_triggers_notifies_followers(
factories, mocker, tmpfile):
mocker.patch(
'funkwhale_api.downloader.download',
return_value={'audio_file_path': tmpfile.name})
mocked_notify = mocker.patch(
'funkwhale_api.music.tasks.import_batch_notify_followers.delay')
batch = factories['music.ImportBatch']()
job = factories['music.ImportJob'](
finished=True, batch=batch)
track = factories['music.Track'](mbid=job.mbid)
batch.update_status()
batch.refresh_from_db()
assert batch.status == 'finished'
mocked_notify.assert_called_once_with(import_batch_id=batch.pk)
def test_import_batch_notifies_followers_skip_on_disabled_federation(
settings, factories, mocker):
mocked_deliver = mocker.patch('funkwhale_api.federation.activity.deliver')
batch = factories['music.ImportBatch'](finished=True)
settings.FEDERATION_ENABLED = False
tasks.import_batch_notify_followers(import_batch_id=batch.pk)
mocked_deliver.assert_not_called()
def test_import_batch_notifies_followers_skip_on_federation_import(
factories, mocker):
mocked_deliver = mocker.patch('funkwhale_api.federation.activity.deliver')
batch = factories['music.ImportBatch'](finished=True, federation=True)
tasks.import_batch_notify_followers(import_batch_id=batch.pk)
mocked_deliver.assert_not_called()
def test_import_batch_notifies_followers(
factories, mocker):
library_actor = actors.SYSTEM_ACTORS['library'].get_actor_instance()
f1 = factories['federation.Follow'](approved=True, target=library_actor)
f2 = factories['federation.Follow'](approved=False, target=library_actor)
f3 = factories['federation.Follow']()
mocked_deliver = mocker.patch('funkwhale_api.federation.activity.deliver')
batch = factories['music.ImportBatch']()
job1 = factories['music.ImportJob'](
finished=True, batch=batch)
job2 = factories['music.ImportJob'](
finished=True, federation=True, batch=batch)
job3 = factories['music.ImportJob'](
status='pending', batch=batch)
batch.status = 'finished'
batch.save()
tasks.import_batch_notify_followers(import_batch_id=batch.pk)
# only f1 match the requirements to be notified
# and only job1 is a non federated track with finished import
expected = {
'@context': federation_serializers.AP_CONTEXT,
'actor': library_actor.url,
'type': 'Create',
'id': batch.get_federation_url(),
'to': [f1.actor.url],
'object': federation_serializers.CollectionSerializer(
{
'id': batch.get_federation_url(),
'items': [job1.track_file],
'actor': library_actor,
'item_serializer': federation_serializers.AudioSerializer
}
).data
}
mocked_deliver.assert_called_once_with(
expected,
on_behalf_of=library_actor,
to=[f1.actor.url]
)
......@@ -13,6 +13,7 @@ services:
- "${WEBPACK_DEVSERVER_PORT_BINDING-8080:}${WEBPACK_DEVSERVER_PORT-8080}"
volumes:
- './front:/app'
- '/app/node_modules'
- './po:/po'
networks:
- federation
......
......@@ -4,7 +4,7 @@ EXPOSE 8080
WORKDIR /app/
ADD package.json .
RUN yarn install
VOLUME ["/app/node_modules"]
COPY . .
CMD ["npm", "run", "dev"]
......@@ -33,7 +33,7 @@
:disabled="isLoading"
:class="['ui', 'basic', {loading: isLoading}, 'green', 'button']">
<i18next v-if="manuallyApprovesFollowers" path="Send a follow request"/>
<i18next v-else path="Follow">
<i18next v-else path="Follow"/>
</div>
<router-link
v-else
......
<template>
<div>
<div class="ui inline form">
<input type="text" v-model="search" placeholder="Search by title, artist, domain..." />
<div class="fields">
<div class="ui field">
<label>{{ $t('Search') }}</label>
<input type="text" v-model="search" placeholder="Search by title, artist, domain..." />
</div>
<div class="ui field">
<label>{{ $t('Import status') }}</label>
<select class="ui dropdown" v-model="importedFilter">
<option :value="null">{{ $t('Any') }}</option>
<option :value="true">{{ $t('Imported') }}</option>
<option :value="false">{{ $t('Not imported') }}</option>
</select>
</div>
</div>
</div>
<table v-if="result" class="ui compact very basic single line unstackable table">
<thead>
......@@ -65,22 +78,18 @@
</th>
<th v-if="result && result.results.length > 0">
<i18next path="Showing results {%0%}-{%1%} on {%2%}">
{{ ((page-1) * paginateBy) + 1 }}
{{ ((page-1) * paginateBy) + result.results.length }}
{{ result.count }}
</i18next>
{{ $t('Showing results {%start%}-{%end%} on {%total%}', {start: ((page-1) * paginateBy) + 1 , end: ((page-1) * paginateBy) + result.results.length, total: result.count})}}
<th>
<button
@click="launchImport"
:disabled="checked.length === 0 || isImporting"
:class="['ui', 'green', {loading: isImporting}, 'button']">
<i18next path="Import {%count%} tracks" :count="checked.length"/>
{{ $t('Import {%count%} tracks', {'count': checked.length}) }}
</button>
<router-link
v-if="importBatch"
:to="{name: 'library.import.batches.detail', params: {id: importBatch.id }}">
<i18next path="Import #{%id%} launched" :id="importBatch.id"/>
<i18next path="Import #{%id%} launched" :id="importBatch.id"/>
</router-link>
</th>
<th></th>
......@@ -116,7 +125,8 @@ export default {
search: '',
checked: {},
isImporting: false,
importBatch: null
importBatch: null,
importedFilter: null
}
},
created () {
......@@ -129,6 +139,9 @@ export default {
'page_size': this.paginateBy,
'q': this.search
}, this.filters)
if (this.importedFilter !== null) {
params.imported = this.importedFilter
}
let self = this
self.isLoading = true
self.checked = []
......@@ -185,6 +198,9 @@ export default {
},
page () {
this.fetchData()
},
importedFilter () {
this.fetchData()
}
}
}
......
......@@ -18,7 +18,10 @@
<table class="ui collapsing very basic table">
<tbody>
<tr>
<td>Follow status</td>
<td >
Follow status
<span :data-tooltip="$t('This indicate if the remote library granted you access')"><i class="question circle icon"></i></span>
</td>
<td>
<template v-if="object.follow.approved === null">
<i class="loading icon"></i> Pending approval
......@@ -34,7 +37,10 @@
</td>
</tr>
<tr>
<td>Federation</td>
<td>
Federation
<span :data-tooltip="$t('Use this flag to enable/disable federation with this library')"><i class="question circle icon"></i></span>
</td>
<td>
<div class="ui toggle checkbox">
<input
......@@ -46,9 +52,11 @@
<td>
</td>
</tr>
<!-- Disabled until properly implemented on the backend
<tr>
<td>Auto importing</td>
<td>
Auto importing
<span :data-tooltip="$t('When enabled, auto importing will automatically import new tracks published in this library')"><i class="question circle icon"></i></span>
</td>
<td>
<div class="ui toggle checkbox">
<input
......@@ -59,6 +67,7 @@
</td>
<td></td>
</tr>
<!-- Disabled until properly implemented on the backend
<tr>
<td>File mirroring</td>
<td>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment