Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
Menu
Open sidebar
interfect
funkwhale
Commits
520fb9d0
Verified
Commit
520fb9d0
authored
Apr 11, 2018
by
Eliot Berriot
Browse files
Started work on library scanning
parent
472cc7e2
Changes
9
Hide whitespace changes
Inline
Side-by-side
api/funkwhale_api/federation/library.py
View file @
520fb9d0
...
...
@@ -144,3 +144,24 @@ def get_library_data(library_url):
}
return
serializer
.
validated_data
def
get_library_page
(
library
,
page_url
):
actor
=
actors
.
SYSTEM_ACTORS
[
'library'
].
get_actor_instance
()
auth
=
signing
.
get_auth
(
actor
.
private_key
,
actor
.
private_key_id
)
response
=
session
.
get_session
().
get
(
page_url
,
auth
=
auth
,
timeout
=
5
,
verify
=
settings
.
EXTERNAL_REQUESTS_VERIFY_SSL
,
headers
=
{
'Content-Type'
:
'application/activity+json'
}
)
serializer
=
serializers
.
CollectionPageSerializer
(
data
=
response
.
json
(),
context
=
{
'library'
:
library
,
'item_serializer'
:
serializers
.
AudioSerializer
})
serializer
.
is_valid
(
raise_exception
=
True
)
return
serializer
.
validated_data
api/funkwhale_api/federation/models.py
View file @
520fb9d0
...
...
@@ -2,6 +2,7 @@ import uuid
from
django.conf
import
settings
from
django.contrib.postgres.fields
import
JSONField
from
django.core.serializers.json
import
DjangoJSONEncoder
from
django.db
import
models
from
django.utils
import
timezone
...
...
@@ -160,4 +161,5 @@ class LibraryTrack(models.Model):
artist_name
=
models
.
CharField
(
max_length
=
500
)
album_title
=
models
.
CharField
(
max_length
=
500
)
title
=
models
.
CharField
(
max_length
=
500
)
metadata
=
JSONField
(
default
=
{},
max_length
=
10000
)
metadata
=
JSONField
(
default
=
{},
max_length
=
10000
,
encoder
=
DjangoJSONEncoder
)
api/funkwhale_api/federation/serializers.py
View file @
520fb9d0
...
...
@@ -494,6 +494,8 @@ class PaginatedCollectionSerializer(serializers.Serializer):
totalItems
=
serializers
.
IntegerField
(
min_value
=
0
)
actor
=
serializers
.
URLField
()
id
=
serializers
.
URLField
()
first
=
serializers
.
URLField
()
last
=
serializers
.
URLField
()
def
to_representation
(
self
,
conf
):
paginator
=
Paginator
(
...
...
@@ -524,10 +526,22 @@ class CollectionPageSerializer(serializers.Serializer):
items
=
serializers
.
ListField
()
actor
=
serializers
.
URLField
()
id
=
serializers
.
URLField
()
prev
=
serializers
.
URLField
(
required
=
False
)
first
=
serializers
.
URLField
()
last
=
serializers
.
URLField
()
next
=
serializers
.
URLField
(
required
=
False
)
prev
=
serializers
.
URLField
(
required
=
False
)
partOf
=
serializers
.
URLField
()
def
validate_items
(
self
,
v
):
item_serializer
=
self
.
context
.
get
(
'item_serializer'
)
if
not
item_serializer
:
return
v
raw_items
=
[
item_serializer
(
data
=
i
,
context
=
self
.
context
)
for
i
in
v
]
for
i
in
raw_items
:
i
.
is_valid
(
raise_exception
=
True
)
return
raw_items
def
to_representation
(
self
,
conf
):
page
=
conf
[
'page'
]
first
=
funkwhale_utils
.
set_query_parameter
(
...
...
api/funkwhale_api/federation/tasks.py
0 → 100644
View file @
520fb9d0
from
funkwhale_api.taskapp
import
celery
from
.
import
library
as
lb
from
.
import
models
@
celery
.
app
.
task
(
name
=
'federation.scan_library'
)
@
celery
.
require_instance
(
models
.
Library
,
'library'
)
def
scan_library
(
library
):
if
not
library
.
federation_enabled
:
return
data
=
lb
.
get_library_data
(
library
.
url
)
scan_library_page
.
delay
(
library_id
=
library
.
id
,
page_url
=
data
[
'first'
])
@
celery
.
app
.
task
(
name
=
'federation.scan_library_page'
)
@
celery
.
require_instance
(
models
.
Library
,
'library'
)
def
scan_library_page
(
library
,
page_url
):
if
not
library
.
federation_enabled
:
return
data
=
lb
.
get_library_page
(
library
,
page_url
)
lts
=
[]
for
item_serializer
in
data
[
'items'
]:
lts
.
append
(
item_serializer
.
save
())
api/funkwhale_api/radios/models.py
View file @
520fb9d0
...
...
@@ -4,6 +4,7 @@ from django.core.exceptions import ValidationError
from
django.contrib.postgres.fields
import
JSONField
from
django.contrib.contenttypes.fields
import
GenericForeignKey
from
django.contrib.contenttypes.models
import
ContentType
from
django.core.serializers.json
import
DjangoJSONEncoder
from
funkwhale_api.music.models
import
Track
...
...
@@ -23,7 +24,7 @@ class Radio(models.Model):
creation_date
=
models
.
DateTimeField
(
default
=
timezone
.
now
)
is_public
=
models
.
BooleanField
(
default
=
False
)
version
=
models
.
PositiveIntegerField
(
default
=
0
)
config
=
JSONField
()
config
=
JSONField
(
encoder
=
DjangoJSONEncoder
)
def
get_candidates
(
self
):
return
filters
.
run
(
self
.
config
)
...
...
api/tests/federation/test_serializers.py
View file @
520fb9d0
...
...
@@ -386,6 +386,8 @@ def test_paginated_collection_serializer_validation():
'id'
:
'https://test.federation/test'
,
'totalItems'
:
5
,
'actor'
:
'http://test.actor'
,
'first'
:
'https://test.federation/test?page=1'
,
'last'
:
'https://test.federation/test?page=1'
,
'items'
:
[]
}
...
...
@@ -407,6 +409,8 @@ def test_collection_page_serializer_validation():
'totalItems'
:
5
,
'actor'
:
'https://test.actor'
,
'items'
:
[],
'first'
:
'https://test.federation/test?page=1'
,
'last'
:
'https://test.federation/test?page=3'
,
'prev'
:
base
+
'?page=1'
,
'next'
:
base
+
'?page=3'
,
'partOf'
:
base
,
...
...
@@ -426,6 +430,21 @@ def test_collection_page_serializer_validation():
assert
serializer
.
validated_data
[
'partOf'
]
==
data
[
'partOf'
]
def
test_collection_page_serializer_can_validate_child
():
base
=
'https://test.federation/test'
data
=
{
'items'
:
[{
'in'
:
'valid'
}],
}
serializer
=
serializers
.
CollectionPageSerializer
(
data
=
data
,
context
=
{
'item_serializer'
:
serializers
.
AudioSerializer
}
)
assert
serializer
.
is_valid
()
is
False
assert
'items'
in
serializer
.
errors
def
test_collection_page_serializer
(
factories
):
tfs
=
factories
[
'music.TrackFile'
].
create_batch
(
size
=
5
)
actor
=
factories
[
'federation.Actor'
](
local
=
True
)
...
...
api/tests/federation/test_tasks.py
0 → 100644
View file @
520fb9d0
from
django.core.paginator
import
Paginator
from
funkwhale_api.federation
import
serializers
from
funkwhale_api.federation
import
tasks
def
test_scan_library_does_nothing_if_federation_disabled
(
mocker
,
factories
):
library
=
factories
[
'federation.Library'
](
federation_enabled
=
False
)
tasks
.
scan_library
(
library_id
=
library
.
pk
)
assert
library
.
tracks
.
count
()
==
0
def
test_scan_library_page_does_nothing_if_federation_disabled
(
mocker
,
factories
):
library
=
factories
[
'federation.Library'
](
federation_enabled
=
False
)
tasks
.
scan_library_page
(
library_id
=
library
.
pk
,
page_url
=
None
)
assert
library
.
tracks
.
count
()
==
0
def
test_scan_library_fetches_page_and_calls_scan_page
(
mocker
,
factories
,
r_mock
):
library
=
factories
[
'federation.Library'
](
federation_enabled
=
True
)
collection_conf
=
{
'actor'
:
library
.
actor
,
'id'
:
library
.
url
,
'page_size'
:
10
,
'items'
:
range
(
10
),
}
collection
=
serializers
.
PaginatedCollectionSerializer
(
collection_conf
)
scan_page
=
mocker
.
patch
(
'funkwhale_api.federation.tasks.scan_library_page.delay'
)
r_mock
.
get
(
collection_conf
[
'id'
],
json
=
collection
.
data
)
tasks
.
scan_library
(
library_id
=
library
.
pk
)
scan_page
.
assert_called_once_with
(
library_id
=
library
.
id
,
page_url
=
collection
.
data
[
'first'
],
)
def
test_scan_page_fetches_page_and_creates_tracks
(
mocker
,
factories
,
r_mock
):
library
=
factories
[
'federation.Library'
](
federation_enabled
=
True
)
tfs
=
factories
[
'music.TrackFile'
].
create_batch
(
size
=
5
)
page_conf
=
{
'actor'
:
library
.
actor
,
'id'
:
library
.
url
,
'page'
:
Paginator
(
tfs
,
5
).
page
(
1
),
'item_serializer'
:
serializers
.
AudioSerializer
,
}
page
=
serializers
.
CollectionPageSerializer
(
page_conf
)
#scan_page = mocker.patch(
# 'funkwhale_api.federation.tasks.scan_library_page.delay')
r_mock
.
get
(
page
.
data
[
'id'
],
json
=
page
.
data
)
tasks
.
scan_library_page
(
library_id
=
library
.
pk
,
page_url
=
page
.
data
[
'id'
])
lts
=
list
(
library
.
tracks
.
all
().
order_by
(
'-published_date'
))
assert
len
(
lts
)
==
5
front/src/components/federation/LibraryForm.vue
View file @
520fb9d0
...
...
@@ -43,7 +43,7 @@ export default {
data
()
{
return
{
isLoading
:
false
,
libraryUsername
:
'
library@node2.funkwhale.test
'
,
libraryUsername
:
''
,
result
:
null
,
errors
:
[]
}
...
...
front/src/views/federation/LibraryDetail.vue
View file @
520fb9d0
...
...
@@ -77,6 +77,14 @@
</td>
<td></td>
</tr>
<tr>
<td>
Last fetched
</td>
<td>
<human-date
v-if=
"object.fetched_date"
:date=
"object.fetched_date"
></human-date>
<
template
v-else
>
Never
</
template
>
</td>
<td></td>
</tr>
</tbody>
</table>
</div>
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment