Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
jovuit
funkwhale
Commits
6dde4b73
Commit
6dde4b73
authored
Jul 08, 2019
by
Eliot Berriot
Browse files
Implement tag models
parent
c170ee93
Changes
28
Hide whitespace changes
Inline
Side-by-side
api/config/settings/common.py
View file @
6dde4b73
...
...
@@ -161,7 +161,6 @@ THIRD_PARTY_APPS = (
"oauth2_provider"
,
"rest_framework"
,
"rest_framework.authtoken"
,
"taggit"
,
"rest_auth"
,
"rest_auth.registration"
,
"dynamic_preferences"
,
...
...
@@ -201,6 +200,7 @@ LOCAL_APPS = (
"funkwhale_api.history"
,
"funkwhale_api.playlists"
,
"funkwhale_api.subsonic"
,
"funkwhale_api.tags"
,
)
# See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
...
...
api/funkwhale_api/common/management/commands/load_test_data.py
0 → 100644
View file @
6dde4b73
import
math
import
random
from
django.conf
import
settings
from
django.core.management.base
import
BaseCommand
from
django.db
import
transaction
from
funkwhale_api.federation
import
keys
from
funkwhale_api.federation
import
models
as
federation_models
from
funkwhale_api.music
import
models
as
music_models
from
funkwhale_api.tags
import
models
as
tags_models
from
funkwhale_api.users
import
models
as
users_models
BATCH_SIZE
=
500
def
create_local_accounts
(
factories
,
count
,
dependencies
):
password
=
factories
[
"users.User"
].
build
().
password
users
=
factories
[
"users.User"
].
build_batch
(
size
=
count
)
for
user
in
users
:
# we set the hashed password by hand, because computing one for each user
# is CPU intensive
user
.
password
=
password
users
=
users_models
.
User
.
objects
.
bulk_create
(
users
,
batch_size
=
BATCH_SIZE
)
actors
=
[]
domain
=
federation_models
.
Domain
.
objects
.
get_or_create
(
name
=
settings
.
FEDERATION_HOSTNAME
)[
0
]
users
=
[
u
for
u
in
users
if
u
.
pk
]
private
,
public
=
keys
.
get_key_pair
()
for
user
in
users
:
if
not
user
.
pk
:
continue
actor
=
federation_models
.
Actor
(
private_key
=
private
.
decode
(
"utf-8"
),
public_key
=
public
.
decode
(
"utf-8"
),
**
users_models
.
get_actor_data
(
user
.
username
,
domain
=
domain
)
)
actors
.
append
(
actor
)
actors
=
federation_models
.
Actor
.
objects
.
bulk_create
(
actors
,
batch_size
=
BATCH_SIZE
)
for
user
,
actor
in
zip
(
users
,
actors
):
user
.
actor
=
actor
users_models
.
User
.
objects
.
bulk_update
(
users
,
[
"actor"
])
return
actors
def
create_tagged_tracks
(
factories
,
count
,
dependencies
):
objs
=
[]
for
track
in
dependencies
[
"tracks"
]:
tag
=
random
.
choice
(
dependencies
[
"tags"
])
objs
.
append
(
factories
[
"tags.TaggedItem"
](
content_object
=
track
,
tag
=
tag
))
return
tags_models
.
TaggedItem
.
objects
.
bulk_create
(
objs
,
batch_size
=
BATCH_SIZE
,
ignore_conflicts
=
True
)
CONFIG
=
[
{
"id"
:
"tracks"
,
"model"
:
music_models
.
Track
,
"factory"
:
"music.Track"
,
"factory_kwargs"
:
{
"artist"
:
None
,
"album"
:
None
},
"depends_on"
:
[
{
"field"
:
"album"
,
"id"
:
"albums"
,
"default_factor"
:
0.1
},
{
"field"
:
"artist"
,
"id"
:
"artists"
,
"default_factor"
:
0.05
},
],
},
{
"id"
:
"albums"
,
"model"
:
music_models
.
Album
,
"factory"
:
"music.Album"
,
"factory_kwargs"
:
{
"artist"
:
None
},
"depends_on"
:
[{
"field"
:
"artist"
,
"id"
:
"artists"
,
"default_factor"
:
0.3
}],
},
{
"id"
:
"artists"
,
"model"
:
music_models
.
Artist
,
"factory"
:
"music.Artist"
},
{
"id"
:
"local_accounts"
,
"model"
:
federation_models
.
Actor
,
"handler"
:
create_local_accounts
,
},
{
"id"
:
"local_libraries"
,
"model"
:
music_models
.
Library
,
"factory"
:
"music.Library"
,
"factory_kwargs"
:
{
"actor"
:
None
},
"depends_on"
:
[{
"field"
:
"actor"
,
"id"
:
"local_accounts"
,
"default_factor"
:
1
}],
},
{
"id"
:
"local_uploads"
,
"model"
:
music_models
.
Upload
,
"factory"
:
"music.Upload"
,
"factory_kwargs"
:
{
"import_status"
:
"finished"
,
"library"
:
None
,
"track"
:
None
},
"depends_on"
:
[
{
"field"
:
"library"
,
"id"
:
"local_libraries"
,
"default_factor"
:
0.05
,
"queryset"
:
music_models
.
Library
.
objects
.
all
().
select_related
(
"actor__user"
),
},
{
"field"
:
"track"
,
"id"
:
"tracks"
,
"default_factor"
:
1
},
],
},
{
"id"
:
"tags"
,
"model"
:
tags_models
.
Tag
,
"factory"
:
"tags.Tag"
},
{
"id"
:
"track_tags"
,
"model"
:
tags_models
.
TaggedItem
,
"handler"
:
create_tagged_tracks
,
"depends_on"
:
[
{
"field"
:
"tag"
,
"id"
:
"tags"
,
"default_factor"
:
0.1
,
"queryset"
:
tags_models
.
Tag
.
objects
.
all
(),
"set"
:
False
,
},
{
"field"
:
"content_object"
,
"id"
:
"tracks"
,
"default_factor"
:
1
,
"set"
:
False
,
},
],
},
]
CONFIG_BY_ID
=
{
c
[
"id"
]:
c
for
c
in
CONFIG
}
class
Rollback
(
Exception
):
pass
def
create_objects
(
row
,
factories
,
count
,
**
factory_kwargs
):
return
factories
[
row
[
"factory"
]].
build_batch
(
size
=
count
,
**
factory_kwargs
)
class
Command
(
BaseCommand
):
help
=
"""
Inject demo data into your database. Useful for load testing, or setting up a demo instance.
Use with caution and only if you know what you are doing.
"""
def
add_arguments
(
self
,
parser
):
parser
.
add_argument
(
"--no-dry-run"
,
action
=
"store_false"
,
dest
=
"dry_run"
,
help
=
"Commit the changes to the database"
,
)
parser
.
add_argument
(
"--create-dependencies"
,
action
=
"store_true"
,
dest
=
"create_dependencies"
)
for
row
in
CONFIG
:
parser
.
add_argument
(
"--{}"
.
format
(
row
[
"id"
].
replace
(
"_"
,
"-"
)),
dest
=
row
[
"id"
],
type
=
int
,
help
=
"Number of {} objects to create"
.
format
(
row
[
"id"
]),
)
dependencies
=
row
.
get
(
"depends_on"
,
[])
for
dependency
in
dependencies
:
parser
.
add_argument
(
"--{}-{}-factor"
.
format
(
row
[
"id"
],
dependency
[
"field"
]),
dest
=
"{}_{}_factor"
.
format
(
row
[
"id"
],
dependency
[
"field"
]),
type
=
float
,
help
=
"Number of {} objects to create per {} object"
.
format
(
dependency
[
"id"
],
row
[
"id"
]
),
)
def
handle
(
self
,
*
args
,
**
options
):
from
django.apps
import
apps
from
funkwhale_api
import
factories
app_names
=
[
app
.
name
for
app
in
apps
.
app_configs
.
values
()]
factories
.
registry
.
autodiscover
(
app_names
)
try
:
return
self
.
inner_handle
(
*
args
,
**
options
)
except
Rollback
:
pass
@
transaction
.
atomic
def
inner_handle
(
self
,
*
args
,
**
options
):
results
=
{}
for
row
in
CONFIG
:
self
.
create_batch
(
row
,
results
,
options
,
count
=
options
.
get
(
row
[
"id"
]))
self
.
stdout
.
write
(
"
\n
Final state of database:
\n\n
"
)
for
row
in
CONFIG
:
model
=
row
[
"model"
]
total
=
model
.
objects
.
all
().
count
()
self
.
stdout
.
write
(
"- {} {} objects"
.
format
(
total
,
row
[
"id"
]))
self
.
stdout
.
write
(
""
)
if
options
[
"dry_run"
]:
self
.
stdout
.
write
(
"Run this command with --no-dry-run to commit the changes to the database"
)
raise
Rollback
()
self
.
stdout
.
write
(
self
.
style
.
SUCCESS
(
"Done!"
))
def
create_batch
(
self
,
row
,
results
,
options
,
count
):
from
funkwhale_api
import
factories
if
row
[
"id"
]
in
results
:
# already generated
return
results
[
row
[
"id"
]]
if
not
count
:
return
[]
dependencies
=
row
.
get
(
"depends_on"
,
[])
dependencies_results
=
{}
create_dependencies
=
options
.
get
(
"create_dependencies"
)
for
dependency
in
dependencies
:
if
not
create_dependencies
:
continue
dep_count
=
options
.
get
(
dependency
[
"id"
])
if
dep_count
is
None
:
factor
=
options
[
"{}_{}_factor"
.
format
(
row
[
"id"
],
dependency
[
"field"
])
]
or
dependency
.
get
(
"default_factor"
)
dep_count
=
math
.
ceil
(
factor
*
count
)
dependencies_results
[
dependency
[
"id"
]]
=
self
.
create_batch
(
CONFIG_BY_ID
[
dependency
[
"id"
]],
results
,
options
,
count
=
dep_count
)
self
.
stdout
.
write
(
"Creating {} {}…"
.
format
(
count
,
row
[
"id"
]))
handler
=
row
.
get
(
"handler"
)
if
handler
:
objects
=
handler
(
factories
.
registry
,
count
,
dependencies
=
dependencies_results
)
else
:
objects
=
create_objects
(
row
,
factories
.
registry
,
count
,
**
row
.
get
(
"factory_kwargs"
,
{})
)
for
dependency
in
dependencies
:
if
not
dependency
.
get
(
"set"
,
True
):
continue
if
create_dependencies
:
candidates
=
dependencies_results
[
dependency
[
"id"
]]
else
:
# we use existing objects in the database
queryset
=
dependency
.
get
(
"queryset"
,
CONFIG_BY_ID
[
dependency
[
"id"
]][
"model"
].
objects
.
all
()
)
candidates
=
list
(
queryset
.
values_list
(
"pk"
,
flat
=
True
))
picked_pks
=
[
random
.
choice
(
candidates
)
for
_
in
objects
]
picked_objects
=
{
o
.
pk
:
o
for
o
in
queryset
.
filter
(
pk__in
=
picked_pks
)}
for
i
,
obj
in
enumerate
(
objects
):
if
create_dependencies
:
value
=
random
.
choice
(
candidates
)
else
:
value
=
picked_objects
[
picked_pks
[
i
]]
setattr
(
obj
,
dependency
[
"field"
],
value
)
if
not
handler
:
objects
=
row
[
"model"
].
objects
.
bulk_create
(
objects
,
batch_size
=
BATCH_SIZE
)
results
[
row
[
"id"
]]
=
objects
return
objects
api/funkwhale_api/common/migrations/0003_cit_extension.py
0 → 100644
View file @
6dde4b73
# Generated by Django 2.0.2 on 2018-02-27 18:43
from
django.db
import
migrations
from
django.contrib.postgres.operations
import
CITextExtension
class
CustomCITExtension
(
CITextExtension
):
def
database_forwards
(
self
,
app_label
,
schema_editor
,
from_state
,
to_state
):
check_sql
=
"SELECT 1 FROM pg_extension WHERE extname = 'citext'"
with
schema_editor
.
connection
.
cursor
()
as
cursor
:
cursor
.
execute
(
check_sql
)
result
=
cursor
.
fetchall
()
if
result
:
return
return
super
().
database_forwards
(
app_label
,
schema_editor
,
from_state
,
to_state
)
class
Migration
(
migrations
.
Migration
):
dependencies
=
[(
"common"
,
"0002_mutation"
)]
operations
=
[
CustomCITExtension
()]
api/funkwhale_api/factories.py
View file @
6dde4b73
import
uuid
import
factory
import
random
import
persisting_theory
from
django.conf
import
settings
...
...
@@ -46,6 +47,268 @@ class NoUpdateOnCreate:
return
TAGS_DATA
=
{
"type"
:
[
"acoustic"
,
"acid"
,
"ambient"
,
"alternative"
,
"brutalist"
,
"chill"
,
"club"
,
"cold"
,
"cool"
,
"contemporary"
,
"dark"
,
"doom"
,
"electro"
,
"folk"
,
"freestyle"
,
"fusion"
,
"garage"
,
"glitch"
,
"hard"
,
"healing"
,
"industrial"
,
"instrumental"
,
"hardcore"
,
"holiday"
,
"hot"
,
"liquid"
,
"modern"
,
"minimalist"
,
"new"
,
"parody"
,
"postmodern"
,
"progressive"
,
"smooth"
,
"symphonic"
,
"traditional"
,
"tribal"
,
"metal"
,
],
"genre"
:
[
"blues"
,
"classical"
,
"chiptune"
,
"dance"
,
"disco"
,
"funk"
,
"jazz"
,
"house"
,
"hiphop"
,
"NewAge"
,
"pop"
,
"punk"
,
"rap"
,
"RnB"
,
"reggae"
,
"rock"
,
"soul"
,
"soundtrack"
,
"ska"
,
"swing"
,
"trance"
,
],
"nationality"
:
[
"Afghan"
,
"Albanian"
,
"Algerian"
,
"American"
,
"Andorran"
,
"Angolan"
,
"Antiguans"
,
"Argentinean"
,
"Armenian"
,
"Australian"
,
"Austrian"
,
"Azerbaijani"
,
"Bahamian"
,
"Bahraini"
,
"Bangladeshi"
,
"Barbadian"
,
"Barbudans"
,
"Batswana"
,
"Belarusian"
,
"Belgian"
,
"Belizean"
,
"Beninese"
,
"Bhutanese"
,
"Bolivian"
,
"Bosnian"
,
"Brazilian"
,
"British"
,
"Bruneian"
,
"Bulgarian"
,
"Burkinabe"
,
"Burmese"
,
"Burundian"
,
"Cambodian"
,
"Cameroonian"
,
"Canadian"
,
"Cape Verdean"
,
"Central African"
,
"Chadian"
,
"Chilean"
,
"Chinese"
,
"Colombian"
,
"Comoran"
,
"Congolese"
,
"Costa Rican"
,
"Croatian"
,
"Cuban"
,
"Cypriot"
,
"Czech"
,
"Danish"
,
"Djibouti"
,
"Dominican"
,
"Dutch"
,
"East Timorese"
,
"Ecuadorean"
,
"Egyptian"
,
"Emirian"
,
"Equatorial Guinean"
,
"Eritrean"
,
"Estonian"
,
"Ethiopian"
,
"Fijian"
,
"Filipino"
,
"Finnish"
,
"French"
,
"Gabonese"
,
"Gambian"
,
"Georgian"
,
"German"
,
"Ghanaian"
,
"Greek"
,
"Grenadian"
,
"Guatemalan"
,
"Guinea-Bissauan"
,
"Guinean"
,
"Guyanese"
,
"Haitian"
,
"Herzegovinian"
,
"Honduran"
,
"Hungarian"
,
"I-Kiribati"
,
"Icelander"
,
"Indian"
,
"Indonesian"
,
"Iranian"
,
"Iraqi"
,
"Irish"
,
"Israeli"
,
"Italian"
,
"Ivorian"
,
"Jamaican"
,
"Japanese"
,
"Jordanian"
,
"Kazakhstani"
,
"Kenyan"
,
"Kittian and Nevisian"
,
"Kuwaiti"
,
"Kyrgyz"
,
"Laotian"
,
"Latvian"
,
"Lebanese"
,
"Liberian"
,
"Libyan"
,
"Liechtensteiner"
,
"Lithuanian"
,
"Luxembourger"
,
"Macedonian"
,
"Malagasy"
,
"Malawian"
,
"Malaysian"
,
"Maldivian"
,
"Malian"
,
"Maltese"
,
"Marshallese"
,
"Mauritanian"
,
"Mauritian"
,
"Mexican"
,
"Micronesian"
,
"Moldovan"
,
"Monacan"
,
"Mongolian"
,
"Moroccan"
,
"Mosotho"
,
"Motswana"
,
"Mozambican"
,
"Namibian"
,
"Nauruan"
,
"Nepalese"
,
"New Zealander"
,
"Ni-Vanuatu"
,
"Nicaraguan"
,
"Nigerian"
,
"Nigerien"
,
"North Korean"
,
"Northern Irish"
,
"Norwegian"
,
"Omani"
,
"Pakistani"
,
"Palauan"
,
"Panamanian"
,
"Papua New Guinean"
,
"Paraguayan"
,
"Peruvian"
,
"Polish"
,
"Portuguese"
,
"Qatari"
,
"Romanian"
,
"Russian"
,
"Rwandan"
,
"Saint Lucian"
,
"Salvadoran"
,
"Samoan"
,
"San Marinese"
,
"Sao Tomean"
,
"Saudi"
,
"Scottish"
,
"Senegalese"
,
"Serbian"
,
"Seychellois"
,
"Sierra Leonean"
,
"Singaporean"
,
"Slovakian"
,
"Slovenian"
,
"Solomon Islander"
,
"Somali"
,
"South African"
,
"South Korean"
,
"Spanish"
,
"Sri Lankan"
,
"Sudanese"
,
"Surinamer"
,
"Swazi"
,
"Swedish"
,
"Swiss"
,
"Syrian"
,
"Taiwanese"
,
"Tajik"
,
"Tanzanian"
,
"Thai"
,
"Togolese"
,
"Tongan"
,
"Trinidadian"
,
"Tunisian"
,
"Turkish"
,
"Tuvaluan"
,
"Ugandan"
,
"Ukrainian"
,
"Uruguayan"
,
"Uzbekistani"
,
"Venezuelan"
,
"Vietnamese"
,
"Welsh"
,
"Yemenite"
,
"Zambian"
,
"Zimbabwean"
,
],
}
class
FunkwhaleProvider
(
internet_provider
.
Provider
):
"""
Our own faker data generator, since built-in ones are sometimes
...
...
@@ -61,5 +324,40 @@ class FunkwhaleProvider(internet_provider.Provider):
path
=
path_generator
()
return
"{}://{}/{}"
.
format
(
protocol
,
domain
,
path
)
def
user_name
(
self
):
u
=
super
().
user_name
()
return
"{}{}"
.
format
(
u
,
random
.
randint
(
10
,
999
))
def
music_genre
(
self
):
return
random
.
choice
(
TAGS_DATA
[
"genre"
])
def
music_type
(
self
):
return
random
.
choice
(
TAGS_DATA
[
"type"
])
def
music_nationality
(
self
):
return
random
.
choice
(
TAGS_DATA
[
"nationality"
])
def
music_hashtag
(
self
,
prefix_length
=
4
):
genre
=
self
.
music_genre
()
prefixes
=
[
genre
]