Commit 6dde4b73 authored by Eliot Berriot's avatar Eliot Berriot
Browse files

Implement tag models

parent c170ee93
......@@ -161,7 +161,6 @@ THIRD_PARTY_APPS = (
"oauth2_provider",
"rest_framework",
"rest_framework.authtoken",
"taggit",
"rest_auth",
"rest_auth.registration",
"dynamic_preferences",
......@@ -201,6 +200,7 @@ LOCAL_APPS = (
"funkwhale_api.history",
"funkwhale_api.playlists",
"funkwhale_api.subsonic",
"funkwhale_api.tags",
)
# See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
......
import math
import random
from django.conf import settings
from django.core.management.base import BaseCommand
from django.db import transaction
from funkwhale_api.federation import keys
from funkwhale_api.federation import models as federation_models
from funkwhale_api.music import models as music_models
from funkwhale_api.tags import models as tags_models
from funkwhale_api.users import models as users_models
BATCH_SIZE = 500
def create_local_accounts(factories, count, dependencies):
password = factories["users.User"].build().password
users = factories["users.User"].build_batch(size=count)
for user in users:
# we set the hashed password by hand, because computing one for each user
# is CPU intensive
user.password = password
users = users_models.User.objects.bulk_create(users, batch_size=BATCH_SIZE)
actors = []
domain = federation_models.Domain.objects.get_or_create(
name=settings.FEDERATION_HOSTNAME
)[0]
users = [u for u in users if u.pk]
private, public = keys.get_key_pair()
for user in users:
if not user.pk:
continue
actor = federation_models.Actor(
private_key=private.decode("utf-8"),
public_key=public.decode("utf-8"),
**users_models.get_actor_data(user.username, domain=domain)
)
actors.append(actor)
actors = federation_models.Actor.objects.bulk_create(actors, batch_size=BATCH_SIZE)
for user, actor in zip(users, actors):
user.actor = actor
users_models.User.objects.bulk_update(users, ["actor"])
return actors
def create_tagged_tracks(factories, count, dependencies):
objs = []
for track in dependencies["tracks"]:
tag = random.choice(dependencies["tags"])
objs.append(factories["tags.TaggedItem"](content_object=track, tag=tag))
return tags_models.TaggedItem.objects.bulk_create(
objs, batch_size=BATCH_SIZE, ignore_conflicts=True
)
CONFIG = [
{
"id": "tracks",
"model": music_models.Track,
"factory": "music.Track",
"factory_kwargs": {"artist": None, "album": None},
"depends_on": [
{"field": "album", "id": "albums", "default_factor": 0.1},
{"field": "artist", "id": "artists", "default_factor": 0.05},
],
},
{
"id": "albums",
"model": music_models.Album,
"factory": "music.Album",
"factory_kwargs": {"artist": None},
"depends_on": [{"field": "artist", "id": "artists", "default_factor": 0.3}],
},
{"id": "artists", "model": music_models.Artist, "factory": "music.Artist"},
{
"id": "local_accounts",
"model": federation_models.Actor,
"handler": create_local_accounts,
},
{
"id": "local_libraries",
"model": music_models.Library,
"factory": "music.Library",
"factory_kwargs": {"actor": None},
"depends_on": [{"field": "actor", "id": "local_accounts", "default_factor": 1}],
},
{
"id": "local_uploads",
"model": music_models.Upload,
"factory": "music.Upload",
"factory_kwargs": {"import_status": "finished", "library": None, "track": None},
"depends_on": [
{
"field": "library",
"id": "local_libraries",
"default_factor": 0.05,
"queryset": music_models.Library.objects.all().select_related(
"actor__user"
),
},
{"field": "track", "id": "tracks", "default_factor": 1},
],
},
{"id": "tags", "model": tags_models.Tag, "factory": "tags.Tag"},
{
"id": "track_tags",
"model": tags_models.TaggedItem,
"handler": create_tagged_tracks,
"depends_on": [
{
"field": "tag",
"id": "tags",
"default_factor": 0.1,
"queryset": tags_models.Tag.objects.all(),
"set": False,
},
{
"field": "content_object",
"id": "tracks",
"default_factor": 1,
"set": False,
},
],
},
]
CONFIG_BY_ID = {c["id"]: c for c in CONFIG}
class Rollback(Exception):
pass
def create_objects(row, factories, count, **factory_kwargs):
return factories[row["factory"]].build_batch(size=count, **factory_kwargs)
class Command(BaseCommand):
help = """
Inject demo data into your database. Useful for load testing, or setting up a demo instance.
Use with caution and only if you know what you are doing.
"""
def add_arguments(self, parser):
parser.add_argument(
"--no-dry-run",
action="store_false",
dest="dry_run",
help="Commit the changes to the database",
)
parser.add_argument(
"--create-dependencies", action="store_true", dest="create_dependencies"
)
for row in CONFIG:
parser.add_argument(
"--{}".format(row["id"].replace("_", "-")),
dest=row["id"],
type=int,
help="Number of {} objects to create".format(row["id"]),
)
dependencies = row.get("depends_on", [])
for dependency in dependencies:
parser.add_argument(
"--{}-{}-factor".format(row["id"], dependency["field"]),
dest="{}_{}_factor".format(row["id"], dependency["field"]),
type=float,
help="Number of {} objects to create per {} object".format(
dependency["id"], row["id"]
),
)
def handle(self, *args, **options):
from django.apps import apps
from funkwhale_api import factories
app_names = [app.name for app in apps.app_configs.values()]
factories.registry.autodiscover(app_names)
try:
return self.inner_handle(*args, **options)
except Rollback:
pass
@transaction.atomic
def inner_handle(self, *args, **options):
results = {}
for row in CONFIG:
self.create_batch(row, results, options, count=options.get(row["id"]))
self.stdout.write("\nFinal state of database:\n\n")
for row in CONFIG:
model = row["model"]
total = model.objects.all().count()
self.stdout.write("- {} {} objects".format(total, row["id"]))
self.stdout.write("")
if options["dry_run"]:
self.stdout.write(
"Run this command with --no-dry-run to commit the changes to the database"
)
raise Rollback()
self.stdout.write(self.style.SUCCESS("Done!"))
def create_batch(self, row, results, options, count):
from funkwhale_api import factories
if row["id"] in results:
# already generated
return results[row["id"]]
if not count:
return []
dependencies = row.get("depends_on", [])
dependencies_results = {}
create_dependencies = options.get("create_dependencies")
for dependency in dependencies:
if not create_dependencies:
continue
dep_count = options.get(dependency["id"])
if dep_count is None:
factor = options[
"{}_{}_factor".format(row["id"], dependency["field"])
] or dependency.get("default_factor")
dep_count = math.ceil(factor * count)
dependencies_results[dependency["id"]] = self.create_batch(
CONFIG_BY_ID[dependency["id"]], results, options, count=dep_count
)
self.stdout.write("Creating {} {}…".format(count, row["id"]))
handler = row.get("handler")
if handler:
objects = handler(
factories.registry, count, dependencies=dependencies_results
)
else:
objects = create_objects(
row, factories.registry, count, **row.get("factory_kwargs", {})
)
for dependency in dependencies:
if not dependency.get("set", True):
continue
if create_dependencies:
candidates = dependencies_results[dependency["id"]]
else:
# we use existing objects in the database
queryset = dependency.get(
"queryset", CONFIG_BY_ID[dependency["id"]]["model"].objects.all()
)
candidates = list(queryset.values_list("pk", flat=True))
picked_pks = [random.choice(candidates) for _ in objects]
picked_objects = {o.pk: o for o in queryset.filter(pk__in=picked_pks)}
for i, obj in enumerate(objects):
if create_dependencies:
value = random.choice(candidates)
else:
value = picked_objects[picked_pks[i]]
setattr(obj, dependency["field"], value)
if not handler:
objects = row["model"].objects.bulk_create(objects, batch_size=BATCH_SIZE)
results[row["id"]] = objects
return objects
# Generated by Django 2.0.2 on 2018-02-27 18:43
from django.db import migrations
from django.contrib.postgres.operations import CITextExtension
class CustomCITExtension(CITextExtension):
def database_forwards(self, app_label, schema_editor, from_state, to_state):
check_sql = "SELECT 1 FROM pg_extension WHERE extname = 'citext'"
with schema_editor.connection.cursor() as cursor:
cursor.execute(check_sql)
result = cursor.fetchall()
if result:
return
return super().database_forwards(app_label, schema_editor, from_state, to_state)
class Migration(migrations.Migration):
dependencies = [("common", "0002_mutation")]
operations = [CustomCITExtension()]
import uuid
import factory
import random
import persisting_theory
from django.conf import settings
......@@ -46,6 +47,268 @@ class NoUpdateOnCreate:
return
TAGS_DATA = {
"type": [
"acoustic",
"acid",
"ambient",
"alternative",
"brutalist",
"chill",
"club",
"cold",
"cool",
"contemporary",
"dark",
"doom",
"electro",
"folk",
"freestyle",
"fusion",
"garage",
"glitch",
"hard",
"healing",
"industrial",
"instrumental",
"hardcore",
"holiday",
"hot",
"liquid",
"modern",
"minimalist",
"new",
"parody",
"postmodern",
"progressive",
"smooth",
"symphonic",
"traditional",
"tribal",
"metal",
],
"genre": [
"blues",
"classical",
"chiptune",
"dance",
"disco",
"funk",
"jazz",
"house",
"hiphop",
"NewAge",
"pop",
"punk",
"rap",
"RnB",
"reggae",
"rock",
"soul",
"soundtrack",
"ska",
"swing",
"trance",
],
"nationality": [
"Afghan",
"Albanian",
"Algerian",
"American",
"Andorran",
"Angolan",
"Antiguans",
"Argentinean",
"Armenian",
"Australian",
"Austrian",
"Azerbaijani",
"Bahamian",
"Bahraini",
"Bangladeshi",
"Barbadian",
"Barbudans",
"Batswana",
"Belarusian",
"Belgian",
"Belizean",
"Beninese",
"Bhutanese",
"Bolivian",
"Bosnian",
"Brazilian",
"British",
"Bruneian",
"Bulgarian",
"Burkinabe",
"Burmese",
"Burundian",
"Cambodian",
"Cameroonian",
"Canadian",
"Cape Verdean",
"Central African",
"Chadian",
"Chilean",
"Chinese",
"Colombian",
"Comoran",
"Congolese",
"Costa Rican",
"Croatian",
"Cuban",
"Cypriot",
"Czech",
"Danish",
"Djibouti",
"Dominican",
"Dutch",
"East Timorese",
"Ecuadorean",
"Egyptian",
"Emirian",
"Equatorial Guinean",
"Eritrean",
"Estonian",
"Ethiopian",
"Fijian",
"Filipino",
"Finnish",
"French",
"Gabonese",
"Gambian",
"Georgian",
"German",
"Ghanaian",
"Greek",
"Grenadian",
"Guatemalan",
"Guinea-Bissauan",
"Guinean",
"Guyanese",
"Haitian",
"Herzegovinian",
"Honduran",
"Hungarian",
"I-Kiribati",
"Icelander",
"Indian",
"Indonesian",
"Iranian",
"Iraqi",
"Irish",
"Israeli",
"Italian",
"Ivorian",
"Jamaican",
"Japanese",
"Jordanian",
"Kazakhstani",
"Kenyan",
"Kittian and Nevisian",
"Kuwaiti",
"Kyrgyz",
"Laotian",
"Latvian",
"Lebanese",
"Liberian",
"Libyan",
"Liechtensteiner",
"Lithuanian",
"Luxembourger",
"Macedonian",
"Malagasy",
"Malawian",
"Malaysian",
"Maldivian",
"Malian",
"Maltese",
"Marshallese",
"Mauritanian",
"Mauritian",
"Mexican",
"Micronesian",
"Moldovan",
"Monacan",
"Mongolian",
"Moroccan",
"Mosotho",
"Motswana",
"Mozambican",
"Namibian",
"Nauruan",
"Nepalese",
"New Zealander",
"Ni-Vanuatu",
"Nicaraguan",
"Nigerian",
"Nigerien",
"North Korean",
"Northern Irish",
"Norwegian",
"Omani",
"Pakistani",
"Palauan",
"Panamanian",
"Papua New Guinean",
"Paraguayan",
"Peruvian",
"Polish",
"Portuguese",
"Qatari",
"Romanian",
"Russian",
"Rwandan",
"Saint Lucian",
"Salvadoran",
"Samoan",
"San Marinese",
"Sao Tomean",
"Saudi",
"Scottish",
"Senegalese",
"Serbian",
"Seychellois",
"Sierra Leonean",
"Singaporean",
"Slovakian",
"Slovenian",
"Solomon Islander",
"Somali",
"South African",
"South Korean",
"Spanish",
"Sri Lankan",
"Sudanese",
"Surinamer",
"Swazi",
"Swedish",
"Swiss",
"Syrian",
"Taiwanese",
"Tajik",
"Tanzanian",
"Thai",
"Togolese",
"Tongan",
"Trinidadian",
"Tunisian",
"Turkish",
"Tuvaluan",
"Ugandan",
"Ukrainian",
"Uruguayan",
"Uzbekistani",
"Venezuelan",
"Vietnamese",
"Welsh",
"Yemenite",
"Zambian",
"Zimbabwean",
],
}
class FunkwhaleProvider(internet_provider.Provider):
"""
Our own faker data generator, since built-in ones are sometimes
......@@ -61,5 +324,40 @@ class FunkwhaleProvider(internet_provider.Provider):
path = path_generator()
return "{}://{}/{}".format(protocol, domain, path)
def user_name(self):
u = super().user_name()
return "{}{}".format(u, random.randint(10, 999))
def music_genre(self):
return random.choice(TAGS_DATA["genre"])
def music_type(self):
return random.choice(TAGS_DATA["type"])
def music_nationality(self):
return random.choice(TAGS_DATA["nationality"])
def music_hashtag(self, prefix_length=4):
genre = self.music_genre()
prefixes = [genre]