Added checks table and check cleaning/creation logic

README.md

@@ -4,4 +4,4 @@ Running tests
 -------------
 
 
-Ensure you have a postgres server running (via docker: `docker run --rm -p 5432:5432 -e POSTGRES_DB=funkwhale_network_test postgres`)
+Ensure you have a postgres server running (via docker: `docker run --rm --name=pg -p 5432:5432 -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=funkwhale_network_test timescale/timescaledb:latest-pg11`)

funkwhale_network/crawler.py

+import psycopg2
+
 from . import schemas
 
 
@@ -24,3 +26,91 @@ async def get_nodeinfo(session, nodeinfo):
 def clean_nodeinfo(data):
     schema = schemas.NodeInfo2Schema()
     return schema.load(data)
+
+
+def recursive_getattr(obj, key, permissive=True):
+    """
+    Given a dictionary such as {'user': {'name': 'Bob'}} and
+    a dotted string such as user.name, returns 'Bob'.
+
+    If the value is not present, returns None
+    """
+    v = obj
+    for k in key.split("."):
+        try:
+            v = v.get(k)
+        except (TypeError, AttributeError):
+            if not permissive:
+                raise
+            return
+        if v is None:
+            return
+
+    return v
+
+
+def clean_check(check_data, nodeinfo_data):
+    return {
+        "domain": check_data["domain"],
+        "up": check_data["up"],
+        "https": check_data["https"],
+        "open_registrations": recursive_getattr(nodeinfo_data, "openRegistrations"),
+        "federation_enabled": recursive_getattr(
+            nodeinfo_data, "metadata.library.federationEnabled"
+        ),
+        "anonymous_can_listen": recursive_getattr(
+            nodeinfo_data, "metadata.library.anonymousCanListen"
+        ),
+        "private": recursive_getattr(nodeinfo_data, "metadata.private"),
+        "usage_users_total": recursive_getattr(nodeinfo_data, "usage.users.total"),
+        "usage_users_active_half_year": recursive_getattr(
+            nodeinfo_data, "usage.users.activeHalfyear"
+        ),
+        "usage_users_active_month": recursive_getattr(
+            nodeinfo_data, "usage.users.activeMonth"
+        ),
+        "usage_listenings_total": recursive_getattr(
+            nodeinfo_data, "metadata.usage.listenings.total"
+        ),
+        "library_tracks_total": recursive_getattr(
+            nodeinfo_data, "metadata.library.tracks.total"
+        ),
+        "library_albums_total": recursive_getattr(
+            nodeinfo_data, "metadata.library.albums.total"
+        ),
+        "library_artists_total": recursive_getattr(
+            nodeinfo_data, "metadata.library.artists.total"
+        ),
+        "library_music_hours": recursive_getattr(
+            nodeinfo_data, "metadata.library.music.hours"
+        ),
+        "software_name": recursive_getattr(nodeinfo_data, "software.name"),
+        "software_version_major": recursive_getattr(
+            nodeinfo_data, "software.version.major"
+        ),
+        "software_version_minor": recursive_getattr(
+            nodeinfo_data, "software.version.minor"
+        ),
+        "software_version_patch": recursive_getattr(
+            nodeinfo_data, "software.version.patch"
+        ),
+        "software_prerelease": recursive_getattr(
+            nodeinfo_data, "software.version.prerelease"
+        ),
+        "software_build": recursive_getattr(nodeinfo_data, "software.version.build"),
+    }
+
+
+async def save_check(conn, data):
+    fields, values = [], []
+    for field, value in data.items():
+        fields.append(field)
+        values.append(value)
+
+    sql = "INSERT INTO checks (time, {}) VALUES (NOW(), {}) RETURNING *".format(
+        ", ".join(fields), ", ".join(["%s" for _ in values])
+    )
+    async with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cursor:
+        await cursor.execute(sql, values)
+        check = await cursor.fetchone()
+        return check

funkwhale_network/db.py

@@ -5,14 +5,51 @@ async def get_pool(db_dsn):
     return await aiopg.create_pool(db_dsn)
 
 
-async def create_domain_table(cursor):
+async def create_domains_table(cursor):
     await cursor.execute(
         """CREATE TABLE domains (
-            name varchar(255) PRIMARY KEY
+            name VARCHAR(255) PRIMARY KEY,
+            node_name VARCHAR(255) NULL,
+            private    BOOLEAN              NULL,
+            open_registrations    BOOLEAN              NULL,
+            blocked    BOOLEAN              DEFAULT false,
+            first_seen    TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+            last_successful_check    TIMESTAMP WITH TIME ZONE NULL
         )"""
     )
 
 
+async def create_checks_table(cursor):
+    sql = """
+    CREATE TABLE checks (
+        time        TIMESTAMPTZ       NOT NULL,
+        domain      VARCHAR(255) REFERENCES domains(name),
+        up    BOOLEAN              NOT NULL,
+        https    BOOLEAN              NOT NULL,
+        open_registrations    BOOLEAN              NULL,
+        private    BOOLEAN              NULL,
+        federation_enabled BOOLEAN              NULL,
+        anonymous_can_listen BOOLEAN              NULL,
+        usage_users_total INTEGER NULL,
+        usage_users_active_half_year INTEGER NULL,
+        usage_users_active_month INTEGER NULL,
+        usage_listenings_total INTEGER NULL,
+        library_tracks_total INTEGER NULL,
+        library_albums_total INTEGER NULL,
+        library_artists_total INTEGER NULL,
+        library_music_hours INTEGER NULL,
+        software_name VARCHAR(255) NULL,
+        software_version_major SMALLINT NULL,
+        software_version_minor SMALLINT NULL,
+        software_version_patch SMALLINT NULL,
+        software_prerelease VARCHAR(255) NULL,
+        software_build VARCHAR(255) NULL
+    );
+    SELECT create_hypertable('checks', 'time');
+    """
+    await cursor.execute(sql)
+
+
 async def create(conn):
     async with conn.cursor() as cursor:
         for table, create_handler in TABLES:
@@ -22,7 +59,7 @@ async def create(conn):
 async def clear(conn):
     async with conn.cursor() as cursor:
         for table, _ in TABLES:
-            await cursor.execute("DROP TABLE IF EXISTS {}".format(table))
+            await cursor.execute("DROP TABLE IF EXISTS {} CASCADE".format(table))
 
 
-TABLES = [("domains", create_domain_table)]
+TABLES = [("domains", create_domains_table), ("checks", create_checks_table)]

funkwhale_network/schemas.py

@@ -60,19 +60,37 @@ class StatisticsSchema(marshmallow.Schema):
     total = marshmallow.fields.Integer(required=False)
 
 
+class UsageStatisticsSchema(StatisticsSchema):
+    activeHalfyear = marshmallow.fields.Integer(required=False)
+    activeMonth = marshmallow.fields.Integer(required=False)
+
+
 class UsageSchema(marshmallow.Schema):
-    users = marshmallow.fields.Nested(StatisticsSchema, required=False)
+    users = marshmallow.fields.Nested(UsageStatisticsSchema, required=False)
+
+
+class MusicSchema(marshmallow.Schema):
+    hours = marshmallow.fields.Integer(required=False)
 
 
 class LibraryMetadataSchema(marshmallow.Schema):
     anonymousCanListen = marshmallow.fields.Boolean(required=True)
     federationEnabled = marshmallow.fields.Boolean(required=True)
+    tracks = marshmallow.fields.Nested(StatisticsSchema, required=False)
+    albums = marshmallow.fields.Nested(StatisticsSchema, required=False)
+    artists = marshmallow.fields.Nested(StatisticsSchema, required=False)
+    music = marshmallow.fields.Nested(MusicSchema, required=False)
+
+
+class MetadataUsageSchema(marshmallow.Schema):
+    listenings = marshmallow.fields.Nested(StatisticsSchema, required=False)
 
 
 class MetadataSchema(marshmallow.Schema):
     nodeName = marshmallow.fields.String(required=True)
     private = marshmallow.fields.Boolean(required=True)
-    library = usage = marshmallow.fields.Nested(LibraryMetadataSchema, required=False)
+    library = marshmallow.fields.Nested(LibraryMetadataSchema, required=False)
+    usage = marshmallow.fields.Nested(MetadataUsageSchema, required=False)
 
 
 class NodeInfo2Schema(marshmallow.Schema):

funkwhale_network/serializers.py

@@ -2,7 +2,7 @@ import psycopg2
 
 
 async def create_domain(conn, data):
-    sql = "INSERT INTO domains (name) VALUES (%s) RETURNING *"
+    sql = "INSERT INTO domains (name) VALUES (%s) ON CONFLICT DO NOTHING RETURNING *"
     async with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cursor:
         await cursor.execute(sql, [data["name"]])
         domain = await cursor.fetchone()

tests/conftest.py

 import os
 
 import pytest
+import psycopg2
 import aiohttp
 from aioresponses import aioresponses
 
@@ -43,6 +44,18 @@ async def populated_db(db_pool):
         await db.clear(conn)
 
 
+@pytest.fixture
+async def db_conn(db_pool):
+    async with db_pool.acquire() as conn:
+        yield conn
+
+
+@pytest.fixture
+async def db_cursor(db_conn):
+    async with db_conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cursor:
+        yield cursor
+
+
 @pytest.fixture
 def responses():
     with aioresponses() as m:

tests/test_crawler.py

-from funkwhale_network import crawler
+from funkwhale_network import crawler, serializers
 
 
 async def test_fetch(session, responses):
@@ -25,7 +25,7 @@ def test_validate_data(populated_db):
         "version": "2.0",
         "software": {"name": "funkwhale", "version": "0.18-dev+git.b575999e"},
         "openRegistrations": False,
-        "usage": {"users": {"total": 78}},
+        "usage": {"users": {"total": 78, "activeHalfyear": 42, "activeMonth": 23}},
         "metadata": {
             "private": False,
             "nodeName": "Funkwhale 101",
@@ -56,12 +56,114 @@ def test_validate_data(populated_db):
             },
         },
         "openRegistrations": False,
-        "usage": {"users": {"total": 78}},
+        "usage": {"users": {"total": 78, "activeHalfyear": 42, "activeMonth": 23}},
         "metadata": {
             "private": False,
             "nodeName": "Funkwhale 101",
-            "library": {"federationEnabled": True, "anonymousCanListen": True},
+            "library": {
+                "federationEnabled": True,
+                "anonymousCanListen": True,
+                "tracks": {"total": 98552},
+                "artists": {"total": 9831},
+                "albums": {"total": 10872},
+                "music": {"hours": 7650},
+            },
+            "usage": {"listenings": {"total": 50294}},
         },
     }
     result = crawler.clean_nodeinfo(payload)
     assert result.data == expected
+
+
+async def test_clean_check_result():
+    check = {"up": True, "https": True, "domain": "test.domain"}
+
+    data = {
+        "software": {
+            "name": "funkwhale",
+            "version": {
+                "major": 0,
+                "minor": 18,
+                "patch": 0,
+                "prerelease": "dev",
+                "build": "git.b575999e",
+            },
+        },
+        "openRegistrations": False,
+        "usage": {"users": {"total": 78, "activeHalfyear": 42, "activeMonth": 23}},
+        "metadata": {
+            "private": False,
+            "nodeName": "Funkwhale 101",
+            "library": {
+                "federationEnabled": True,
+                "anonymousCanListen": True,
+                "tracks": {"total": 98552},
+                "artists": {"total": 9831},
+                "albums": {"total": 10872},
+                "music": {"hours": 7650},
+            },
+            "usage": {"listenings": {"total": 50294}},
+        },
+    }
+
+    expected = {
+        "domain": "test.domain",
+        "up": True,
+        "https": True,
+        "open_registrations": False,
+        "federation_enabled": True,
+        "anonymous_can_listen": True,
+        "private": False,
+        "usage_users_total": 78,
+        "usage_users_active_half_year": 42,
+        "usage_users_active_month": 23,
+        "usage_listenings_total": 50294,
+        "library_tracks_total": 98552,
+        "library_albums_total": 10872,
+        "library_artists_total": 9831,
+        "library_music_hours": 7650,
+        "software_name": "funkwhale",
+        "software_version_major": 0,
+        "software_version_minor": 18,
+        "software_version_patch": 0,
+        "software_prerelease": "dev",
+        "software_build": "git.b575999e",
+    }
+
+    assert crawler.clean_check(check, data) == expected
+
+
+async def test_save_check(populated_db, db_cursor, db_conn):
+    await serializers.create_domain(db_conn, {"name": "test.domain"})
+    data = {
+        "domain": "test.domain",
+        "up": True,
+        "https": True,
+        "open_registrations": False,
+        "federation_enabled": True,
+        "anonymous_can_listen": True,
+        "private": False,
+        "usage_users_total": 78,
+        "usage_users_active_half_year": 42,
+        "usage_users_active_month": 23,
+        "usage_listenings_total": 50294,
+        "library_tracks_total": 98552,
+        "library_albums_total": 10872,
+        "library_artists_total": 9831,
+        "library_music_hours": 7650,
+        "software_name": "funkwhale",
+        "software_version_major": 0,
+        "software_version_minor": 18,
+        "software_version_patch": 0,
+        "software_prerelease": "dev",
+        "software_build": "git.b575999e",
+    }
+
+    sql = "SELECT * from checks"
+    result = await crawler.save_check(db_conn, data)
+
+    await db_cursor.execute(sql)
+    row = await db_cursor.fetchone()
+    data["time"] = result["time"]
+    assert data == result
+    assert row == data

tests/test_db.py

@@ -5,7 +5,7 @@ async def test_db_create(db_pool):
     try:
         async with db_pool.acquire() as conn:
             await db.create(conn)
-            tables = ["domains"]
+            tables = ["domains", "checks"]
             async with conn.cursor() as cursor:
                 for t in tables:
                     await cursor.execute("SELECT * from {}".format(t))

tests/test_serializers.py

+from funkwhale_network import serializers
+
+
+async def test_create_domain_ignore_duplicate(populated_db, db_conn):
+    r1 = await serializers.create_domain(db_conn, {"name": "test.domain"})
+    r2 = await serializers.create_domain(db_conn, {"name": "test.domain"})
+
+    assert r1 != r2
+    assert r2 is None