Can now retrieve downloads count when crawling

funkwhale_network/cli.py

@@ -229,6 +229,7 @@ async def crawl(domain, use_public, detail, passes, sort):
                 "Active users (30d)",
                 "Users",
                 "Listenings",
+                "Downloads",
                 "Open registrations",
                 "Anonymous access",
                 "Private",
@@ -256,6 +257,7 @@ async def crawl(domain, use_public, detail, passes, sort):
                 "Active users (180d)",
                 "Users",
                 "Listenings",
+                "Downloads",
                 "Tracks",
                 "Albums",
                 "Artists",
@@ -282,6 +284,7 @@ def aggregate_crawl_results(domains_info):
         "usage_users_active_half_year": permissive_sum,
         "usage_users_active_month": permissive_sum,
         "usage_listenings_total": permissive_sum,
+        "usage_downloads_total": permissive_sum,
         "library_tracks_total": permissive_sum,
         "library_albums_total": permissive_sum,
         "library_artists_total": permissive_sum,

funkwhale_network/crawler.py

@@ -169,6 +169,9 @@ def clean_check(check_data, nodeinfo_data):
         "usage_listenings_total": recursive_getattr(
             nodeinfo_data, "metadata.usage.listenings.total"
         ),
+        "usage_downloads_total": recursive_getattr(
+            nodeinfo_data, "metadata.usage.downloads.total"
+        ),
         "library_tracks_total": recursive_getattr(
             nodeinfo_data, "metadata.library.tracks.total"
         ),

funkwhale_network/db.py

@@ -8,18 +8,20 @@ async def get_pool(db_dsn):
 
 async def create_domains_table(cursor):
     await cursor.execute(
-        """CREATE TABLE domains (
+        """
+        CREATE TABLE IF NOT EXISTS domains (
             name VARCHAR(255) PRIMARY KEY,
             node_name VARCHAR(255) NULL,
             blocked    BOOLEAN              DEFAULT false,
             first_seen    TIMESTAMP WITH TIME ZONE DEFAULT NOW()
-        )"""
+        );
+        """
     )
 
 
 async def create_checks_table(cursor):
     sql = """
-    CREATE TABLE checks (
+    CREATE TABLE IF NOT EXISTS checks (
         time        TIMESTAMPTZ       NOT NULL,
         domain      VARCHAR(255) REFERENCES domains(name),
         up    BOOLEAN              NOT NULL,
@@ -42,7 +44,8 @@ async def create_checks_table(cursor):
         software_prerelease VARCHAR(255) NULL,
         software_build VARCHAR(255) NULL
     );
-    SELECT create_hypertable('checks', 'time');
+    ALTER TABLE checks ADD COLUMN IF NOT EXISTS usage_downloads_total INTEGER NULL;
+    SELECT create_hypertable('checks', 'time', if_not_exists => TRUE);
     """
     await cursor.execute(sql)
 
@@ -94,6 +97,7 @@ async def get_stats(conn):
         "albums": {"total": 0},
         "tracks": {"total": 0},
         "listenings": {"total": 0},
+        "downloads": {"total": 0},
     }
     for check in checks:
         increment_stat(data["users"], "total", check["usage_users_total"])
@@ -114,6 +118,7 @@ async def get_stats(conn):
         increment_stat(
             data["listenings"], "total", int(check["usage_listenings_total"])
         )
+        increment_stat(data["downloads"], "total", int(check["usage_downloads_total"]))
     return data
 
 

funkwhale_network/output.py

@@ -7,6 +7,7 @@ FIELDS = {
         "Active users (180d)": {"field": "usage_users_active_half_year"},
         "Users": {"field": "usage_users_total"},
         "Listenings": {"field": "usage_listenings_total"},
+        "Downloads": {"field": "usage_downloads_total"},
         "Tracks": {"field": "library_tracks_total"},
         "Albums": {"field": "library_albums_total"},
         "Artists": {"field": "library_artists_total"},
@@ -22,6 +23,7 @@ FIELDS = {
         "Active users (180d)": {"field": "usage_users_active_half_year"},
         "Users": {"field": "usage_users_total"},
         "Listenings": {"field": "usage_listenings_total"},
+        "Downloads": {"field": "usage_download_total"},
         "Tracks": {"field": "library_tracks_total"},
         "Albums": {"field": "library_albums_total"},
         "Artists": {"field": "library_artists_total"},

funkwhale_network/schemas.py

@@ -102,6 +102,7 @@ class LibraryMetadataSchema(marshmallow.Schema):
 
 class MetadataUsageSchema(marshmallow.Schema):
     listenings = marshmallow.fields.Nested(StatisticsSchema, required=False)
+    downloads = marshmallow.fields.Nested(StatisticsSchema, required=False)
 
     class Meta:
         unknown = marshmallow.EXCLUDE

funkwhale_network/serializers.py

@@ -26,6 +26,7 @@ def serialize_check(raw):
                 "activeMonth": raw["usage_users_active_month"],
             },
             "listenings": {"total": raw["usage_listenings_total"]},
+            "downloads": {"total": raw["usage_downloads_total"]},
         },
         "library": {
             "music": {"hours": raw["library_music_hours"]},

tests/factories.py

@@ -58,6 +58,7 @@ class CheckFactory(DBFactory):
     usage_users_active_half_year = factory.Faker("random_int")
     usage_users_active_month = factory.Faker("random_int")
     usage_listenings_total = factory.Faker("random_int")
+    usage_downloads_total = factory.Faker("random_int")
     library_tracks_total = factory.Faker("random_int")
     library_albums_total = factory.Faker("random_int")
     library_artists_total = factory.Faker("random_int")

tests/test_crawler.py

@@ -147,7 +147,7 @@ async def test_clean_check_result():
                 "albums": {"total": 10872},
                 "music": {"hours": 7650},
             },
-            "usage": {"listenings": {"total": 50294}},
+            "usage": {"listenings": {"total": 50294}, "downloads": {"total": 91273}},
         },
     }
 
@@ -163,6 +163,7 @@ async def test_clean_check_result():
         "usage_users_active_half_year": 42,
         "usage_users_active_month": 23,
         "usage_listenings_total": 50294,
+        "usage_downloads_total": 91273,
         "library_tracks_total": 98552,
         "library_albums_total": 10872,
         "library_artists_total": 9831,
@@ -193,6 +194,7 @@ async def test_save_check(populated_db, db_conn, factories):
         "usage_users_active_half_year": 42,
         "usage_users_active_month": 23,
         "usage_listenings_total": 50294,
+        "usage_downloads_total": 7092,
         "library_tracks_total": 98552,
         "library_albums_total": 10872,
         "library_artists_total": 9831,

tests/test_db.py

@@ -46,6 +46,7 @@ async def test_get_stats(factories, db_conn):
         usage_users_active_half_year=1,
         usage_users_active_month=2,
         usage_listenings_total=20,
+        usage_downloads_total=30,
         library_tracks_total=6,
         library_albums_total=30,
         library_artists_total=36,
@@ -59,6 +60,7 @@ async def test_get_stats(factories, db_conn):
         usage_users_active_half_year=3,
         usage_users_active_month=1,
         usage_listenings_total=22,
+        usage_downloads_total=33,
         library_tracks_total=15,
         library_albums_total=13,
         library_artists_total=40,
@@ -71,5 +73,6 @@ async def test_get_stats(factories, db_conn):
         "albums": {"total": 43},
         "tracks": {"total": 21},
         "listenings": {"total": 42},
+        "downloads": {"total": 63},
     }
     assert await db.get_stats(db_conn) == expected

tests/test_serializers.py

@@ -23,6 +23,7 @@ def test_serialize_check():
         "usage_users_active_half_year": 2,
         "usage_users_active_month": 3,
         "usage_listenings_total": 4,
+        "usage_downloads_total": 44,
         "library_tracks_total": 5,
         "library_albums_total": 6,
         "library_artists_total": 7,
@@ -52,6 +53,7 @@ def test_serialize_check():
         "usage": {
             "users": {"total": 1, "activeHalfyear": 2, "activeMonth": 3},
             "listenings": {"total": 4},
+            "downloads": {"total": 44},
         },
         "software": {
             "name": "funkwhale",