worker.py

import aiohttp

from funkwhale_network import crawler
from funkwhale_network import db
from funkwhale_network import settings

from aiohttp import ClientSession
from arq import cron


async def poll(ctx, domain):
    session: ClientSession = ctx["session"]
    pool = await db.get_pool(settings.DB_DSN)
    async with pool as conn:
        return await crawler.check(conn=conn, session=session, domain=domain)


async def update_all(ctx):
    pool = await db.get_pool(settings.DB_DSN)
    for check in await db.get_latest_check_by_domain(pool):
        await poll(ctx, check["domain"])


class WorkerSettings:
    cron_jobs = [cron(update_all, minute=None)]
    max_concurrent_tasks = 20
    shutdown_delay = 5
    timeout_seconds = 15