Skip to content
Snippets Groups Projects
Verified Commit 9f4b1634 authored by Georg Krause's avatar Georg Krause
Browse files

Schedule a job for each domain instead of running all in one

parent 23108a31
No related branches found
No related tags found
No related merge requests found
......@@ -152,22 +152,22 @@ async def launch_domain_poll(session, domain):
@cli.command()
@click.argument("domain", type=str, nargs=-1)
@click.argument("domains", type=str, nargs=-1)
@async_command
async def poll(domain):
async def poll(domains):
"""
Retrieve and store data for the specified domains.
"""
from . import crawler, worker
from . import crawler
if not domain:
click.echo("Polling all domains…")
crawler = worker.Crawler()
return await crawler.poll_all()
if not domains:
async with DB() as db:
domains_db = await db.get_all_domains()
domains = [d["name"] for d in domains_db]
kwargs = crawler.get_session_kwargs()
async with aiohttp.ClientSession(**kwargs) as session:
tasks = [launch_domain_poll(session, d) for d in domain]
tasks = [launch_domain_poll(session, d) for d in domains]
return await asyncio.wait(tasks)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment