60 lines
1.4 KiB
Python
60 lines
1.4 KiB
Python
import aiohttp
|
|
import argparse
|
|
import asyncio
|
|
import json
|
|
|
|
from app.chesscom import (
|
|
Exporter as ChesscomExporter,
|
|
Scraper as ChesscomScraper,
|
|
)
|
|
from app.lichess import (
|
|
Exporter as LichessExporter,
|
|
Scraper as LichessScraper,
|
|
)
|
|
from app.repo import Site
|
|
|
|
|
|
async def run():
|
|
parser = argparse.ArgumentParser(
|
|
prog="coach-scraper",
|
|
description="Scraping/exporting of chess coaches.",
|
|
)
|
|
parser.add_argument("-u", "--user-agent", required=True)
|
|
parser.add_argument(
|
|
"-s",
|
|
"--site",
|
|
required=True,
|
|
choices=[
|
|
Site.CHESSCOM.value,
|
|
Site.LICHESS.value,
|
|
],
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
async with aiohttp.ClientSession(
|
|
headers={"User-Agent": f"BoardWise coach-scraper ({args.user_agent})"}
|
|
) as session:
|
|
if args.site == Site.CHESSCOM.value:
|
|
scraper = ChesscomScraper(session)
|
|
exporter_cls = ChesscomExporter
|
|
elif args.site == Site.LICHESS.value:
|
|
scraper = LichessScraper(session)
|
|
exporter_cls = LichessExporter
|
|
|
|
dump = {}
|
|
|
|
usernames = await scraper.scrape()
|
|
for username in usernames:
|
|
dump[username] = exporter_cls(username).export()
|
|
|
|
with open(scraper.path_site_file("export.json"), "w") as f:
|
|
json.dump(dump, f, indent=2)
|
|
|
|
|
|
def main():
|
|
asyncio.run(run())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|