boardwise
/
coach-scraper
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add logging around failed network requests.

main
Joshua Potter 2023-12-09 15:39:56 -07:00
parent 811cf997d1
commit 3ab4f893b7
1 changed files with 2 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
coach_scraper/pipeline.py
View File

@ -1,4 +1,5 @@
import asyncio import asyncio
import logging
import os.path import os.path
from typing import Any, List, Tuple from typing import Any, List, Tuple
@ -58,6 +59,7 @@ class Fetcher:
async with self.session.get(url) as response: async with self.session.get(url) as response:
if response.status == 200: if response.status == 200:
return await response.text(), 200 return await response.text(), 200
logging.error(f"Could not fetch URL {url}. Status code: {response.status}")
return None, response.status return None, response.status
async def scrape_usernames(self, page_no: int) -> List[str] | None: async def scrape_usernames(self, page_no: int) -> List[str] | None: