boardwise
/
coach-scraper
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix classname.

pull/2/head
Joshua Potter 2023-11-27 13:41:06 -07:00
parent 23aa622126
commit 659988355c
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
main.py
View File

@ -23,7 +23,7 @@ async def scrape_coach_links(page_no):
return return
html = await response.text() html = await response.text()
soup = BeautifulSoup(html, "html.parser") soup = BeautifulSoup(html, "html.parser")
members = soup.find_all("a", class_="members-categories-blahblah") members = soup.find_all("a", class_="members-categories-username")
for member in members: for member in members:
links.append(member.get("href")) links.append(member.get("href"))
@ -37,10 +37,10 @@ async def scrape_all_coach_links(max_pages=64):
filepath = DATA_COACH_LIST.format(i) filepath = DATA_COACH_LIST.format(i)
if os.path.isfile(filepath): if os.path.isfile(filepath):
with open(filepath, "r") as f: with open(filepath, "r") as f:
links = f.readlines() links.extend(f.readlines())
print(f"{filepath} already exists.") print(f"{filepath} already exists.")
else: else:
links = await scrape_coach_links(i) links.extend(await scrape_coach_links(i))
with open(filepath, "w") as f: with open(filepath, "w") as f:
for link in links: for link in links:
f.write(f"{link}\n") f.write(f"{link}\n")