Initial commit.
commit
1710e1aefa
|
|
@ -0,0 +1,7 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
if command -v git > /dev/null && on_git_branch; then
|
||||
git config --local core.hooksPath .githooks/
|
||||
fi
|
||||
|
||||
use flake
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
#!/usr/bin/env bash
|
||||
set -e
|
||||
|
||||
filesToFormat=$(
|
||||
git --no-pager diff --name-status --no-color --cached | \
|
||||
awk '$1 != "D" && $2 ~ /\.py$/ {print $NF}'
|
||||
)
|
||||
|
||||
for path in $filesToFormat
|
||||
do
|
||||
black --quiet "$path"
|
||||
git add "$path"
|
||||
done;
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
# Directory used by `direnv` to hold `use flake`-generated profiles.
|
||||
.direnv/
|
||||
|
||||
# A symlink produced by default when running `nix build`.
|
||||
result
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
# Poetry Flake Template
|
||||
|
||||
This is a template for constructing a working environment for Python (version
|
||||
3.11.6) development. Packaging and dependency management relies on [poetry](https://python-poetry.org/)
|
||||
(version 1.7.0). [direnv](https://direnv.net/) can be used to a launch a dev
|
||||
shell upon entering this directory (refer to `.envrc`). Otherwise run via:
|
||||
```bash
|
||||
$> nix develop
|
||||
```
|
||||
|
||||
## Language Server
|
||||
|
||||
The [python-lsp-server](https://github.com/python-lsp/python-lsp-server)
|
||||
(version v1.9.0) is included in this flake, along with the [python-lsp-black](https://github.com/python-lsp/python-lsp-black)
|
||||
plugin for formatting purposes. `pylsp` is expected to be configured to use
|
||||
[McCabe](https://github.com/PyCQA/mccabe), [pycodestyle](https://pycodestyle.pycqa.org/en/latest/),
|
||||
and [pyflakes](https://github.com/PyCQA/pyflakes). Refer to your editor for
|
||||
configuration details.
|
||||
|
||||
## Formatting
|
||||
|
||||
Formatting depends on the [black](https://black.readthedocs.io/en/stable/index.html)
|
||||
(version 23.9.1) tool. A `pre-commit` hook is included in `.githooks` that can
|
||||
be used to format all `*.py` files prior to commit. Install via:
|
||||
```bash
|
||||
$> git config --local core.hooksPath .githooks/
|
||||
```
|
||||
If running [direnv](https://direnv.net/), this hook is installed automatically
|
||||
when entering the directory.
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
(import
|
||||
(
|
||||
let lock = builtins.fromJSON (builtins.readFile ./flake.lock); in
|
||||
fetchTarball {
|
||||
url = lock.nodes.flake-compat.locked.url or "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz";
|
||||
sha256 = lock.nodes.flake-compat.locked.narHash;
|
||||
}
|
||||
)
|
||||
{ src = ./.; }
|
||||
).defaultNix
|
||||
|
|
@ -0,0 +1,190 @@
|
|||
{
|
||||
"nodes": {
|
||||
"flake-compat": {
|
||||
"locked": {
|
||||
"lastModified": 1696426674,
|
||||
"narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
|
||||
"rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
|
||||
"revCount": 57,
|
||||
"type": "tarball",
|
||||
"url": "https://api.flakehub.com/f/pinned/edolstra/flake-compat/1.0.1/018afb31-abd1-7bff-a5e4-cff7e18efb7a/source.tar.gz"
|
||||
},
|
||||
"original": {
|
||||
"type": "tarball",
|
||||
"url": "https://flakehub.com/f/edolstra/flake-compat/1.tar.gz"
|
||||
}
|
||||
},
|
||||
"flake-utils": {
|
||||
"inputs": {
|
||||
"systems": "systems"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1694529238,
|
||||
"narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "ff7b65b44d01cf9ba6a71320833626af21126384",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-utils_2": {
|
||||
"inputs": {
|
||||
"systems": "systems_2"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1694529238,
|
||||
"narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "ff7b65b44d01cf9ba6a71320833626af21126384",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nix-github-actions": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"poetry2nix",
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1698974481,
|
||||
"narHash": "sha256-yPncV9Ohdz1zPZxYHQf47S8S0VrnhV7nNhCawY46hDA=",
|
||||
"owner": "nix-community",
|
||||
"repo": "nix-github-actions",
|
||||
"rev": "4bb5e752616262457bc7ca5882192a564c0472d2",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-community",
|
||||
"repo": "nix-github-actions",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1700794826,
|
||||
"narHash": "sha256-RyJTnTNKhO0yqRpDISk03I/4A67/dp96YRxc86YOPgU=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "5a09cb4b393d58f9ed0d9ca1555016a8543c2ac8",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "nixos-unstable",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"poetry2nix": {
|
||||
"inputs": {
|
||||
"flake-utils": "flake-utils_2",
|
||||
"nix-github-actions": "nix-github-actions",
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
],
|
||||
"systems": "systems_3",
|
||||
"treefmt-nix": "treefmt-nix"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1701035916,
|
||||
"narHash": "sha256-Cbe/3H9/z7vIXAeMr9m9iXs1gRxpE3w1mrx9aaxWtGU=",
|
||||
"owner": "nix-community",
|
||||
"repo": "poetry2nix",
|
||||
"rev": "2bee2516bc054458b0cbca10b18e2ec63cea8726",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-community",
|
||||
"repo": "poetry2nix",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"flake-compat": "flake-compat",
|
||||
"flake-utils": "flake-utils",
|
||||
"nixpkgs": "nixpkgs",
|
||||
"poetry2nix": "poetry2nix"
|
||||
}
|
||||
},
|
||||
"systems": {
|
||||
"locked": {
|
||||
"lastModified": 1681028828,
|
||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"systems_2": {
|
||||
"locked": {
|
||||
"lastModified": 1681028828,
|
||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"systems_3": {
|
||||
"locked": {
|
||||
"lastModified": 1681028828,
|
||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"id": "systems",
|
||||
"type": "indirect"
|
||||
}
|
||||
},
|
||||
"treefmt-nix": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"poetry2nix",
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1699786194,
|
||||
"narHash": "sha256-3h3EH1FXQkIeAuzaWB+nK0XK54uSD46pp+dMD3gAcB4=",
|
||||
"owner": "numtide",
|
||||
"repo": "treefmt-nix",
|
||||
"rev": "e82f32aa7f06bbbd56d7b12186d555223dc399d1",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "numtide",
|
||||
"repo": "treefmt-nix",
|
||||
"type": "github"
|
||||
}
|
||||
}
|
||||
},
|
||||
"root": "root",
|
||||
"version": 7
|
||||
}
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
{
|
||||
description = ''
|
||||
An opinionated poetry flake.
|
||||
|
||||
This flake has been adapted from the `app` template found in:
|
||||
https://github.com/nix-community/poetry2nix
|
||||
|
||||
To generate a copy of this template elsewhere, run:
|
||||
$> bootstrap poetry
|
||||
'';
|
||||
|
||||
inputs = {
|
||||
flake-compat.url = "https://flakehub.com/f/edolstra/flake-compat/1.tar.gz";
|
||||
flake-utils.url = "github:numtide/flake-utils";
|
||||
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
|
||||
poetry2nix = {
|
||||
url = "github:nix-community/poetry2nix";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
};
|
||||
|
||||
outputs = { self, nixpkgs, flake-utils, poetry2nix, ... }:
|
||||
flake-utils.lib.eachDefaultSystem (system:
|
||||
let
|
||||
# See https://github.com/nix-community/poetry2nix/tree/master#api for
|
||||
# more functions and examples.
|
||||
pkgs = nixpkgs.legacyPackages.${system};
|
||||
|
||||
inherit
|
||||
(poetry2nix.lib.mkPoetry2Nix { inherit pkgs; })
|
||||
mkPoetryApplication
|
||||
defaultPoetryOverrides;
|
||||
|
||||
# https://github.com/nix-community/poetry2nix/blob/ec4364021900f8e0d425d901b6e6ff03cf201efb/docs/edgecases.md
|
||||
# `poetry2nix`, by default, prefers building from source. To build
|
||||
# certain dependencies, we need to augment its build dependencies by
|
||||
# adding the corresponding build backend (e.g. `setuptools`).
|
||||
#
|
||||
# For example, you can write:
|
||||
# ```nix
|
||||
# pypkgs-build-requirements = {
|
||||
# ...
|
||||
# coolname = [ "setuptools" ];
|
||||
# ...
|
||||
# };
|
||||
# ```
|
||||
# after encountering a build error like:
|
||||
#
|
||||
# > ModuleNotFoundError: No module named 'setuptools'
|
||||
pypkgs-build-requirements = {};
|
||||
poetry2nix-overrides = defaultPoetryOverrides.extend (self: super:
|
||||
builtins.mapAttrs (package: build-requirements:
|
||||
(builtins.getAttr package super).overridePythonAttrs (old: {
|
||||
buildInputs =
|
||||
(old.buildInputs or []) ++
|
||||
(builtins.map (pkg:
|
||||
if builtins.isString pkg then
|
||||
builtins.getAttr pkg super
|
||||
else
|
||||
pkg) build-requirements);
|
||||
})
|
||||
) pypkgs-build-requirements
|
||||
);
|
||||
in
|
||||
{
|
||||
packages = {
|
||||
tmpl-app = mkPoetryApplication {
|
||||
projectDir = ./.;
|
||||
overrides = poetry2nix-overrides;
|
||||
};
|
||||
default = self.packages.${system}.tmpl-app;
|
||||
};
|
||||
|
||||
devShells.default = pkgs.mkShell {
|
||||
inputsFrom = [ self.packages.${system}.tmpl-app ];
|
||||
packages = [
|
||||
pkgs.poetry
|
||||
] ++ (with pkgs.python311Packages; [
|
||||
black
|
||||
debugpy
|
||||
mccabe
|
||||
pycodestyle
|
||||
pyflakes
|
||||
python-lsp-black
|
||||
python-lsp-server
|
||||
]);
|
||||
};
|
||||
});
|
||||
}
|
||||
|
|
@ -0,0 +1,99 @@
|
|||
import aiohttp
|
||||
import asyncio
|
||||
import os
|
||||
import os.path
|
||||
import random
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
|
||||
# References to paths we use to save any scraped content.
|
||||
DATA_COACH_LINKS = "data/coach_links.txt"
|
||||
DATA_COACH_DIR = "data/coach/{}/{}"
|
||||
|
||||
|
||||
async def scrape_coach_links(page_no):
|
||||
"""Scrape a single coach page listing."""
|
||||
links = []
|
||||
async with aiohttp.ClientSession() as session:
|
||||
href = f"https://www.chess.com/coaches?sortBy=alphabetical&page={page_no}"
|
||||
async with session.get(href) as response:
|
||||
if response.status != 200:
|
||||
print(f"Encountered {response.status} when retrieving {href}")
|
||||
return
|
||||
html = await response.text()
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
members = soup.find_all("a", class_="members-categories-username")
|
||||
for member in members:
|
||||
links.append(member.get("href"))
|
||||
|
||||
return links
|
||||
|
||||
|
||||
async def scrape_all_coach_links(max_pages=62):
|
||||
"""Scans through chess.com/coaches for all member links."""
|
||||
if os.path.isfile(DATA_COACH_LINKS):
|
||||
with open(DATA_COACH_LINKS, "r") as f:
|
||||
return f.readlines()
|
||||
for i in range(1, max_pages + 1):
|
||||
# Nest the file context manager here so I can `tail -f` the file.
|
||||
with open(DATA_COACH_LINKS, "a") as f:
|
||||
links = await scrape_coach_links(i)
|
||||
for link in links:
|
||||
f.write(f"{link}\n")
|
||||
await asyncio.sleep(random.randint(2, 5))
|
||||
return links
|
||||
|
||||
|
||||
async def download_member_info(member_name, filename, href):
|
||||
"""Download member-specific content.
|
||||
|
||||
@return: True if we downloaded content. False if the results already
|
||||
existed locally.
|
||||
"""
|
||||
target = DATA_COACH_DIR.format(member_name, filename)
|
||||
if os.path.isfile(target):
|
||||
return False
|
||||
with open(target, "w") as f:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(href) as response:
|
||||
if response.status != 200:
|
||||
print(f"Encountered {response.status} when retrieving {href}")
|
||||
return
|
||||
f.write(await response.text())
|
||||
return True
|
||||
|
||||
|
||||
async def main():
|
||||
links = await scrape_all_coach_links()
|
||||
for link in links:
|
||||
href = link.strip()
|
||||
member_name = href[len("https://www.chess.com/member/") :]
|
||||
downloaded = await asyncio.gather(
|
||||
download_member_info(
|
||||
member_name,
|
||||
f"{member_name}.html",
|
||||
href,
|
||||
),
|
||||
download_member_info(
|
||||
member_name,
|
||||
"activity.json",
|
||||
f"https://www.chess.com/callback/member/activity/{member_name}?page=1",
|
||||
),
|
||||
download_member_info(
|
||||
member_name,
|
||||
"stats.json",
|
||||
f"https://www.chess.com/callback/member/stats/{member_name}",
|
||||
),
|
||||
)
|
||||
# Only want to sleep if the files didn't already exist.
|
||||
if any(downloaded):
|
||||
await asyncio.sleep(random.randint(2, 5))
|
||||
print(f"Downloaded {member_name}")
|
||||
else:
|
||||
print(f"Skipping {member_name}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
os.makedirs("data/coach", exist_ok=True)
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand.
|
||||
package = []
|
||||
|
||||
[metadata]
|
||||
lock-version = "2.0"
|
||||
python-versions = "^3.11"
|
||||
content-hash = "81b2fa642d7f2d1219cf80112ace12d689d053d81be7f7addb98144d56fc0fb2"
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
[tool.poetry]
|
||||
name = "chesscom-scraper"
|
||||
version = "0.1.0"
|
||||
description = ""
|
||||
authors = ["jrpotter2112@gmail.com"]
|
||||
readme = "README.md"
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = "^3.11"
|
||||
beautifulsoup4 = "^4.12.2"
|
||||
aiohttp = "^3.8.6"
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry-core"]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
Loading…
Reference in New Issue