aeon/mandu-crawler
1
0
Fork 0
Code Activity

rename to sane class name

Browse Source
This commit is contained in:
Sangha Lee
2025-08-04 08:16:44 +09:00
parent 920ecae2cd
commit 6dbc4a4e54
2 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
adapter.py
View File

@@ -1,6 +1,6 @@
from requests.adapters import HTTPAdapter from requests.adapters import HTTPAdapter
class WhyTheFuckRequestsHasNoTimeoutInAdapter(HTTPAdapter): class TimeoutHTTPAdapter(HTTPAdapter):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
if "timeout" in kwargs: if "timeout" in kwargs:
self.timeout = kwargs["timeout"] self.timeout = kwargs["timeout"]

4
scraper.py
View File

@@ -18,7 +18,7 @@ from requests.cookies import create_cookie
from bs4 import BeautifulSoup, Tag from bs4 import BeautifulSoup, Tag
from cgi import parse_header from cgi import parse_header
from adapter import WhyTheFuckRequestsHasNoTimeoutInAdapter from adapter import TimeoutHTTPAdapter
@dataclass @dataclass
@@ -161,7 +161,7 @@ class Scraper(Session):
scraper = Scraper(concurrency=5) scraper = Scraper(concurrency=5)
retries = Retry(total=5, backoff_factor=0.1, status_forcelist=[404]) retries = Retry(total=5, backoff_factor=0.1, status_forcelist=[404])
scraper.mount('https://', WhyTheFuckRequestsHasNoTimeoutInAdapter(timeout=1, max_retries=retries)) scraper.mount('https://', TimeoutHTTPAdapter(timeout=1, max_retries=retries))
document_path = Path('archives') document_path = Path('archives')
attachment_path = document_path / 'attachments' attachment_path = document_path / 'attachments'