diff --git a/adapter.py b/adapter.py index 9240c10..8d00758 100644 --- a/adapter.py +++ b/adapter.py @@ -1,6 +1,6 @@ from requests.adapters import HTTPAdapter -class WhyTheFuckRequestsHasNoTimeoutInAdapter(HTTPAdapter): +class TimeoutHTTPAdapter(HTTPAdapter): def __init__(self, *args, **kwargs): if "timeout" in kwargs: self.timeout = kwargs["timeout"] diff --git a/scraper.py b/scraper.py index d33e9d3..fb9bb16 100644 --- a/scraper.py +++ b/scraper.py @@ -18,7 +18,7 @@ from requests.cookies import create_cookie from bs4 import BeautifulSoup, Tag from cgi import parse_header -from adapter import WhyTheFuckRequestsHasNoTimeoutInAdapter +from adapter import TimeoutHTTPAdapter @dataclass @@ -161,7 +161,7 @@ class Scraper(Session): scraper = Scraper(concurrency=5) retries = Retry(total=5, backoff_factor=0.1, status_forcelist=[404]) -scraper.mount('https://', WhyTheFuckRequestsHasNoTimeoutInAdapter(timeout=1, max_retries=retries)) +scraper.mount('https://', TimeoutHTTPAdapter(timeout=1, max_retries=retries)) document_path = Path('archives') attachment_path = document_path / 'attachments'