37 lines
1.1 KiB
Python
37 lines
1.1 KiB
Python
from pathlib import Path
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
|
from requests.adapters import Retry
|
|
|
|
from adapter import TimeoutHTTPAdapter
|
|
from scraper import Scraper
|
|
|
|
|
|
scraper = Scraper(concurrency=5)
|
|
|
|
retries = Retry(total=5, backoff_factor=0.1, status_forcelist=[404])
|
|
scraper.mount('https://', TimeoutHTTPAdapter(timeout=1, max_retries=retries))
|
|
|
|
document_path = Path('archives')
|
|
attachment_path = document_path / 'attachments'
|
|
attachment_path.mkdir(parents=True, exist_ok=True)
|
|
|
|
try:
|
|
for offset in range(22760681, 22760681 + (10000 * 100), 10000):
|
|
with ThreadPoolExecutor() as executor:
|
|
posts = scraper.search('aoegame', 'name', -offset, 'ori')
|
|
print(f'searching offset {-offset}, found {len(posts)} post(s)')
|
|
as_completed([
|
|
executor.submit(
|
|
scraper.view,
|
|
post,
|
|
document_path=document_path,
|
|
attachment_path=attachment_path
|
|
)
|
|
for post in scraper.search('aoegame', 'name', -offset, 'ori')
|
|
])
|
|
|
|
|
|
except KeyboardInterrupt:
|
|
print(':-)')
|