feat: fetching attachment
This commit is contained in:
16
scraper.py
16
scraper.py
@@ -1,21 +1,31 @@
|
||||
import json
|
||||
import asyncio
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from utils.middlewares import SemaphoreMiddleware
|
||||
from utils.scraper import Scraper
|
||||
from models.post import Post
|
||||
|
||||
|
||||
archive_dir = Path('archives')
|
||||
|
||||
async def main():
|
||||
middlewares = (
|
||||
SemaphoreMiddleware(5),
|
||||
)
|
||||
|
||||
async with Scraper(middlewares=middlewares) as scraper:
|
||||
posts = await scraper.list('roh', 'person')
|
||||
post = Post(
|
||||
id=2341247,
|
||||
boardId='event_voicere',
|
||||
boardPath='board'
|
||||
)
|
||||
|
||||
await scraper.view(post)
|
||||
await scraper.download_attachments(post, archive_dir)
|
||||
|
||||
for future in asyncio.as_completed([scraper.view(p) for p in posts]):
|
||||
await future
|
||||
print(post)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
Reference in New Issue
Block a user