23 lines
482 B
Python
23 lines
482 B
Python
import asyncio
|
|
|
|
from pathlib import Path
|
|
|
|
from utils.middlewares import SemaphoreMiddleware
|
|
from utils.scraper import Scraper
|
|
|
|
|
|
async def main():
|
|
middlewares = (
|
|
SemaphoreMiddleware(5),
|
|
)
|
|
|
|
async with Scraper(middlewares=middlewares) as scraper:
|
|
posts = await scraper.list('roh', 'person')
|
|
|
|
for future in asyncio.as_completed([scraper.view(p) for p in posts]):
|
|
await future
|
|
|
|
|
|
if __name__ == '__main__':
|
|
asyncio.run(main())
|