aeon/mandu-crawler
1
0
Fork 0
Code Activity

fix: remove hardcoded integer

Browse Source
This commit is contained in:
Sangha Lee
2025-08-04 13:31:48 +09:00
parent 69a5fd2e97
commit f3344c57c3
1 changed files with 7 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
utils/scraper.py
View File

@@ -18,6 +18,9 @@ from .typings import BoardPath, SearchType
from models import Attachment, Post from models import Attachment, Post
LIST_MAX_POSTS = 200
class Scraper(ClientSession): class Scraper(ClientSession):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
@@ -29,7 +32,7 @@ class Scraper(ClientSession):
# 게시글 목록 조회로 한 번에 불러올 항목 수 # 게시글 목록 조회로 한 번에 불러올 항목 수
self.cookie_jar.update_cookies({ self.cookie_jar.update_cookies({
'list_count': '200' 'list_count': LIST_MAX_POSTS
}) })
async def __aenter__(self) -> 'Scraper': async def __aenter__(self) -> 'Scraper':
@@ -84,7 +87,7 @@ class Scraper(ClientSession):
for tag in document.select('.gall-detail-lnktb') for tag in document.select('.gall-detail-lnktb')
] ]
async def view(self, post: Post): async def view(self, post: Post) -> Post:
""" """
게시글 내용을 조회합니다 게시글 내용을 조회합니다
@@ -137,8 +140,8 @@ class Scraper(ClientSession):
# 불필요한 본문 요소 제거 # 불필요한 본문 요소 제거
for tag in post.body.select('script, style'): for tag in post.body.select('script, style'):
tag.extract() tag.extract()
print(f'{post.board_id}/{post.id}: {post.title}') return post
async def fetch_voice(self, id: str): async def fetch_voice(self, id: str):
""" """