refactor: vwts_scraper — proper Python package structure
- Split single file into 8 focused modules (399 lines total): config.py — constants fetch.py — HTTP GET with retries paginate.py — page_count + HEAD verification parse.py — HTML parsing (topics, posts, tags) state.py — per-section state.json output.py — JSONL with file rotation run.py — main orchestration loop __main__.py — CLI entry point - Run: python -m vwts_scraper <URL> - Page count method fully tested (13 sections, 100+ topics, 0 errors) - Resume support (Ctrl+C → state saved) - Per-section state files (no conflicts between sections)
This commit is contained in:
@@ -0,0 +1,47 @@
|
||||
"""Парсинг HTML: темы, посты, теги."""
|
||||
|
||||
import re
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
|
||||
def parse_topics(soup: BeautifulSoup) -> list:
|
||||
"""Список тем со страницы раздела."""
|
||||
items = []
|
||||
for div in soup.select("div.structItem"):
|
||||
a = div.select_one("div.structItem-title a")
|
||||
if not a:
|
||||
continue
|
||||
m = re.search(r"/topic/(\d+)/", a.get("href", ""))
|
||||
if not m:
|
||||
continue
|
||||
items.append({
|
||||
"id": int(m.group(1)),
|
||||
"title": a.text.strip(),
|
||||
"url": f"https://vwts.ru/forum/topic/{m.group(1)}/"
|
||||
})
|
||||
return items
|
||||
|
||||
|
||||
def parse_posts(soup: BeautifulSoup) -> list:
|
||||
"""Посты с одной страницы темы."""
|
||||
out = []
|
||||
for art in soup.select("article.message"):
|
||||
aname = art.select_one("h4.message-name")
|
||||
author = aname.get_text(strip=True) if aname else "?"
|
||||
t = art.select_one("time.u-dt")
|
||||
date = t.get("datetime", "") if t else ""
|
||||
n = art.select_one("ul.message-attribution-opposite a")
|
||||
num = n.text.strip() if n else ""
|
||||
content = art.select_one("div.message-content")
|
||||
text = ""
|
||||
if content:
|
||||
for h in content.select("div[style*='none'], span[style*='none'], details"):
|
||||
h.decompose()
|
||||
text = content.get_text("\n", strip=True)
|
||||
out.append({"author": author, "date": date, "num": num, "text": text})
|
||||
return out
|
||||
|
||||
|
||||
def parse_tags(soup: BeautifulSoup) -> list:
|
||||
"""Теги из шапки темы."""
|
||||
return [t.text.strip() for t in soup.select("li.tagItem a")]
|
||||
Reference in New Issue
Block a user