naeel
010b10c43e
- Split single file into 8 focused modules (399 lines total): config.py — constants fetch.py — HTTP GET with retries paginate.py — page_count + HEAD verification parse.py — HTML parsing (topics, posts, tags) state.py — per-section state.json output.py — JSONL with file rotation run.py — main orchestration loop __main__.py — CLI entry point - Run: python -m vwts_scraper <URL> - Page count method fully tested (13 sections, 100+ topics, 0 errors) - Resume support (Ctrl+C → state saved) - Per-section state files (no conflicts between sections)
48 lines
1.6 KiB
Python
48 lines
1.6 KiB
Python
"""Парсинг HTML: темы, посты, теги."""
|
|
|
|
import re
|
|
from bs4 import BeautifulSoup
|
|
|
|
|
|
def parse_topics(soup: BeautifulSoup) -> list:
|
|
"""Список тем со страницы раздела."""
|
|
items = []
|
|
for div in soup.select("div.structItem"):
|
|
a = div.select_one("div.structItem-title a")
|
|
if not a:
|
|
continue
|
|
m = re.search(r"/topic/(\d+)/", a.get("href", ""))
|
|
if not m:
|
|
continue
|
|
items.append({
|
|
"id": int(m.group(1)),
|
|
"title": a.text.strip(),
|
|
"url": f"https://vwts.ru/forum/topic/{m.group(1)}/"
|
|
})
|
|
return items
|
|
|
|
|
|
def parse_posts(soup: BeautifulSoup) -> list:
|
|
"""Посты с одной страницы темы."""
|
|
out = []
|
|
for art in soup.select("article.message"):
|
|
aname = art.select_one("h4.message-name")
|
|
author = aname.get_text(strip=True) if aname else "?"
|
|
t = art.select_one("time.u-dt")
|
|
date = t.get("datetime", "") if t else ""
|
|
n = art.select_one("ul.message-attribution-opposite a")
|
|
num = n.text.strip() if n else ""
|
|
content = art.select_one("div.message-content")
|
|
text = ""
|
|
if content:
|
|
for h in content.select("div[style*='none'], span[style*='none'], details"):
|
|
h.decompose()
|
|
text = content.get_text("\n", strip=True)
|
|
out.append({"author": author, "date": date, "num": num, "text": text})
|
|
return out
|
|
|
|
|
|
def parse_tags(soup: BeautifulSoup) -> list:
|
|
"""Теги из шапки темы."""
|
|
return [t.text.strip() for t in soup.select("li.tagItem a")]
|