refactor: vwts_scraper — proper Python package structure

- Split single file into 8 focused modules (399 lines total):
  config.py — constants
  fetch.py — HTTP GET with retries
  paginate.py — page_count + HEAD verification
  parse.py — HTML parsing (topics, posts, tags)
  state.py — per-section state.json
  output.py — JSONL with file rotation
  run.py — main orchestration loop
  __main__.py — CLI entry point
- Run: python -m vwts_scraper <URL>
- Page count method fully tested (13 sections, 100+ topics, 0 errors)
- Resume support (Ctrl+C → state saved)
- Per-section state files (no conflicts between sections)
This commit is contained in:
2026-06-04 08:55:23 +03:00
parent 9f547d8d9e
commit 010b10c43e
12 changed files with 405 additions and 1 deletions
+41
View File
@@ -0,0 +1,41 @@
"""Сохранение темы в JSONL. Ротация каждые TOPICS_PER_FILE тем."""
import json, logging
from datetime import datetime
from pathlib import Path
from .config import TOPICS_PER_FILE
OUTPUT_DIR = Path("vwts_data")
log = logging.getLogger(__name__)
def save_topic(section_slug: str, section_name: str, topic: dict,
posts: list, tags: list, state: dict):
"""Записать тему в part_XXXX.jsonl. Ротирует файл каждые N тем."""
st = state
# ротация
if st["topic_count"] > 0 and st["topic_count"] % TOPICS_PER_FILE == 0:
st["file_index"] += 1
record = {
"section": section_name,
"topic_id": topic["id"],
"topic_title": topic["title"],
"topic_url": topic["url"],
"total_posts": len(posts),
"posts": posts,
"tags": tags,
"scraped_at": datetime.now().isoformat(),
}
d = OUTPUT_DIR / section_slug
d.mkdir(parents=True, exist_ok=True)
fpath = d / f"part_{st['file_index']:04d}.jsonl"
with open(fpath, "a", encoding="utf-8") as fh:
fh.write(json.dumps(record, ensure_ascii=False) + "\n")
st["topic_count"] += 1
log.info(f" 💾 part_{st['file_index']:04d}.jsonl "
f"({fpath.stat().st_size/1024/1024:.1f} MB)")