naeel
010b10c43e
- Split single file into 8 focused modules (399 lines total): config.py — constants fetch.py — HTTP GET with retries paginate.py — page_count + HEAD verification parse.py — HTML parsing (topics, posts, tags) state.py — per-section state.json output.py — JSONL with file rotation run.py — main orchestration loop __main__.py — CLI entry point - Run: python -m vwts_scraper <URL> - Page count method fully tested (13 sections, 100+ topics, 0 errors) - Resume support (Ctrl+C → state saved) - Per-section state files (no conflicts between sections)
42 lines
1.3 KiB
Python
42 lines
1.3 KiB
Python
"""Сохранение темы в JSONL. Ротация каждые TOPICS_PER_FILE тем."""
|
|
|
|
import json, logging
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from .config import TOPICS_PER_FILE
|
|
|
|
OUTPUT_DIR = Path("vwts_data")
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
def save_topic(section_slug: str, section_name: str, topic: dict,
|
|
posts: list, tags: list, state: dict):
|
|
"""Записать тему в part_XXXX.jsonl. Ротирует файл каждые N тем."""
|
|
st = state
|
|
|
|
# ротация
|
|
if st["topic_count"] > 0 and st["topic_count"] % TOPICS_PER_FILE == 0:
|
|
st["file_index"] += 1
|
|
|
|
record = {
|
|
"section": section_name,
|
|
"topic_id": topic["id"],
|
|
"topic_title": topic["title"],
|
|
"topic_url": topic["url"],
|
|
"total_posts": len(posts),
|
|
"posts": posts,
|
|
"tags": tags,
|
|
"scraped_at": datetime.now().isoformat(),
|
|
}
|
|
|
|
d = OUTPUT_DIR / section_slug
|
|
d.mkdir(parents=True, exist_ok=True)
|
|
fpath = d / f"part_{st['file_index']:04d}.jsonl"
|
|
|
|
with open(fpath, "a", encoding="utf-8") as fh:
|
|
fh.write(json.dumps(record, ensure_ascii=False) + "\n")
|
|
|
|
st["topic_count"] += 1
|
|
log.info(f" 💾 part_{st['file_index']:04d}.jsonl "
|
|
f"({fpath.stat().st_size/1024/1024:.1f} MB)")
|