naeel
010b10c43e
- Split single file into 8 focused modules (399 lines total): config.py — constants fetch.py — HTTP GET with retries paginate.py — page_count + HEAD verification parse.py — HTML parsing (topics, posts, tags) state.py — per-section state.json output.py — JSONL with file rotation run.py — main orchestration loop __main__.py — CLI entry point - Run: python -m vwts_scraper <URL> - Page count method fully tested (13 sections, 100+ topics, 0 errors) - Resume support (Ctrl+C → state saved) - Per-section state files (no conflicts between sections)
25 lines
698 B
Python
25 lines
698 B
Python
"""Управление состоянием (state.json в папке раздела)."""
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
OUTPUT_DIR = Path("vwts_data")
|
|
|
|
|
|
def _state_file(section_slug: str) -> Path:
|
|
d = OUTPUT_DIR / section_slug
|
|
d.mkdir(parents=True, exist_ok=True)
|
|
return d / "state.json"
|
|
|
|
|
|
def load(section_slug: str) -> dict:
|
|
f = _state_file(section_slug)
|
|
if f.exists():
|
|
return json.loads(f.read_text(encoding="utf-8"))
|
|
return {"topics_done": {}, "pages_done": [], "file_index": 0, "topic_count": 0}
|
|
|
|
|
|
def save(section_slug: str, st: dict):
|
|
_state_file(section_slug).write_text(
|
|
json.dumps(st, ensure_ascii=False, indent=2), encoding="utf-8")
|