refactor: vwts_scraper — proper Python package structure
- Split single file into 8 focused modules (399 lines total): config.py — constants fetch.py — HTTP GET with retries paginate.py — page_count + HEAD verification parse.py — HTML parsing (topics, posts, tags) state.py — per-section state.json output.py — JSONL with file rotation run.py — main orchestration loop __main__.py — CLI entry point - Run: python -m vwts_scraper <URL> - Page count method fully tested (13 sections, 100+ topics, 0 errors) - Resume support (Ctrl+C → state saved) - Per-section state files (no conflicts between sections)
This commit is contained in:
@@ -0,0 +1,24 @@
|
||||
"""Управление состоянием (state.json в папке раздела)."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
OUTPUT_DIR = Path("vwts_data")
|
||||
|
||||
|
||||
def _state_file(section_slug: str) -> Path:
|
||||
d = OUTPUT_DIR / section_slug
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
return d / "state.json"
|
||||
|
||||
|
||||
def load(section_slug: str) -> dict:
|
||||
f = _state_file(section_slug)
|
||||
if f.exists():
|
||||
return json.loads(f.read_text(encoding="utf-8"))
|
||||
return {"topics_done": {}, "pages_done": [], "file_index": 0, "topic_count": 0}
|
||||
|
||||
|
||||
def save(section_slug: str, st: dict):
|
||||
_state_file(section_slug).write_text(
|
||||
json.dumps(st, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
Reference in New Issue
Block a user