refactor: vwts_scraper — proper Python package structure
- Split single file into 8 focused modules (399 lines total): config.py — constants fetch.py — HTTP GET with retries paginate.py — page_count + HEAD verification parse.py — HTML parsing (topics, posts, tags) state.py — per-section state.json output.py — JSONL with file rotation run.py — main orchestration loop __main__.py — CLI entry point - Run: python -m vwts_scraper <URL> - Page count method fully tested (13 sections, 100+ topics, 0 errors) - Resume support (Ctrl+C → state saved) - Per-section state files (no conflicts between sections)
This commit is contained in:
@@ -0,0 +1,48 @@
|
||||
"""Точка входа.
|
||||
|
||||
Использование:
|
||||
python -m vwts_scraper https://vwts.ru/forum/vag/benzinovye-dvigateli/
|
||||
|
||||
Пауза: Ctrl+C → состояние сохраняется.
|
||||
Продолжить: та же команда.
|
||||
"""
|
||||
|
||||
import sys, logging
|
||||
from pathlib import Path
|
||||
|
||||
# Настройка логирования
|
||||
OUTPUT_DIR = Path("vwts_data")
|
||||
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||
datefmt="%H:%M:%S",
|
||||
handlers=[
|
||||
logging.StreamHandler(sys.stdout),
|
||||
logging.FileHandler(OUTPUT_DIR / "scraper.log", encoding="utf-8"),
|
||||
],
|
||||
)
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2:
|
||||
print("❌ Укажи URL раздела. Пример:")
|
||||
print(" python -m vwts_scraper "
|
||||
"https://vwts.ru/forum/vag/benzinovye-dvigateli/")
|
||||
sys.exit(1)
|
||||
|
||||
from .run import run
|
||||
section_url = sys.argv[1].rstrip("/") + "/"
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
log.info("=" * 60)
|
||||
log.info(f"🚀 vwts.ru — раздел: {section_url.split('/')[-2]}")
|
||||
log.info(f"📂 {OUTPUT_DIR}/{section_url.split('/')[-2]}/part_*.jsonl")
|
||||
log.info("🛑 Ctrl+C = пауза | Повторить команду = продолжить")
|
||||
log.info("=" * 60)
|
||||
|
||||
try:
|
||||
run(section_url)
|
||||
except KeyboardInterrupt:
|
||||
log.info("\n🛑 ПАУЗА. Продолжить: та же команда.")
|
||||
sys.exit(0)
|
||||
Reference in New Issue
Block a user