naeel
1cfa6d1595
- paginate.py: max(nums) on empty set → guard - paginate.py: HEAD fallback to GET if server doesn't support HEAD - output.py: disk health check every 500 topics - run.py: avoid page-1 extra redirect - __main__.py: URL validation (must contain vwts.ru)
49 lines
1.5 KiB
Python
49 lines
1.5 KiB
Python
"""Точка входа.
|
|
|
|
Использование:
|
|
python -m vwts_scraper https://vwts.ru/forum/vag/benzinovye-dvigateli/
|
|
|
|
Пауза: Ctrl+C → состояние сохраняется.
|
|
Продолжить: та же команда.
|
|
"""
|
|
|
|
import sys, logging
|
|
from pathlib import Path
|
|
|
|
# Настройка логирования
|
|
OUTPUT_DIR = Path("vwts_data")
|
|
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s [%(levelname)s] %(message)s",
|
|
datefmt="%H:%M:%S",
|
|
handlers=[
|
|
logging.StreamHandler(sys.stdout),
|
|
logging.FileHandler(OUTPUT_DIR / "scraper.log", encoding="utf-8"),
|
|
],
|
|
)
|
|
|
|
if __name__ == "__main__":
|
|
if len(sys.argv) < 2 or "vwts.ru" not in sys.argv[1]:
|
|
print("❌ Укажи URL раздела vwts.ru. Пример:")
|
|
print(" python -m vwts_scraper "
|
|
"https://vwts.ru/forum/vag/benzinovye-dvigateli/")
|
|
sys.exit(1)
|
|
|
|
from .run import run
|
|
section_url = sys.argv[1].rstrip("/") + "/"
|
|
|
|
log = logging.getLogger(__name__)
|
|
log.info("=" * 60)
|
|
log.info(f"🚀 vwts.ru — раздел: {section_url.split('/')[-2]}")
|
|
log.info(f"📂 {OUTPUT_DIR}/{section_url.split('/')[-2]}/part_*.jsonl")
|
|
log.info("🛑 Ctrl+C = пауза | Повторить команду = продолжить")
|
|
log.info("=" * 60)
|
|
|
|
try:
|
|
run(section_url)
|
|
except KeyboardInterrupt:
|
|
log.info("\n🛑 ПАУЗА. Продолжить: та же команда.")
|
|
sys.exit(0)
|