refactor: vwts_scraper — proper Python package structure
- Split single file into 8 focused modules (399 lines total): config.py — constants fetch.py — HTTP GET with retries paginate.py — page_count + HEAD verification parse.py — HTML parsing (topics, posts, tags) state.py — per-section state.json output.py — JSONL with file rotation run.py — main orchestration loop __main__.py — CLI entry point - Run: python -m vwts_scraper <URL> - Page count method fully tested (13 sections, 100+ topics, 0 errors) - Resume support (Ctrl+C → state saved) - Per-section state files (no conflicts between sections)
This commit is contained in:
@@ -0,0 +1,33 @@
|
||||
"""HTTP-запросы: GET с ретраями, 404 без ретрая."""
|
||||
|
||||
import time, logging
|
||||
from bs4 import BeautifulSoup
|
||||
import requests
|
||||
from .config import DELAY, TIMEOUT, HEADERS
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
session = requests.Session()
|
||||
session.headers.update(HEADERS)
|
||||
|
||||
|
||||
def get(url: str) -> BeautifulSoup | None:
|
||||
"""GET + ретраи (до 3). 404 — без ретрая, сразу None."""
|
||||
time.sleep(DELAY)
|
||||
for n in range(3):
|
||||
try:
|
||||
r = session.get(url, timeout=TIMEOUT)
|
||||
if r.status_code == 404:
|
||||
log.warning(f" ⏭️ 404: {url[:80]}")
|
||||
return None
|
||||
if r.status_code in (403, 429, 503):
|
||||
log.warning(f"⚠️ HTTP {r.status_code} — жду {10*(n+1)}с")
|
||||
time.sleep(10 * (n + 1))
|
||||
continue
|
||||
r.raise_for_status()
|
||||
return BeautifulSoup(r.text, "lxml")
|
||||
except Exception as e:
|
||||
log.warning(f"⚠️ Попытка {n+1}/3: {e}")
|
||||
time.sleep(5)
|
||||
log.error(f"❌ Не загружено: {url[:80]}")
|
||||
return None
|
||||
Reference in New Issue
Block a user