naeel
010b10c43e
- Split single file into 8 focused modules (399 lines total): config.py — constants fetch.py — HTTP GET with retries paginate.py — page_count + HEAD verification parse.py — HTML parsing (topics, posts, tags) state.py — per-section state.json output.py — JSONL with file rotation run.py — main orchestration loop __main__.py — CLI entry point - Run: python -m vwts_scraper <URL> - Page count method fully tested (13 sections, 100+ topics, 0 errors) - Resume support (Ctrl+C → state saved) - Per-section state files (no conflicts between sections)
34 lines
1.1 KiB
Python
34 lines
1.1 KiB
Python
"""HTTP-запросы: GET с ретраями, 404 без ретрая."""
|
||
|
||
import time, logging
|
||
from bs4 import BeautifulSoup
|
||
import requests
|
||
from .config import DELAY, TIMEOUT, HEADERS
|
||
|
||
log = logging.getLogger(__name__)
|
||
|
||
session = requests.Session()
|
||
session.headers.update(HEADERS)
|
||
|
||
|
||
def get(url: str) -> BeautifulSoup | None:
|
||
"""GET + ретраи (до 3). 404 — без ретрая, сразу None."""
|
||
time.sleep(DELAY)
|
||
for n in range(3):
|
||
try:
|
||
r = session.get(url, timeout=TIMEOUT)
|
||
if r.status_code == 404:
|
||
log.warning(f" ⏭️ 404: {url[:80]}")
|
||
return None
|
||
if r.status_code in (403, 429, 503):
|
||
log.warning(f"⚠️ HTTP {r.status_code} — жду {10*(n+1)}с")
|
||
time.sleep(10 * (n + 1))
|
||
continue
|
||
r.raise_for_status()
|
||
return BeautifulSoup(r.text, "lxml")
|
||
except Exception as e:
|
||
log.warning(f"⚠️ Попытка {n+1}/3: {e}")
|
||
time.sleep(5)
|
||
log.error(f"❌ Не загружено: {url[:80]}")
|
||
return None
|