lada_scraper: парсер phpBB форума нива-лада.рф + доку

lada_scraper/fetch.py

@@ -0,0 +1,44 @@
+"""HTTP-запросы: GET с ретраями."""
+
+import time, logging
+from bs4 import BeautifulSoup
+import requests
+from .config import DELAY, TIMEOUT, HEADERS
+
+log = logging.getLogger(__name__)
+
+throttle = None
+
+
+def make_session() -> requests.Session:
+    s = requests.Session()
+    s.headers.update(HEADERS)
+    return s
+
+
+def get(url: str, session: requests.Session = None) -> BeautifulSoup | None:
+    """GET + ретраи (до 3)."""
+    if session is None:
+        session = make_session()
+    if throttle:
+        throttle.wait()
+    else:
+        time.sleep(DELAY)
+
+    for n in range(3):
+        try:
+            r = session.get(url, timeout=TIMEOUT)
+            if r.status_code == 404:
+                log.warning(f"   ⏭️  404: {url[:80]}")
+                return None
+            if r.status_code in (403, 429, 503):
+                log.warning(f"⚠️  HTTP {r.status_code} — жду {10*(n+1)}с")
+                time.sleep(10 * (n + 1))
+                continue
+            r.raise_for_status()
+            return BeautifulSoup(r.text, "lxml")
+        except Exception as e:
+            log.warning(f"⚠️  Попытка {n+1}/3: {e}")
+            time.sleep(5)
+    log.error(f"❌ Не загружено: {url[:80]}")
+    return None