45 lines
1.2 KiB
Python
45 lines
1.2 KiB
Python
"""HTTP-запросы: GET с ретраями."""
|
||
|
||
import time, logging
|
||
from bs4 import BeautifulSoup
|
||
import requests
|
||
from .config import DELAY, TIMEOUT, HEADERS
|
||
|
||
log = logging.getLogger(__name__)
|
||
|
||
throttle = None
|
||
|
||
|
||
def make_session() -> requests.Session:
|
||
s = requests.Session()
|
||
s.headers.update(HEADERS)
|
||
return s
|
||
|
||
|
||
def get(url: str, session: requests.Session = None) -> BeautifulSoup | None:
|
||
"""GET + ретраи (до 3)."""
|
||
if session is None:
|
||
session = make_session()
|
||
if throttle:
|
||
throttle.wait()
|
||
else:
|
||
time.sleep(DELAY)
|
||
|
||
for n in range(3):
|
||
try:
|
||
r = session.get(url, timeout=TIMEOUT)
|
||
if r.status_code == 404:
|
||
log.warning(f" ⏭️ 404: {url[:80]}")
|
||
return None
|
||
if r.status_code in (403, 429, 503):
|
||
log.warning(f"⚠️ HTTP {r.status_code} — жду {10*(n+1)}с")
|
||
time.sleep(10 * (n + 1))
|
||
continue
|
||
r.raise_for_status()
|
||
return BeautifulSoup(r.text, "lxml")
|
||
except Exception as e:
|
||
log.warning(f"⚠️ Попытка {n+1}/3: {e}")
|
||
time.sleep(5)
|
||
log.error(f"❌ Не загружено: {url[:80]}")
|
||
return None
|