96 lines
3.0 KiB
Bash
Executable File
96 lines
3.0 KiB
Bash
Executable File
#!/bin/bash
|
|
set -e
|
|
|
|
WORKERS=4
|
|
if [ "$1" = "--1" ]; then
|
|
WORKERS=1
|
|
echo "Режим: 1 поток"
|
|
else
|
|
echo "Режим: $WORKERS потока"
|
|
fi
|
|
echo ""
|
|
|
|
SECTIONS=(
|
|
"https://vwts.ru/forum/vag/sistema-ohlazhdeniya-otoplenie-konditsionirovanie/"
|
|
"https://vwts.ru/forum/vag/sistema-smazki/"
|
|
"https://vwts.ru/forum/vag/sistemy-vpryska-i-zazhiganiya/"
|
|
"https://vwts.ru/forum/vag/podveska/"
|
|
"https://vwts.ru/forum/vag/rulevoe-upravlenie/"
|
|
"https://vwts.ru/forum/vag/tormoznaya-sistema/"
|
|
"https://vwts.ru/forum/vag/korobki-peredach-stseplenie/"
|
|
"https://vwts.ru/forum/vag/dokumentatsiya-po-volkswagen/"
|
|
"https://vwts.ru/forum/vag/baza-znaniy-diagnostika-i-neispravnosti/"
|
|
"https://vwts.ru/forum/vag/baza-znaniy-benzinovye-dvigateli/"
|
|
"https://vwts.ru/forum/vag/baza-znaniy-dizelnye-dvigateli/"
|
|
"https://vwts.ru/forum/vag/baza-znaniy-sistema-ohlazhdeniya/"
|
|
"https://vwts.ru/forum/vag/baza-znaniy-sistema-smazki/"
|
|
"https://vwts.ru/forum/vag/baza-znaniy-sistemy-vpryska-i-zazhiganiya/"
|
|
"https://vwts.ru/forum/vag/baza-znaniy-toplivnaya-sistema/"
|
|
"https://vwts.ru/forum/vag/baza-znaniy-vypusknaya-sistema/"
|
|
"https://vwts.ru/forum/vag/baza-znaniy-podveska/"
|
|
"https://vwts.ru/forum/vag/baza-znaniy-rulevoe-upravlenie/"
|
|
"https://vwts.ru/forum/vag/baza-znaniy-tormoznaya-sistema/"
|
|
"https://vwts.ru/forum/vag/baza-znaniy-elektrooborudovanie/"
|
|
"https://vwts.ru/forum/vag/elektronnye-sistemy-bloki-upravleniya-vag/"
|
|
"https://vwts.ru/forum/vag/baza-znaniy-korobki-peredach-stsepleniya/"
|
|
)
|
|
|
|
BANNED=false
|
|
TOTAL=${#SECTIONS[@]}
|
|
CURRENT=0
|
|
|
|
for URL in "${SECTIONS[@]}"; do
|
|
CURRENT=$((CURRENT + 1))
|
|
SLUG=$(echo "$URL" | sed 's|/$||' | awk -F/ '{print $NF}')
|
|
|
|
echo ""
|
|
echo "========================================"
|
|
echo "[$CURRENT/$TOTAL] Проверка бана для: $SLUG"
|
|
echo "========================================"
|
|
|
|
python3 << 'PYEOF'
|
|
import sys, logging, requests
|
|
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s', datefmt='%H:%M:%S')
|
|
log = logging.getLogger()
|
|
url = "'"$URL"'"
|
|
log.info('Проверка: 5 запросов...')
|
|
s = requests.Session()
|
|
s.headers.update({'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'})
|
|
ban = False
|
|
for i in range(5):
|
|
r = s.get(url, timeout=(10, 30))
|
|
log.info(' [%d/5] HTTP %d' % (i+1, r.status_code))
|
|
if r.status_code in (403, 429):
|
|
ban = True
|
|
break
|
|
s.close()
|
|
sys.exit(1 if ban else 0)
|
|
PYEOF
|
|
|
|
EXIT=$?
|
|
|
|
if [ $EXIT -eq 1 ]; then
|
|
echo ""
|
|
echo "=== БАН! Многопоточный режим не работает. ==="
|
|
echo "Перезапусти с --1 для 1 потока"
|
|
BANNED=true
|
|
break
|
|
fi
|
|
|
|
echo ""
|
|
echo "========================================"
|
|
echo "[$CURRENT/$TOTAL] Загрузка: $SLUG"
|
|
echo "========================================"
|
|
echo ""
|
|
|
|
python3 -m vwts_scraper "$URL" --workers $WORKERS
|
|
|
|
echo ""
|
|
echo "OK [$CURRENT/$TOTAL] $SLUG готов"
|
|
echo ""
|
|
done
|
|
|
|
if [ "$BANNED" = false ]; then
|
|
echo "=== ВСЕ РАЗДЕЛЫ ЗАГРУЖЕНЫ ==="
|
|
fi
|